org.apache.flink.shaded.guava18.com.google.common.cache.CacheBuilder Java Examples

@VisibleForTesting
CompletedOperationCache(final Ticker ticker) {
	completedOperations = CacheBuilder.newBuilder()
		.expireAfterWrite(COMPLETED_OPERATION_RESULT_CACHE_DURATION_SECONDS, TimeUnit.SECONDS)
		.removalListener((RemovalListener<K, ResultAccessTracker<R>>) removalNotification -> {
			if (removalNotification.wasEvicted()) {
				Preconditions.checkState(removalNotification.getKey() != null);
				Preconditions.checkState(removalNotification.getValue() != null);

				// When shutting down the cache, we wait until all results are accessed.
				// When a result gets evicted from the cache, it will not be possible to access
				// it any longer, and we might be in the process of shutting down, so we mark
				// the result as accessed to avoid waiting indefinitely.
				removalNotification.getValue().markAccessed();

				LOGGER.info("Evicted result with trigger id {} because its TTL of {}s has expired.",
					removalNotification.getKey().getTriggerId(),
					COMPLETED_OPERATION_RESULT_CACHE_DURATION_SECONDS);
			}
		})
		.ticker(ticker)
		.build();
}
 

protected AbstractTaskManagerFileHandler(
		@Nonnull GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		@Nonnull Time timeout,
		@Nonnull Map<String, String> responseHeaders,
		@Nonnull UntypedResponseMessageHeaders<EmptyRequestBody, M> untypedResponseMessageHeaders,
		@Nonnull GatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
		@Nonnull TransientBlobService transientBlobService,
		@Nonnull Time cacheEntryDuration) {
	super(leaderRetriever, timeout, responseHeaders, untypedResponseMessageHeaders);

	this.resourceManagerGatewayRetriever = Preconditions.checkNotNull(resourceManagerGatewayRetriever);

	this.transientBlobService = Preconditions.checkNotNull(transientBlobService);

	this.fileBlobKeys = CacheBuilder
		.newBuilder()
		.expireAfterWrite(cacheEntryDuration.toMilliseconds(), TimeUnit.MILLISECONDS)
		.removalListener(this::removeBlob)
		.build(
			new CacheLoader<ResourceID, CompletableFuture<TransientBlobKey>>() {
				@Override
				public CompletableFuture<TransientBlobKey> load(ResourceID resourceId) throws Exception {
					return loadTaskManagerFile(resourceId);
				}
		});
}
 

@VisibleForTesting
CompletedOperationCache(final Ticker ticker) {
	completedOperations = CacheBuilder.newBuilder()
		.expireAfterWrite(COMPLETED_OPERATION_RESULT_CACHE_DURATION_SECONDS, TimeUnit.SECONDS)
		.removalListener((RemovalListener<K, ResultAccessTracker<R>>) removalNotification -> {
			if (removalNotification.wasEvicted()) {
				Preconditions.checkState(removalNotification.getKey() != null);
				Preconditions.checkState(removalNotification.getValue() != null);

				// When shutting down the cache, we wait until all results are accessed.
				// When a result gets evicted from the cache, it will not be possible to access
				// it any longer, and we might be in the process of shutting down, so we mark
				// the result as accessed to avoid waiting indefinitely.
				removalNotification.getValue().markAccessed();

				LOGGER.info("Evicted result with trigger id {} because its TTL of {}s has expired.",
					removalNotification.getKey().getTriggerId(),
					COMPLETED_OPERATION_RESULT_CACHE_DURATION_SECONDS);
			}
		})
		.ticker(ticker)
		.build();
}
 

protected AbstractTaskManagerFileHandler(
		@Nonnull GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		@Nonnull Time timeout,
		@Nonnull Map<String, String> responseHeaders,
		@Nonnull UntypedResponseMessageHeaders<EmptyRequestBody, M> untypedResponseMessageHeaders,
		@Nonnull GatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
		@Nonnull TransientBlobService transientBlobService,
		@Nonnull Time cacheEntryDuration) {
	super(leaderRetriever, timeout, responseHeaders, untypedResponseMessageHeaders);

	this.resourceManagerGatewayRetriever = Preconditions.checkNotNull(resourceManagerGatewayRetriever);

	this.transientBlobService = Preconditions.checkNotNull(transientBlobService);

	this.fileBlobKeys = CacheBuilder
		.newBuilder()
		.expireAfterWrite(cacheEntryDuration.toMilliseconds(), TimeUnit.MILLISECONDS)
		.removalListener(this::removeBlob)
		.build(
			new CacheLoader<Tuple2<ResourceID, String>, CompletableFuture<TransientBlobKey>>() {
				@Override
				public CompletableFuture<TransientBlobKey> load(Tuple2<ResourceID, String> taskManagerIdAndFileName) throws Exception {
					return loadTaskManagerFile(taskManagerIdAndFileName);
				}
		});
}
 

protected AbstractTaskManagerFileHandler(
		@Nonnull GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		@Nonnull Time timeout,
		@Nonnull Map<String, String> responseHeaders,
		@Nonnull UntypedResponseMessageHeaders<EmptyRequestBody, M> untypedResponseMessageHeaders,
		@Nonnull GatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
		@Nonnull TransientBlobService transientBlobService,
		@Nonnull Time cacheEntryDuration) {
	super(leaderRetriever, timeout, responseHeaders, untypedResponseMessageHeaders);

	this.resourceManagerGatewayRetriever = Preconditions.checkNotNull(resourceManagerGatewayRetriever);

	this.transientBlobService = Preconditions.checkNotNull(transientBlobService);

	this.fileBlobKeys = CacheBuilder
		.newBuilder()
		.expireAfterWrite(cacheEntryDuration.toMilliseconds(), TimeUnit.MILLISECONDS)
		.removalListener(this::removeBlob)
		.build(
			new CacheLoader<ResourceID, CompletableFuture<TransientBlobKey>>() {
				@Override
				public CompletableFuture<TransientBlobKey> load(ResourceID resourceId) throws Exception {
					return loadTaskManagerFile(resourceId);
				}
		});
}
 

public AbstractClientProvider(Configuration properties) {
	this.stsRoleArn = properties.getString(BlinkOptions.STS.STS_ROLE_ARN);
	this.stsAccessId = properties.getString(BlinkOptions.STS.STS_ACCESS_ID);
	this.stsAccessKey = properties.getString(BlinkOptions.STS.STS_ACCESS_KEY);
	this.stsAssumeRoleFor = properties.getString(BlinkOptions.STS.STS_UID);
	this.stsSessionName = String.valueOf(System.currentTimeMillis());
	this.stsExpireSeconds = properties.getInteger(BlinkOptions.STS.STS_ROLEARN_UPDATE_SECONDS);
	useSts = true;
	synchronized (mutex) {
		if (this.cacheArnResponse == null) {
			this.cacheArnResponse = CacheBuilder.newBuilder().concurrencyLevel(5).initialCapacity(1).maximumSize(3)
												.expireAfterWrite(stsExpireSeconds, TimeUnit.SECONDS).build(
							new StsIdentityLoader(this, properties));
		}
	}
}
 

@VisibleForTesting
CompletedOperationCache(final Ticker ticker) {
	completedOperations = CacheBuilder.newBuilder()
		.expireAfterWrite(COMPLETED_OPERATION_RESULT_CACHE_DURATION_SECONDS, TimeUnit.SECONDS)
		.removalListener((RemovalListener<K, ResultAccessTracker<R>>) removalNotification -> {
			if (removalNotification.wasEvicted()) {
				Preconditions.checkState(removalNotification.getKey() != null);
				Preconditions.checkState(removalNotification.getValue() != null);

				// When shutting down the cache, we wait until all results are accessed.
				// When a result gets evicted from the cache, it will not be possible to access
				// it any longer, and we might be in the process of shutting down, so we mark
				// the result as accessed to avoid waiting indefinitely.
				removalNotification.getValue().markAccessed();

				LOGGER.info("Evicted result with trigger id {} because its TTL of {}s has expired.",
					removalNotification.getKey().getTriggerId(),
					COMPLETED_OPERATION_RESULT_CACHE_DURATION_SECONDS);
			}
		})
		.ticker(ticker)
		.build();
}
 

/**
 * Creates a back pressure statistics tracker.
 *
 * @param cleanUpInterval     Clean up interval for completed stats.
 * @param numSamples          Number of stack trace samples when determining back pressure.
 * @param delayBetweenSamples Delay between samples when determining back pressure.
 */
public BackPressureStatsTrackerImpl(
		StackTraceSampleCoordinator coordinator,
		int cleanUpInterval,
		int numSamples,
		int backPressureStatsRefreshInterval,
		Time delayBetweenSamples) {

	this.coordinator = checkNotNull(coordinator, "Stack trace sample coordinator");

	checkArgument(cleanUpInterval >= 0, "Clean up interval");
	this.cleanUpInterval = cleanUpInterval;

	checkArgument(numSamples >= 1, "Number of samples");
	this.numSamples = numSamples;

	checkArgument(
		backPressureStatsRefreshInterval >= 0,
		"backPressureStatsRefreshInterval must be greater than or equal to 0");
	this.backPressureStatsRefreshInterval = backPressureStatsRefreshInterval;

	this.delayBetweenSamples = checkNotNull(delayBetweenSamples, "Delay between samples");

	this.operatorStatsCache = CacheBuilder.newBuilder()
			.concurrencyLevel(1)
			.expireAfterAccess(cleanUpInterval, TimeUnit.MILLISECONDS)
			.build();
}
 

/**
 * Creates a back pressure statistics tracker.
 *
 * @param coordinator Coordinator for back pressure stats request.
 * @param cleanUpInterval Clean up interval for completed stats.
 * @param refreshInterval Time interval after which the available back pressure
 *                        stats are deprecated and need to be refreshed.
 */
public BackPressureStatsTrackerImpl(
		BackPressureRequestCoordinator coordinator,
		int cleanUpInterval,
		int refreshInterval) {
	checkArgument(cleanUpInterval >= 0, "The cleanup interval must be non-negative.");
	checkArgument(refreshInterval >= 0, "The back pressure stats refresh interval must be non-negative.");

	this.coordinator = checkNotNull(coordinator);
	this.backPressureStatsRefreshInterval = refreshInterval;
	this.operatorStatsCache = CacheBuilder.newBuilder()
			.concurrencyLevel(1)
			.expireAfterAccess(cleanUpInterval, TimeUnit.MILLISECONDS)
			.build();
}
 

public CheckpointStatsCache(int maxNumEntries) {
	if (maxNumEntries > 0) {
		this.cache = CacheBuilder.<Long, AbstractCheckpointStats>newBuilder()
			.maximumSize(maxNumEntries)
			.build();
	} else {
		this.cache = null;
	}
}
 

/**
 * Compiles a generated code to a Class.
 * @param cl the ClassLoader used to load the class
 * @param name  the class name
 * @param code  the generated code
 * @param <T>   the class type
 * @return  the compiled class
 */
@SuppressWarnings("unchecked")
public static <T> Class<T> compile(ClassLoader cl, String name, String code) {
	try {
		Cache<ClassLoader, Class> compiledClasses = COMPILED_CACHE.get(name,
				() -> CacheBuilder.newBuilder().maximumSize(5).weakKeys().softValues().build());
		return compiledClasses.get(cl, () -> doCompile(cl, name, code));
	} catch (Exception e) {
		throw new FlinkRuntimeException(e.getMessage(), e);
	}
}
 

@Override
public void open(FunctionContext context) throws Exception {
	try {
		establishConnectionAndStatement();
		this.cache = cacheMaxSize == -1 || cacheExpireMs == -1 ? null : CacheBuilder.newBuilder()
				.expireAfterWrite(cacheExpireMs, TimeUnit.MILLISECONDS)
				.maximumSize(cacheMaxSize)
				.build();
	} catch (SQLException sqe) {
		throw new IllegalArgumentException("open() failed.", sqe);
	} catch (ClassNotFoundException cnfe) {
		throw new IllegalArgumentException("JDBC driver class not found.", cnfe);
	}
}
 

@Override
public void open(FunctionContext context) throws Exception {
	try {
		establishConnectionAndStatement();
		this.cache = cacheMaxSize == -1 || cacheExpireMs == -1 ? null : CacheBuilder.newBuilder()
			.expireAfterWrite(cacheExpireMs, TimeUnit.MILLISECONDS)
			.maximumSize(cacheMaxSize)
			.build();
	} catch (SQLException sqe) {
		throw new IllegalArgumentException("open() failed.", sqe);
	} catch (ClassNotFoundException cnfe) {
		throw new IllegalArgumentException("JDBC driver class not found.", cnfe);
	}
}
 

public JdbcAsyncJoin(JoinContext context, JdbcJoinConfig jdbcJoinConfig)
{
    this.config = jdbcJoinConfig;
    this.schema = context.getSchema();
    this.selectFields = context.getSelectFields();
    this.selectFieldCnt = selectFields.size();
    this.joinType = context.getJoinType();
    this.joinOnMapping = context.getJoinOnMapping();

    String where = context.getJoinOnMapping().values().stream().map(x -> x + " = ?").collect(Collectors.joining(" and "));
    List<String> batchFields = context.getSelectFields().stream().filter(SelectField::isBatchTableField)
            .map(SelectField::getFieldName).collect(Collectors.toList());

    String select = "select %s from %s where %s";

    String batchSelectFields = batchFields.isEmpty() ? "true" : String.join(",", batchFields);

    String jdbcTable = config.getQuery() != null && config.getQuery().trim().length() > 0
            ? "(" + config.getQuery() + ") as " + context.getBatchTable()
            : context.getBatchTable();

    this.sql = String.format(select, batchSelectFields, jdbcTable, where);

    logger.info("batch table join query is [{}]", sql);
    logger.info("join mapping is {}", context.getJoinOnMapping());

    this.cache = CacheBuilder.newBuilder()
            .maximumSize(jdbcJoinConfig.getCacheMaxNumber())   //max cache 1000 value
            .expireAfterWrite(jdbcJoinConfig.getCacheTime(), TimeUnit.SECONDS)
            //.expireAfterAccess(mysqlConfig.getCacheTime(), TimeUnit.SECONDS)  //
            .build();
}
 

public CheckpointStatsCache(int maxNumEntries) {
	if (maxNumEntries > 0) {
		this.cache = CacheBuilder.<Long, AbstractCheckpointStats>newBuilder()
			.maximumSize(maxNumEntries)
			.build();
	} else {
		this.cache = null;
	}
}
 

@Override
public void open(FunctionContext context) throws Exception {
	try {
		establishConnection();
		statement = dbConn.prepareStatement(query);
		this.cache = cacheMaxSize == -1 || cacheExpireMs == -1 ? null : CacheBuilder.newBuilder()
				.expireAfterWrite(cacheExpireMs, TimeUnit.MILLISECONDS)
				.maximumSize(cacheMaxSize)
				.build();
	} catch (SQLException sqe) {
		throw new IllegalArgumentException("open() failed.", sqe);
	} catch (ClassNotFoundException cnfe) {
		throw new IllegalArgumentException("JDBC driver class not found.", cnfe);
	}
}
 

/**
 * Creates a back pressure statistics tracker.
 *
 * @param cleanUpInterval     Clean up interval for completed stats.
 * @param numSamples          Number of stack trace samples when determining back pressure.
 * @param delayBetweenSamples Delay between samples when determining back pressure.
 */
public BackPressureStatsTrackerImpl(
		StackTraceSampleCoordinator coordinator,
		int cleanUpInterval,
		int numSamples,
		int backPressureStatsRefreshInterval,
		Time delayBetweenSamples) {

	this.coordinator = checkNotNull(coordinator, "Stack trace sample coordinator");

	checkArgument(cleanUpInterval >= 0, "Clean up interval");
	this.cleanUpInterval = cleanUpInterval;

	checkArgument(numSamples >= 1, "Number of samples");
	this.numSamples = numSamples;

	checkArgument(
		backPressureStatsRefreshInterval >= 0,
		"backPressureStatsRefreshInterval must be greater than or equal to 0");
	this.backPressureStatsRefreshInterval = backPressureStatsRefreshInterval;

	this.delayBetweenSamples = checkNotNull(delayBetweenSamples, "Delay between samples");

	this.operatorStatsCache = CacheBuilder.newBuilder()
			.concurrencyLevel(1)
			.expireAfterAccess(cleanUpInterval, TimeUnit.MILLISECONDS)
			.build();
}
 

public CheckpointStatsCache(int maxNumEntries) {
	if (maxNumEntries > 0) {
		this.cache = CacheBuilder.<Long, AbstractCheckpointStats>newBuilder()
			.maximumSize(maxNumEntries)
			.build();
	} else {
		this.cache = null;
	}
}
 

public FileArchivedExecutionGraphStore(
		File rootDir,
		Time expirationTime,
		long maximumCacheSizeBytes,
		ScheduledExecutor scheduledExecutor,
		Ticker ticker) throws IOException {

	final File storageDirectory = initExecutionGraphStorageDirectory(rootDir);

	LOG.info(
		"Initializing {}: Storage directory {}, expiration time {}, maximum cache size {} bytes.",
		FileArchivedExecutionGraphStore.class.getSimpleName(),
		storageDirectory,
		expirationTime.toMilliseconds(),
		maximumCacheSizeBytes);

	this.storageDir = Preconditions.checkNotNull(storageDirectory);
	Preconditions.checkArgument(
		storageDirectory.exists() && storageDirectory.isDirectory(),
		"The storage directory must exist and be a directory.");
	this.jobDetailsCache = CacheBuilder.newBuilder()
		.expireAfterWrite(expirationTime.toMilliseconds(), TimeUnit.MILLISECONDS)
		.removalListener(
			(RemovalListener<JobID, JobDetails>) notification -> deleteExecutionGraphFile(notification.getKey()))
		.ticker(ticker)
		.build();

	this.archivedExecutionGraphCache = CacheBuilder.newBuilder()
		.maximumWeight(maximumCacheSizeBytes)
		.weigher(this::calculateSize)
		.build(new CacheLoader<JobID, ArchivedExecutionGraph>() {
			@Override
			public ArchivedExecutionGraph load(JobID jobId) throws Exception {
				return loadExecutionGraph(jobId);
			}});

	this.cleanupFuture = scheduledExecutor.scheduleWithFixedDelay(
		jobDetailsCache::cleanUp,
		expirationTime.toMilliseconds(),
		expirationTime.toMilliseconds(),
		TimeUnit.MILLISECONDS);

	this.shutdownHook = ShutdownHookUtil.addShutdownHook(this, getClass().getSimpleName(), LOG);

	this.numFinishedJobs = 0;
	this.numFailedJobs = 0;
	this.numCanceledJobs = 0;
}
 

public FileArchivedExecutionGraphStore(
		File rootDir,
		Time expirationTime,
		int maximumCapacity,
		long maximumCacheSizeBytes,
		ScheduledExecutor scheduledExecutor,
		Ticker ticker) throws IOException {

	final File storageDirectory = initExecutionGraphStorageDirectory(rootDir);

	LOG.info(
		"Initializing {}: Storage directory {}, expiration time {}, maximum cache size {} bytes.",
		FileArchivedExecutionGraphStore.class.getSimpleName(),
		storageDirectory,
		expirationTime.toMilliseconds(),
		maximumCacheSizeBytes);

	this.storageDir = Preconditions.checkNotNull(storageDirectory);
	Preconditions.checkArgument(
		storageDirectory.exists() && storageDirectory.isDirectory(),
		"The storage directory must exist and be a directory.");
	this.jobDetailsCache = CacheBuilder.newBuilder()
		.expireAfterWrite(expirationTime.toMilliseconds(), TimeUnit.MILLISECONDS)
		.maximumSize(maximumCapacity)
		.removalListener(
			(RemovalListener<JobID, JobDetails>) notification -> deleteExecutionGraphFile(notification.getKey()))
		.ticker(ticker)
		.build();

	this.archivedExecutionGraphCache = CacheBuilder.newBuilder()
		.maximumWeight(maximumCacheSizeBytes)
		.weigher(this::calculateSize)
		.build(new CacheLoader<JobID, ArchivedExecutionGraph>() {
			@Override
			public ArchivedExecutionGraph load(JobID jobId) throws Exception {
				return loadExecutionGraph(jobId);
			}});

	this.cleanupFuture = scheduledExecutor.scheduleWithFixedDelay(
		jobDetailsCache::cleanUp,
		expirationTime.toMilliseconds(),
		expirationTime.toMilliseconds(),
		TimeUnit.MILLISECONDS);

	this.shutdownHook = ShutdownHookUtil.addShutdownHook(this, getClass().getSimpleName(), LOG);

	this.numFinishedJobs = 0;
	this.numFailedJobs = 0;
	this.numCanceledJobs = 0;
}
 

public FileArchivedExecutionGraphStore(
		File rootDir,
		Time expirationTime,
		long maximumCacheSizeBytes,
		ScheduledExecutor scheduledExecutor,
		Ticker ticker) throws IOException {

	final File storageDirectory = initExecutionGraphStorageDirectory(rootDir);

	LOG.info(
		"Initializing {}: Storage directory {}, expiration time {}, maximum cache size {} bytes.",
		FileArchivedExecutionGraphStore.class.getSimpleName(),
		storageDirectory,
		expirationTime.toMilliseconds(),
		maximumCacheSizeBytes);

	this.storageDir = Preconditions.checkNotNull(storageDirectory);
	Preconditions.checkArgument(
		storageDirectory.exists() && storageDirectory.isDirectory(),
		"The storage directory must exist and be a directory.");
	this.jobDetailsCache = CacheBuilder.newBuilder()
		.expireAfterWrite(expirationTime.toMilliseconds(), TimeUnit.MILLISECONDS)
		.removalListener(
			(RemovalListener<JobID, JobDetails>) notification -> deleteExecutionGraphFile(notification.getKey()))
		.ticker(ticker)
		.build();

	this.archivedExecutionGraphCache = CacheBuilder.newBuilder()
		.maximumWeight(maximumCacheSizeBytes)
		.weigher(this::calculateSize)
		.build(new CacheLoader<JobID, ArchivedExecutionGraph>() {
			@Override
			public ArchivedExecutionGraph load(JobID jobId) throws Exception {
				return loadExecutionGraph(jobId);
			}});

	this.cleanupFuture = scheduledExecutor.scheduleWithFixedDelay(
		jobDetailsCache::cleanUp,
		expirationTime.toMilliseconds(),
		expirationTime.toMilliseconds(),
		TimeUnit.MILLISECONDS);

	this.shutdownHook = ShutdownHookUtil.addShutdownHook(this, getClass().getSimpleName(), LOG);

	this.numFinishedJobs = 0;
	this.numFailedJobs = 0;
	this.numCanceledJobs = 0;
}