Java Code Examples for org.apache.flink.api.common.functions.RuntimeContext#getNumberOfParallelSubtasks()

private boolean shouldTaskFailForThisAttempt() {
	RuntimeContext runtimeContext = getRuntimeContext();
	int numSubtasks = runtimeContext.getNumberOfParallelSubtasks();
	int subtaskIdx = runtimeContext.getIndexOfThisSubtask();
	int attempt = runtimeContext.getAttemptNumber();
	return (attempt % numSubtasks) == subtaskIdx;
}
 

@Override
public void open(Configuration parameters) throws Exception {
    super.open(parameters);

    RuntimeContext context = getRuntimeContext();
    _parallelism = context.getNumberOfParallelSubtasks();
    _partition = context.getIndexOfThisSubtask() + 1;
}
 

@Override
public void open(Configuration parameters) throws Exception {
    super.open(parameters);

    RuntimeContext context = getRuntimeContext();
    _parallelism = context.getNumberOfParallelSubtasks();
    _partition = context.getIndexOfThisSubtask() + 1;

    // Get a shorter name. So FetchUrlsFunction -> (Select fetch status, ...
    // becomes FetchUrlsFunction.
    String taskName = context.getTaskName().replaceFirst(" -> .+", "");
    _executor = new ThreadedExecutor("Flink-crawler-" + taskName, _threadCount);
}
 

/**
 * Creates a rate limiter with the runtime context provided.
 * @param runtimeContext
 */
@Override
public void open(RuntimeContext runtimeContext) {
	this.runtimeContext = runtimeContext;
	localRateBytesPerSecond = globalRateBytesPerSecond / runtimeContext.getNumberOfParallelSubtasks();
	this.rateLimiter = RateLimiter.create(localRateBytesPerSecond);
}
 

@Override
public void open(Configuration parameters) throws Exception {
    super.open(parameters);

    RuntimeContext context = getRuntimeContext();
    _parallelism = context.getNumberOfParallelSubtasks();
    _partition = context.getIndexOfThisSubtask() + 1;
}
 

public LogDataFetcher(SourceFunction.SourceContext<T> sourceContext,
                      RuntimeContext context,
                      String project,
                      List<String> logstores,
                      Pattern logstorePattern,
                      Properties configProps,
                      LogDeserializationSchema<T> deserializer,
                      LogClientProxy logClient,
                      CheckpointMode checkpointMode,
                      ShardAssigner shardAssigner) {
    this.sourceContext = sourceContext;
    this.configProps = configProps;
    this.deserializer = deserializer;
    this.totalNumberOfSubtasks = context.getNumberOfParallelSubtasks();
    this.indexOfThisSubtask = context.getIndexOfThisSubtask();
    this.checkpointLock = sourceContext.getCheckpointLock();
    this.shardAssigner = shardAssigner;
    this.subscribedShardsState = new ArrayList<>();
    this.shardConsumersExecutor = createThreadPool(context.getTaskNameWithSubtasks());
    this.error = new AtomicReference<>();
    this.project = project;
    this.logClient = logClient;
    this.checkpointMode = checkpointMode;
    this.consumerGroup = configProps.getProperty(ConfigConstants.LOG_CONSUMERGROUP);
    if (checkpointMode == CheckpointMode.PERIODIC) {
        commitInterval = LogUtil.getCommitIntervalMs(configProps);
        checkArgument(commitInterval > 0,
                "Checkpoint commit interval must be positive: " + commitInterval);
        checkArgument(consumerGroup != null && !consumerGroup.isEmpty(),
                "Missing parameter: " + ConfigConstants.LOG_CONSUMERGROUP);
    }
    this.activeConsumers = new HashMap<>();
    this.logstores = logstores;
    this.logstorePattern = logstorePattern;
    this.subscribedLogstores = new HashSet<>();
}
 

@Override
public void open(Configuration parameters) throws Exception {
	RuntimeContext context = getRuntimeContext();
	format.configure(parameters);
	int indexInSubtaskGroup = context.getIndexOfThisSubtask();
	int currentNumberOfSubtasks = context.getNumberOfParallelSubtasks();
	format.open(indexInSubtaskGroup, currentNumberOfSubtasks);
}
 

@Override
public void open(Configuration parameters)
        throws Exception
{
    super.open(parameters);
    RuntimeContext context = getRuntimeContext();

    // get parallelism id
    int partitionId = (context.getNumberOfParallelSubtasks() > 0) ?
            (context.getIndexOfThisSubtask() + 1) : 0;

    realTimeSink.open(partitionId, 0);
}
 

@Override
public void open(Configuration parameters) throws Exception {
	RuntimeContext context = getRuntimeContext();
	format.configure(parameters);
	int indexInSubtaskGroup = context.getIndexOfThisSubtask();
	int currentNumberOfSubtasks = context.getNumberOfParallelSubtasks();
	format.open(indexInSubtaskGroup, currentNumberOfSubtasks);
}
 

@VisibleForTesting
protected KinesisDataFetcher(List<String> streams,
							SourceFunction.SourceContext<T> sourceContext,
							Object checkpointLock,
							RuntimeContext runtimeContext,
							Properties configProps,
							KinesisDeserializationSchema<T> deserializationSchema,
							KinesisShardAssigner shardAssigner,
							AssignerWithPeriodicWatermarks<T> periodicWatermarkAssigner,
							WatermarkTracker watermarkTracker,
							AtomicReference<Throwable> error,
							List<KinesisStreamShardState> subscribedShardsState,
							HashMap<String, String> subscribedStreamsToLastDiscoveredShardIds,
							FlinkKinesisProxyFactory kinesisProxyFactory) {
	this.streams = checkNotNull(streams);
	this.configProps = checkNotNull(configProps);
	this.sourceContext = checkNotNull(sourceContext);
	this.checkpointLock = checkNotNull(checkpointLock);
	this.runtimeContext = checkNotNull(runtimeContext);
	this.totalNumberOfConsumerSubtasks = runtimeContext.getNumberOfParallelSubtasks();
	this.indexOfThisConsumerSubtask = runtimeContext.getIndexOfThisSubtask();
	this.deserializationSchema = checkNotNull(deserializationSchema);
	this.shardAssigner = checkNotNull(shardAssigner);
	this.periodicWatermarkAssigner = periodicWatermarkAssigner;
	this.watermarkTracker = watermarkTracker;
	this.kinesisProxyFactory = checkNotNull(kinesisProxyFactory);
	this.kinesis = kinesisProxyFactory.create(configProps);

	this.consumerMetricGroup = runtimeContext.getMetricGroup()
		.addGroup(KinesisConsumerMetricConstants.KINESIS_CONSUMER_METRICS_GROUP);

	this.error = checkNotNull(error);
	this.subscribedShardsState = checkNotNull(subscribedShardsState);
	this.subscribedStreamsToLastDiscoveredShardIds = checkNotNull(subscribedStreamsToLastDiscoveredShardIds);

	this.shardConsumersExecutor =
		createShardConsumersThreadPool(runtimeContext.getTaskNameWithSubtasks());
	this.recordEmitter = createRecordEmitter(configProps);
}
 

/**
 * Creates a rate limiter with the runtime context provided.
 * @param runtimeContext
 */
@Override
public void open(RuntimeContext runtimeContext) {
	this.runtimeContext = runtimeContext;
	localRateBytesPerSecond = globalRateBytesPerSecond / runtimeContext.getNumberOfParallelSubtasks();
	this.rateLimiter = RateLimiter.create(localRateBytesPerSecond);
}
 

@Override
public void run(SourceContext<Long> sourceCtx) throws Exception {
	final Object checkpointLock = sourceCtx.getCheckpointLock();

	RuntimeContext runtimeCtx = getRuntimeContext();

	final long stepSize = runtimeCtx.getNumberOfParallelSubtasks();
	final long congruence = runtimeCtx.getIndexOfThisSubtask();
	final long toCollect = (end % stepSize > congruence) ? (end / stepSize + 1) : (end / stepSize);

	final File proceedFile = new File(coordinateDir, PROCEED_MARKER_FILE);
	boolean checkForProceedFile = true;

	while (isRunning && collected < toCollect) {
		// check if the proceed file exists (then we go full speed)
		// if not, we always recheck and sleep
		if (checkForProceedFile) {
			if (proceedFile.exists()) {
				checkForProceedFile = false;
			} else {
				// otherwise wait so that we make slow progress
				Thread.sleep(SLEEP_TIME);
			}
		}

		synchronized (checkpointLock) {
			sourceCtx.collect(collected * stepSize + congruence);
			collected++;
		}
	}
}
 

@VisibleForTesting
protected KinesisDataFetcher(List<String> streams,
							SourceFunction.SourceContext<T> sourceContext,
							Object checkpointLock,
							RuntimeContext runtimeContext,
							Properties configProps,
							KinesisDeserializationSchema<T> deserializationSchema,
							KinesisShardAssigner shardAssigner,
							AssignerWithPeriodicWatermarks<T> periodicWatermarkAssigner,
							WatermarkTracker watermarkTracker,
							AtomicReference<Throwable> error,
							List<KinesisStreamShardState> subscribedShardsState,
							HashMap<String, String> subscribedStreamsToLastDiscoveredShardIds,
							FlinkKinesisProxyFactory kinesisProxyFactory) {
	this.streams = checkNotNull(streams);
	this.configProps = checkNotNull(configProps);
	this.sourceContext = checkNotNull(sourceContext);
	this.checkpointLock = checkNotNull(checkpointLock);
	this.runtimeContext = checkNotNull(runtimeContext);
	this.totalNumberOfConsumerSubtasks = runtimeContext.getNumberOfParallelSubtasks();
	this.indexOfThisConsumerSubtask = runtimeContext.getIndexOfThisSubtask();
	this.deserializationSchema = checkNotNull(deserializationSchema);
	this.shardAssigner = checkNotNull(shardAssigner);
	this.periodicWatermarkAssigner = periodicWatermarkAssigner;
	this.watermarkTracker = watermarkTracker;
	this.kinesisProxyFactory = checkNotNull(kinesisProxyFactory);
	this.kinesis = kinesisProxyFactory.create(configProps);

	this.consumerMetricGroup = runtimeContext.getMetricGroup()
		.addGroup(KinesisConsumerMetricConstants.KINESIS_CONSUMER_METRICS_GROUP);

	this.error = checkNotNull(error);
	this.subscribedShardsState = checkNotNull(subscribedShardsState);
	this.subscribedStreamsToLastDiscoveredShardIds = checkNotNull(subscribedStreamsToLastDiscoveredShardIds);

	this.shardConsumersExecutor =
		createShardConsumersThreadPool(runtimeContext.getTaskNameWithSubtasks());
	this.recordEmitter = createRecordEmitter(configProps);
}
 

@Override
public void run(SourceContext<Long> sourceCtx) throws Exception {
	final Object checkpointLock = sourceCtx.getCheckpointLock();

	RuntimeContext runtimeCtx = getRuntimeContext();

	final long stepSize = runtimeCtx.getNumberOfParallelSubtasks();
	final long congruence = runtimeCtx.getIndexOfThisSubtask();
	final long toCollect = (end % stepSize > congruence) ? (end / stepSize + 1) : (end / stepSize);

	final File proceedFile = new File(coordinateDir, PROCEED_MARKER_FILE);
	boolean checkForProceedFile = true;

	while (isRunning && collected < toCollect) {
		// check if the proceed file exists (then we go full speed)
		// if not, we always recheck and sleep
		if (checkForProceedFile) {
			if (proceedFile.exists()) {
				checkForProceedFile = false;
			} else {
				// otherwise wait so that we make slow progress
				Thread.sleep(SLEEP_TIME);
			}
		}

		synchronized (checkpointLock) {
			sourceCtx.collect(collected * stepSize + congruence);
			collected++;
		}
	}
}
 

private boolean shouldTaskFailForThisAttempt() {
	RuntimeContext runtimeContext = getRuntimeContext();
	int numSubtasks = runtimeContext.getNumberOfParallelSubtasks();
	int subtaskIdx = runtimeContext.getIndexOfThisSubtask();
	int attempt = runtimeContext.getAttemptNumber();
	return (attempt % numSubtasks) == subtaskIdx;
}
 

@Override
public void open(
		String name,
		FunctionInitializationContext context,
		RuntimeContext runtimeContext) throws Exception {
	Preconditions.checkState(this.checkpointedState == null,
			"The " + getClass().getSimpleName() + " has already been initialized.");

	this.checkpointedState = context.getOperatorStateStore().getListState(
			new ListStateDescriptor<>(
					name + "-sequence-state",
					LongSerializer.INSTANCE));
	this.valuesToEmit = new ArrayDeque<>();
	if (context.isRestored()) {
		// upon restoring

		for (Long v : this.checkpointedState.get()) {
			this.valuesToEmit.add(v);
		}
	} else {
		// the first time the job is executed
		final int stepSize = runtimeContext.getNumberOfParallelSubtasks();
		final int taskIdx = runtimeContext.getIndexOfThisSubtask();
		final long congruence = start + taskIdx;

		long totalNoOfElements = Math.abs(end - start + 1);
		final int baseSize = safeDivide(totalNoOfElements, stepSize);
		final int toCollect = (totalNoOfElements % stepSize > taskIdx) ? baseSize + 1 : baseSize;

		for (long collected = 0; collected < toCollect; collected++) {
			this.valuesToEmit.add(collected * stepSize + congruence);
		}
	}
}
 

private boolean shouldTaskFailForThisAttempt() {
	RuntimeContext runtimeContext = getRuntimeContext();
	int numSubtasks = runtimeContext.getNumberOfParallelSubtasks();
	int subtaskIdx = runtimeContext.getIndexOfThisSubtask();
	int attempt = runtimeContext.getAttemptNumber();
	return (attempt % numSubtasks) == subtaskIdx;
}
 

@Override
public String toString() {
  RuntimeContext ctx = getRuntimeContext();

  return ctx.getTaskName() + " (" + ctx.getIndexOfThisSubtask() + "/" + ctx.getNumberOfParallelSubtasks() + ")";
}
 

@Override
public String toString() {
	RuntimeContext ctx = getRuntimeContext();

	return ctx.getTaskName() + " (" + ctx.getIndexOfThisSubtask() + "/" + ctx.getNumberOfParallelSubtasks() + ")";
}
 

@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	final RuntimeContext runtimeContext = getRuntimeContext();
	final int subtaskIdx = runtimeContext.getIndexOfThisSubtask();
	final int parallelism = runtimeContext.getNumberOfParallelSubtasks();
	final int maxParallelism = runtimeContext.getMaxNumberOfParallelSubtasks();

	ListStateDescriptor<Long> unionWatermarksStateDescriptor =
		new ListStateDescriptor<>("watermarks", Long.class);

	lastEventTimes = context.getOperatorStateStore().getUnionListState(unionWatermarksStateDescriptor);

	ListStateDescriptor<KeyRangeStates> stateDescriptor =
		new ListStateDescriptor<>("keyRanges", KeyRangeStates.class);

	snapshotKeyRanges = context.getOperatorStateStore().getListState(stateDescriptor);
	keyRanges = new ArrayList<>();

	if (context.isRestored()) {
		// restore key ranges from the snapshot
		for (KeyRangeStates keyRange : snapshotKeyRanges.get()) {
			keyRanges.add(keyRange);
		}

		// let event time start from the max of all event time progress across subtasks in the last execution
		for (Long lastEventTime : lastEventTimes.get()) {
			monotonousEventTime = Math.max(monotonousEventTime, lastEventTime);
		}
	} else {
		// determine the key ranges that belong to the subtask
		int rangeStartIdx = (subtaskIdx * maxParallelism) / parallelism;
		int rangeEndIdx = ((subtaskIdx + 1) * maxParallelism) / parallelism;

		for (int i = rangeStartIdx; i < rangeEndIdx; ++i) {

			int start = ((i * totalKeySpaceSize + maxParallelism - 1) / maxParallelism);
			int end = 1 + ((i + 1) * totalKeySpaceSize - 1) / maxParallelism;

			if (end - start > 0) {
				keyRanges.add(new KeyRangeStates(start, end));
			}
		}

		// fresh run; start from event time o
		monotonousEventTime = 0L;
	}
}