org.apache.flink.runtime.operators.util.DistributedRuntimeUDFContext Java Examples

The following examples show how to use org.apache.flink.runtime.operators.util.DistributedRuntimeUDFContext. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BatchTask.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
protected <X> void readAndSetBroadcastInput(int inputNum, String bcVarName, DistributedRuntimeUDFContext context, int superstep) throws IOException {
	
	if (LOG.isDebugEnabled()) {
		LOG.debug(formatLogString("Setting broadcast variable '" + bcVarName + "'" + 
			(superstep > 1 ? ", superstep " + superstep : "")));
	}
	
	@SuppressWarnings("unchecked")
	final TypeSerializerFactory<X> serializerFactory =  (TypeSerializerFactory<X>) this.broadcastInputSerializers[inputNum];
	
	final MutableReader<?> reader = this.broadcastInputReaders[inputNum];

	BroadcastVariableMaterialization<X, ?> variable = getEnvironment().getBroadcastVariableManager().materializeBroadcastVariable(bcVarName, superstep, this, reader, serializerFactory);
	context.setBroadcastVariable(bcVarName, variable);
}
Example #2
Source File: DataSourceTask.java    From flink with Apache License 2.0 5 votes vote down vote up 
public DistributedRuntimeUDFContext createRuntimeContext() {
	Environment env = getEnvironment();

	String sourceName =  getEnvironment().getTaskInfo().getTaskName().split("->")[0].trim();
	sourceName = sourceName.startsWith("CHAIN") ? sourceName.substring(6) : sourceName;

	return new DistributedRuntimeUDFContext(env.getTaskInfo(), getUserCodeClassLoader(),
			getExecutionConfig(), env.getDistributedCacheEntries(), env.getAccumulatorRegistry().getUserMap(), 
			getEnvironment().getMetricGroup().getOrAddOperator(sourceName), env.getExternalResourceInfoProvider());
}
Example #3
Source File: ChainedDriver.java    From flink with Apache License 2.0 5 votes vote down vote up 
public void setup(TaskConfig config, String taskName, Collector<OT> outputCollector,
		AbstractInvokable parent, ClassLoader userCodeClassLoader, ExecutionConfig executionConfig,
		Map<String, Accumulator<?,?>> accumulatorMap)
{
	this.config = config;
	this.taskName = taskName;
	this.userCodeClassLoader = userCodeClassLoader;
	this.metrics = parent.getEnvironment().getMetricGroup().getOrAddOperator(taskName);
	this.numRecordsIn = this.metrics.getIOMetricGroup().getNumRecordsInCounter();
	this.numRecordsOut = this.metrics.getIOMetricGroup().getNumRecordsOutCounter();
	this.outputCollector = new CountingCollector<>(outputCollector, numRecordsOut);

	Environment env = parent.getEnvironment();

	if (parent instanceof BatchTask) {
		this.udfContext = ((BatchTask<?, ?>) parent).createRuntimeContext(metrics);
	} else {
		this.udfContext = new DistributedRuntimeUDFContext(env.getTaskInfo(), userCodeClassLoader,
				parent.getExecutionConfig(), env.getDistributedCacheEntries(), accumulatorMap, metrics, env.getExternalResourceInfoProvider()
		);
	}

	this.executionConfig = executionConfig;
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	setup(parent);
}
Example #4
Source File: DataSinkTask.java    From flink with Apache License 2.0 5 votes vote down vote up 
public DistributedRuntimeUDFContext createRuntimeContext() {
	Environment env = getEnvironment();

	return new DistributedRuntimeUDFContext(env.getTaskInfo(), getUserCodeClassLoader(),
			getExecutionConfig(), env.getDistributedCacheEntries(), env.getAccumulatorRegistry().getUserMap(), 
			getEnvironment().getMetricGroup().getOrAddOperator(getEnvironment().getTaskInfo().getTaskName()), env.getExternalResourceInfoProvider());
}
Example #5
Source File: BatchTask.java    From flink with Apache License 2.0 5 votes vote down vote up 
protected void releaseBroadcastVariables(String bcVarName, int superstep, DistributedRuntimeUDFContext context) {
	if (LOG.isDebugEnabled()) {
		LOG.debug(formatLogString("Releasing broadcast variable '" + bcVarName + "'" + 
			(superstep > 1 ? ", superstep " + superstep : "")));
	}
	
	getEnvironment().getBroadcastVariableManager().releaseReference(bcVarName, superstep, this);
	context.clearBroadcastVariable(bcVarName);
}
Example #6
Source File: BatchTask.java    From flink with Apache License 2.0 5 votes vote down vote up 
protected <X> void readAndSetBroadcastInput(int inputNum, String bcVarName, DistributedRuntimeUDFContext context, int superstep) throws IOException {
	
	if (LOG.isDebugEnabled()) {
		LOG.debug(formatLogString("Setting broadcast variable '" + bcVarName + "'" + 
			(superstep > 1 ? ", superstep " + superstep : "")));
	}
	
	@SuppressWarnings("unchecked")
	final TypeSerializerFactory<X> serializerFactory =  (TypeSerializerFactory<X>) this.broadcastInputSerializers[inputNum];
	
	final MutableReader<?> reader = this.broadcastInputReaders[inputNum];

	BroadcastVariableMaterialization<X, ?> variable = getEnvironment().getBroadcastVariableManager().materializeBroadcastVariable(bcVarName, superstep, this, reader, serializerFactory);
	context.setBroadcastVariable(bcVarName, variable);
}
Example #7
Source File: AbstractIterativeTask.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public DistributedRuntimeUDFContext createRuntimeContext(MetricGroup metrics) {
	Environment env = getEnvironment();

	return new IterativeRuntimeUdfContext(env.getTaskInfo(), getUserCodeClassLoader(),
			getExecutionConfig(), env.getDistributedCacheEntries(), this.accumulatorMap, metrics, env.getExternalResourceInfoProvider());
}
Example #8
Source File: DataSourceTask.java    From flink with Apache License 2.0 5 votes vote down vote up 
public DistributedRuntimeUDFContext createRuntimeContext() {
	Environment env = getEnvironment();

	String sourceName =  getEnvironment().getTaskInfo().getTaskName().split("->")[0].trim();
	sourceName = sourceName.startsWith("CHAIN") ? sourceName.substring(6) : sourceName;
	return new DistributedRuntimeUDFContext(env.getTaskInfo(), getUserCodeClassLoader(),
			getExecutionConfig(), env.getDistributedCacheEntries(), env.getAccumulatorRegistry().getUserMap(), 
			getEnvironment().getMetricGroup().getOrAddOperator(sourceName));
}
Example #9
Source File: ChainedDriver.java    From flink with Apache License 2.0 5 votes vote down vote up 
public void setup(TaskConfig config, String taskName, Collector<OT> outputCollector,
		AbstractInvokable parent, ClassLoader userCodeClassLoader, ExecutionConfig executionConfig,
		Map<String, Accumulator<?,?>> accumulatorMap)
{
	this.config = config;
	this.taskName = taskName;
	this.userCodeClassLoader = userCodeClassLoader;
	this.metrics = parent.getEnvironment().getMetricGroup().getOrAddOperator(taskName);
	this.numRecordsIn = this.metrics.getIOMetricGroup().getNumRecordsInCounter();
	this.numRecordsOut = this.metrics.getIOMetricGroup().getNumRecordsOutCounter();
	this.outputCollector = new CountingCollector<>(outputCollector, numRecordsOut);

	Environment env = parent.getEnvironment();

	if (parent instanceof BatchTask) {
		this.udfContext = ((BatchTask<?, ?>) parent).createRuntimeContext(metrics);
	} else {
		this.udfContext = new DistributedRuntimeUDFContext(env.getTaskInfo(), userCodeClassLoader,
				parent.getExecutionConfig(), env.getDistributedCacheEntries(), accumulatorMap, metrics
		);
	}

	this.executionConfig = executionConfig;
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	setup(parent);
}
Example #10
Source File: DataSinkTask.java    From flink with Apache License 2.0 5 votes vote down vote up 
public DistributedRuntimeUDFContext createRuntimeContext() {
	Environment env = getEnvironment();

	return new DistributedRuntimeUDFContext(env.getTaskInfo(), getUserCodeClassLoader(),
			getExecutionConfig(), env.getDistributedCacheEntries(), env.getAccumulatorRegistry().getUserMap(), 
			getEnvironment().getMetricGroup().getOrAddOperator(getEnvironment().getTaskInfo().getTaskName()));
}
Example #11
Source File: BatchTask.java    From flink with Apache License 2.0 5 votes vote down vote up 
protected void releaseBroadcastVariables(String bcVarName, int superstep, DistributedRuntimeUDFContext context) {
	if (LOG.isDebugEnabled()) {
		LOG.debug(formatLogString("Releasing broadcast variable '" + bcVarName + "'" + 
			(superstep > 1 ? ", superstep " + superstep : "")));
	}
	
	getEnvironment().getBroadcastVariableManager().releaseReference(bcVarName, superstep, this);
	context.clearBroadcastVariable(bcVarName);
}
Example #12
Source File: BatchTask.java    From flink with Apache License 2.0 5 votes vote down vote up 
protected <X> void readAndSetBroadcastInput(int inputNum, String bcVarName, DistributedRuntimeUDFContext context, int superstep) throws IOException {
	
	if (LOG.isDebugEnabled()) {
		LOG.debug(formatLogString("Setting broadcast variable '" + bcVarName + "'" + 
			(superstep > 1 ? ", superstep " + superstep : "")));
	}
	
	@SuppressWarnings("unchecked")
	final TypeSerializerFactory<X> serializerFactory =  (TypeSerializerFactory<X>) this.broadcastInputSerializers[inputNum];
	
	final MutableReader<?> reader = this.broadcastInputReaders[inputNum];

	BroadcastVariableMaterialization<X, ?> variable = getEnvironment().getBroadcastVariableManager().materializeBroadcastVariable(bcVarName, superstep, this, reader, serializerFactory);
	context.setBroadcastVariable(bcVarName, variable);
}
Example #13
Source File: DataSourceTask.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public DistributedRuntimeUDFContext createRuntimeContext() {
	Environment env = getEnvironment();

	String sourceName =  getEnvironment().getTaskInfo().getTaskName().split("->")[0].trim();
	sourceName = sourceName.startsWith("CHAIN") ? sourceName.substring(6) : sourceName;
	return new DistributedRuntimeUDFContext(env.getTaskInfo(), getUserCodeClassLoader(),
			getExecutionConfig(), env.getDistributedCacheEntries(), env.getAccumulatorRegistry().getUserMap(), 
			getEnvironment().getMetricGroup().getOrAddOperator(sourceName));
}
Example #14
Source File: ChainedDriver.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public void setup(TaskConfig config, String taskName, Collector<OT> outputCollector,
		AbstractInvokable parent, ClassLoader userCodeClassLoader, ExecutionConfig executionConfig,
		Map<String, Accumulator<?,?>> accumulatorMap)
{
	this.config = config;
	this.taskName = taskName;
	this.userCodeClassLoader = userCodeClassLoader;
	this.metrics = parent.getEnvironment().getMetricGroup().getOrAddOperator(taskName);
	this.numRecordsIn = this.metrics.getIOMetricGroup().getNumRecordsInCounter();
	this.numRecordsOut = this.metrics.getIOMetricGroup().getNumRecordsOutCounter();
	this.outputCollector = new CountingCollector<>(outputCollector, numRecordsOut);

	Environment env = parent.getEnvironment();

	if (parent instanceof BatchTask) {
		this.udfContext = ((BatchTask<?, ?>) parent).createRuntimeContext(metrics);
	} else {
		this.udfContext = new DistributedRuntimeUDFContext(env.getTaskInfo(), userCodeClassLoader,
				parent.getExecutionConfig(), env.getDistributedCacheEntries(), accumulatorMap, metrics
		);
	}

	this.executionConfig = executionConfig;
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	setup(parent);
}
Example #15
Source File: DataSinkTask.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public DistributedRuntimeUDFContext createRuntimeContext() {
	Environment env = getEnvironment();

	return new DistributedRuntimeUDFContext(env.getTaskInfo(), getUserCodeClassLoader(),
			getExecutionConfig(), env.getDistributedCacheEntries(), env.getAccumulatorRegistry().getUserMap(), 
			getEnvironment().getMetricGroup().getOrAddOperator(getEnvironment().getTaskInfo().getTaskName()));
}
Example #16
Source File: BatchTask.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
protected void releaseBroadcastVariables(String bcVarName, int superstep, DistributedRuntimeUDFContext context) {
	if (LOG.isDebugEnabled()) {
		LOG.debug(formatLogString("Releasing broadcast variable '" + bcVarName + "'" + 
			(superstep > 1 ? ", superstep " + superstep : "")));
	}
	
	getEnvironment().getBroadcastVariableManager().releaseReference(bcVarName, superstep, this);
	context.clearBroadcastVariable(bcVarName);
}
Example #17
Source File: BatchTask.java    From flink with Apache License 2.0 4 votes vote down vote up 
public DistributedRuntimeUDFContext createRuntimeContext(MetricGroup metrics) {
	Environment env = getEnvironment();

	return new DistributedRuntimeUDFContext(env.getTaskInfo(), getUserCodeClassLoader(),
			getExecutionConfig(), env.getDistributedCacheEntries(), this.accumulatorMap, metrics);
}
Example #18
Source File: AbstractIterativeTask.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public DistributedRuntimeUDFContext createRuntimeContext(MetricGroup metrics) {
	Environment env = getEnvironment();
	return new IterativeRuntimeUdfContext(env.getTaskInfo(), getUserCodeClassLoader(),
			getExecutionConfig(), env.getDistributedCacheEntries(), this.accumulatorMap, metrics);
}
Example #19
Source File: BatchTask.java    From flink with Apache License 2.0 4 votes vote down vote up 
public DistributedRuntimeUDFContext createRuntimeContext(MetricGroup metrics) {
	Environment env = getEnvironment();

	return new DistributedRuntimeUDFContext(env.getTaskInfo(), getUserCodeClassLoader(),
			getExecutionConfig(), env.getDistributedCacheEntries(), this.accumulatorMap, metrics, env.getExternalResourceInfoProvider());
}
Example #20
Source File: BatchTask.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
public DistributedRuntimeUDFContext createRuntimeContext(MetricGroup metrics) {
	Environment env = getEnvironment();

	return new DistributedRuntimeUDFContext(env.getTaskInfo(), getUserCodeClassLoader(),
			getExecutionConfig(), env.getDistributedCacheEntries(), this.accumulatorMap, metrics);
}
Example #21
Source File: AbstractIterativeTask.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Override
public DistributedRuntimeUDFContext createRuntimeContext(MetricGroup metrics) {
	Environment env = getEnvironment();
	return new IterativeRuntimeUdfContext(env.getTaskInfo(), getUserCodeClassLoader(),
			getExecutionConfig(), env.getDistributedCacheEntries(), this.accumulatorMap, metrics);
}