org.apache.flink.runtime.io.network.api.writer.RecordWriter Java Examples

private static <OUT> RecordWriter<SerializationDelegate<StreamRecord<OUT>>> createRecordWriter(
		StreamEdge edge,
		int outputIndex,
		Environment environment,
		String taskName,
		long bufferTimeout) {
	@SuppressWarnings("unchecked")
	StreamPartitioner<OUT> outputPartitioner = (StreamPartitioner<OUT>) edge.getPartitioner();

	LOG.debug("Using partitioner {} for output {} of task {}", outputPartitioner, outputIndex, taskName);

	ResultPartitionWriter bufferWriter = environment.getWriter(outputIndex);

	// we initialize the partitioner here with the number of key groups (aka max. parallelism)
	if (outputPartitioner instanceof ConfigurableStreamPartitioner) {
		int numKeyGroups = bufferWriter.getNumTargetKeyGroups();
		if (0 < numKeyGroups) {
			((ConfigurableStreamPartitioner) outputPartitioner).configure(numKeyGroups);
		}
	}

	RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output =
		RecordWriter.createRecordWriter(bufferWriter, outputPartitioner, bufferTimeout, taskName);
	output.setMetricGroup(environment.getMetricGroup().getIOMetricGroup());
	return output;
}
 

@Override
protected void initOutputs() throws Exception {
	// initialize the regular outputs first (the ones into the step function).
	super.initOutputs();

	// at this time, the outputs to the step function are created
	// add the outputs for the final solution
	List<RecordWriter<?>> finalOutputWriters = new ArrayList<RecordWriter<?>>();
	final TaskConfig finalOutConfig = this.config.getIterationHeadFinalOutputConfig();
	final ClassLoader userCodeClassLoader = getUserCodeClassLoader();
	this.finalOutputCollector = BatchTask.getOutputCollector(this, finalOutConfig,
			userCodeClassLoader, finalOutputWriters, config.getNumOutputs(), finalOutConfig.getNumOutputs());

	// sanity check the setup
	final int writersIntoStepFunction = this.eventualOutputs.size();
	final int writersIntoFinalResult = finalOutputWriters.size();
	final int syncGateIndex = this.config.getIterationHeadIndexOfSyncOutput();

	if (writersIntoStepFunction + writersIntoFinalResult != syncGateIndex) {
		throw new Exception("Error: Inconsistent head task setup - wrong mapping of output gates.");
	}
	// now, we can instantiate the sync gate
	this.toSync = new RecordWriter<IOReadableWritable>(getEnvironment().getWriter(syncGateIndex));
	this.toSyncPartitionId = getEnvironment().getWriter(syncGateIndex).getPartitionId();
}
 

private RecordWriterOutput<OUT> createStreamOutput(
		RecordWriter<SerializationDelegate<StreamRecord<OUT>>> recordWriter,
		StreamEdge edge,
		StreamConfig upStreamConfig,
		Environment taskEnvironment) {
	OutputTag sideOutputTag = edge.getOutputTag(); // OutputTag, return null if not sideOutput

	TypeSerializer outSerializer = null;

	if (edge.getOutputTag() != null) {
		// side output
		outSerializer = upStreamConfig.getTypeSerializerSideOut(
				edge.getOutputTag(), taskEnvironment.getUserClassLoader());
	} else {
		// main output
		outSerializer = upStreamConfig.getTypeSerializerOut(taskEnvironment.getUserClassLoader());
	}

	return new RecordWriterOutput<>(recordWriter, outSerializer, sideOutputTag, this);
}
 

@SuppressWarnings("unchecked")
public RecordWriterOutput(
		RecordWriter<SerializationDelegate<StreamRecord<OUT>>> recordWriter,
		TypeSerializer<OUT> outSerializer,
		OutputTag outputTag,
		StreamStatusProvider streamStatusProvider) {

	checkNotNull(recordWriter);
	this.outputTag = outputTag;
	// generic hack: cast the writer to generic Object type so we can use it
	// with multiplexed records and watermarks
	this.recordWriter = (RecordWriter<SerializationDelegate<StreamElement>>)
			(RecordWriter<?>) recordWriter;

	TypeSerializer<StreamElement> outRecordSerializer =
			new StreamElementSerializer<>(outSerializer);

	if (outSerializer != null) {
		serializationDelegate = new SerializationDelegate<StreamElement>(outRecordSerializer);
	}

	this.streamStatusProvider = checkNotNull(streamStatusProvider);
}
 

@Override
public void invoke() throws Exception {
	RecordWriter<IntValue> writer = new RecordWriterBuilder<IntValue>().build(getEnvironment().getWriter(0));
	final int numberOfTimesToSend = getTaskConfiguration().getInteger(CONFIG_KEY, 0);

	final IntValue subtaskIndex = new IntValue(
			getEnvironment().getTaskInfo().getIndexOfThisSubtask());

	try {
		for (int i = 0; i < numberOfTimesToSend; i++) {
			writer.emit(subtaskIndex);
		}
		writer.flushAll();
	}
	finally {
		writer.clearBuffers();
	}
}
 

@Override
public void invoke() throws Exception {
	final Object o = new Object();
	RecordWriter<IntValue> recordWriter = new RecordWriter<>(getEnvironment().getWriter(0));

	for (int i = 0; i < 1024; i++) {
		recordWriter.emit(new IntValue(42));
	}

	synchronized (o) {
		//noinspection InfiniteLoopStatement
		while (true) {
			o.wait();
		}
	}

}
 

private static <OUT> List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> createRecordWriters(
		StreamConfig configuration,
		Environment environment) {
	List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> recordWriters = new ArrayList<>();
	List<StreamEdge> outEdgesInOrder = configuration.getOutEdgesInOrder(environment.getUserClassLoader());
	Map<Integer, StreamConfig> chainedConfigs = configuration.getTransitiveChainedTaskConfigsWithSelf(environment.getUserClassLoader());

	for (int i = 0; i < outEdgesInOrder.size(); i++) {
		StreamEdge edge = outEdgesInOrder.get(i);
		recordWriters.add(
			createRecordWriter(
				edge,
				i,
				environment,
				environment.getTaskInfo().getTaskName(),
				chainedConfigs.get(edge.getSourceId()).getBufferTimeout()));
	}
	return recordWriters;
}
 

@Override
public void invoke() throws Exception {
	RecordReader<SpeedTestRecord> reader = new RecordReader<>(
			getEnvironment().getInputGate(0),
			SpeedTestRecord.class,
			getEnvironment().getTaskManagerInfo().getTmpDirectories());

	RecordWriter<SpeedTestRecord> writer = new RecordWriter<>(getEnvironment().getWriter(0));

	try {
		SpeedTestRecord record;
		while ((record = reader.next()) != null) {
			writer.emit(record);
		}
	}
	finally {
		reader.clearBuffers();
		writer.clearBuffers();
		writer.flushAll();
	}
}
 

@Override
public void invoke() throws Exception {
	RecordWriter<IntValue> writer = new RecordWriter<>(getEnvironment().getWriter(0));
	final int numberOfTimesToSend = getTaskConfiguration().getInteger(CONFIG_KEY, 0);

	final IntValue subtaskIndex = new IntValue(
			getEnvironment().getTaskInfo().getIndexOfThisSubtask());

	try {
		for (int i = 0; i < numberOfTimesToSend; i++) {
			writer.emit(subtaskIndex);
		}
		writer.flushAll();
	}
	finally {
		writer.clearBuffers();
	}
}
 

@VisibleForTesting
public static <OUT> List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> createRecordWriters(
		StreamConfig configuration,
		Environment environment) {
	List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> recordWriters = new ArrayList<>();
	List<StreamEdge> outEdgesInOrder = configuration.getOutEdgesInOrder(environment.getUserClassLoader());
	Map<Integer, StreamConfig> chainedConfigs = configuration.getTransitiveChainedTaskConfigsWithSelf(environment.getUserClassLoader());

	for (int i = 0; i < outEdgesInOrder.size(); i++) {
		StreamEdge edge = outEdgesInOrder.get(i);
		recordWriters.add(
			createRecordWriter(
				edge,
				i,
				environment,
				environment.getTaskInfo().getTaskName(),
				chainedConfigs.get(edge.getSourceId()).getBufferTimeout()));
	}
	return recordWriters;
}
 

private RecordWriterOutput<OUT> createStreamOutput(
		RecordWriter<SerializationDelegate<StreamRecord<OUT>>> recordWriter,
		StreamEdge edge,
		StreamConfig upStreamConfig,
		Environment taskEnvironment) {
	OutputTag sideOutputTag = edge.getOutputTag(); // OutputTag, return null if not sideOutput

	TypeSerializer outSerializer = null;

	if (edge.getOutputTag() != null) {
		// side output
		outSerializer = upStreamConfig.getTypeSerializerSideOut(
				edge.getOutputTag(), taskEnvironment.getUserClassLoader());
	} else {
		// main output
		outSerializer = upStreamConfig.getTypeSerializerOut(taskEnvironment.getUserClassLoader());
	}

	return new RecordWriterOutput<>(recordWriter, outSerializer, sideOutputTag, this);
}
 

@SuppressWarnings("unchecked")
public RecordWriterOutput(
		RecordWriter<SerializationDelegate<StreamRecord<OUT>>> recordWriter,
		TypeSerializer<OUT> outSerializer,
		OutputTag outputTag,
		StreamStatusProvider streamStatusProvider) {

	checkNotNull(recordWriter);
	this.outputTag = outputTag;
	// generic hack: cast the writer to generic Object type so we can use it
	// with multiplexed records and watermarks
	this.recordWriter = (RecordWriter<SerializationDelegate<StreamElement>>)
			(RecordWriter<?>) recordWriter;

	TypeSerializer<StreamElement> outRecordSerializer =
			new StreamElementSerializer<>(outSerializer);

	if (outSerializer != null) {
		serializationDelegate = new SerializationDelegate<StreamElement>(outRecordSerializer);
	}

	this.streamStatusProvider = checkNotNull(streamStatusProvider);
}
 

@Override
protected void initOutputs() throws Exception {
	// initialize the regular outputs first (the ones into the step function).
	super.initOutputs();

	// at this time, the outputs to the step function are created
	// add the outputs for the final solution
	List<RecordWriter<?>> finalOutputWriters = new ArrayList<RecordWriter<?>>();
	final TaskConfig finalOutConfig = this.config.getIterationHeadFinalOutputConfig();
	final ClassLoader userCodeClassLoader = getUserCodeClassLoader();
	this.finalOutputCollector = BatchTask.getOutputCollector(this, finalOutConfig,
			userCodeClassLoader, finalOutputWriters, config.getNumOutputs(), finalOutConfig.getNumOutputs());

	// sanity check the setup
	final int writersIntoStepFunction = this.eventualOutputs.size();
	final int writersIntoFinalResult = finalOutputWriters.size();
	final int syncGateIndex = this.config.getIterationHeadIndexOfSyncOutput();

	if (writersIntoStepFunction + writersIntoFinalResult != syncGateIndex) {
		throw new Exception("Error: Inconsistent head task setup - wrong mapping of output gates.");
	}
	// now, we can instantiate the sync gate
	this.toSync = new RecordWriterBuilder<>().build(getEnvironment().getWriter(syncGateIndex));
	this.toSyncPartitionId = getEnvironment().getWriter(syncGateIndex).getPartitionId();
}
 

@Override
public void invoke() throws Exception {
	ResultPartitionWriter resultPartitionWriter = getEnvironment().getWriter(0);
	RecordWriterBuilder<LongValue> recordWriterBuilder = new RecordWriterBuilder<>();
	if (getEnvironment().getExecutionConfig().getExecutionMode() == ExecutionMode.PIPELINED) {
		// enable output flush for pipeline mode
		recordWriterBuilder.setTimeout(100);
	}
	if (useBroadcastPartitioner) {
		recordWriterBuilder.setChannelSelector(new BroadcastPartitioner());
	}
	RecordWriter<LongValue> writer = recordWriterBuilder.build(resultPartitionWriter);

	for (int i = 0; i < NUM_RECORDS_TO_SEND; ++i) {
		writer.broadcastEmit(RECORD_TO_SEND);
	}
	writer.flushAll();
	writer.clearBuffers();
}
 

@Override
public void invoke() throws Exception {
	RecordReader<SpeedTestRecord> reader = new RecordReader<>(
			getEnvironment().getInputGate(0),
			SpeedTestRecord.class,
			getEnvironment().getTaskManagerInfo().getTmpDirectories());

	RecordWriter<SpeedTestRecord> writer = new RecordWriterBuilder().build(getEnvironment().getWriter(0));

	try {
		SpeedTestRecord record;
		while ((record = reader.next()) != null) {
			writer.emit(record);
		}
	}
	finally {
		reader.clearBuffers();
		writer.clearBuffers();
		writer.flushAll();
	}
}
 

@Override
public void invoke() throws Exception {
	RecordReader<SpeedTestRecord> reader = new RecordReader<>(
			getEnvironment().getInputGate(0),
			SpeedTestRecord.class,
			getEnvironment().getTaskManagerInfo().getTmpDirectories());

	RecordWriter<SpeedTestRecord> writer = new RecordWriterBuilder<SpeedTestRecord>().build(getEnvironment().getWriter(0));

	try {
		SpeedTestRecord record;
		while ((record = reader.next()) != null) {
			writer.emit(record);
		}
	}
	finally {
		reader.clearBuffers();
		writer.clearBuffers();
		writer.flushAll();
	}
}
 

@Override
protected void initOutputs() throws Exception {
	// initialize the regular outputs first (the ones into the step function).
	super.initOutputs();

	// at this time, the outputs to the step function are created
	// add the outputs for the final solution
	List<RecordWriter<?>> finalOutputWriters = new ArrayList<RecordWriter<?>>();
	final TaskConfig finalOutConfig = this.config.getIterationHeadFinalOutputConfig();
	final ClassLoader userCodeClassLoader = getUserCodeClassLoader();
	this.finalOutputCollector = BatchTask.getOutputCollector(this, finalOutConfig,
			userCodeClassLoader, finalOutputWriters, config.getNumOutputs(), finalOutConfig.getNumOutputs());

	// sanity check the setup
	final int writersIntoStepFunction = this.eventualOutputs.size();
	final int writersIntoFinalResult = finalOutputWriters.size();
	final int syncGateIndex = this.config.getIterationHeadIndexOfSyncOutput();

	if (writersIntoStepFunction + writersIntoFinalResult != syncGateIndex) {
		throw new Exception("Error: Inconsistent head task setup - wrong mapping of output gates.");
	}
	// now, we can instantiate the sync gate
	this.toSync = new RecordWriterBuilder().build(getEnvironment().getWriter(syncGateIndex));
	this.toSyncPartitionId = getEnvironment().getWriter(syncGateIndex).getPartitionId();
}
 

@SuppressWarnings("unchecked")
public RecordWriterOutput(
		RecordWriter<SerializationDelegate<StreamRecord<OUT>>> recordWriter,
		TypeSerializer<OUT> outSerializer,
		OutputTag outputTag,
		StreamStatusProvider streamStatusProvider) {

	checkNotNull(recordWriter);
	this.outputTag = outputTag;
	// generic hack: cast the writer to generic Object type so we can use it
	// with multiplexed records and watermarks
	this.recordWriter = (RecordWriter<SerializationDelegate<StreamElement>>)
			(RecordWriter<?>) recordWriter;

	TypeSerializer<StreamElement> outRecordSerializer =
			new StreamElementSerializer<>(outSerializer);

	if (outSerializer != null) {
		serializationDelegate = new SerializationDelegate<StreamElement>(outRecordSerializer);
	}

	this.streamStatusProvider = checkNotNull(streamStatusProvider);
}
 

private RecordWriterOutput<OUT> createStreamOutput(
		RecordWriter<SerializationDelegate<StreamRecord<OUT>>> recordWriter,
		StreamEdge edge,
		StreamConfig upStreamConfig,
		Environment taskEnvironment) {
	OutputTag sideOutputTag = edge.getOutputTag(); // OutputTag, return null if not sideOutput

	TypeSerializer outSerializer = null;

	if (edge.getOutputTag() != null) {
		// side output
		outSerializer = upStreamConfig.getTypeSerializerSideOut(
				edge.getOutputTag(), taskEnvironment.getUserClassLoader());
	} else {
		// main output
		outSerializer = upStreamConfig.getTypeSerializerOut(taskEnvironment.getUserClassLoader());
	}

	return new RecordWriterOutput<>(recordWriter, outSerializer, sideOutputTag, this);
}
 

@VisibleForTesting
public static <OUT> List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> createRecordWriters(
		StreamConfig configuration,
		Environment environment) {
	List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> recordWriters = new ArrayList<>();
	List<StreamEdge> outEdgesInOrder = configuration.getOutEdgesInOrder(environment.getUserClassLoader());
	Map<Integer, StreamConfig> chainedConfigs = configuration.getTransitiveChainedTaskConfigsWithSelf(environment.getUserClassLoader());

	for (int i = 0; i < outEdgesInOrder.size(); i++) {
		StreamEdge edge = outEdgesInOrder.get(i);
		recordWriters.add(
			createRecordWriter(
				edge,
				i,
				environment,
				environment.getTaskInfo().getTaskName(),
				chainedConfigs.get(edge.getSourceId()).getBufferTimeout()));
	}
	return recordWriters;
}
 

@Override
public void invoke() throws Exception {
	RecordWriter<IntValue> writer = new RecordWriterBuilder().build(getEnvironment().getWriter(0));
	final int numberOfTimesToSend = getTaskConfiguration().getInteger(CONFIG_KEY, 0);

	final IntValue subtaskIndex = new IntValue(
			getEnvironment().getTaskInfo().getIndexOfThisSubtask());

	try {
		for (int i = 0; i < numberOfTimesToSend; i++) {
			writer.emit(subtaskIndex);
		}
		writer.flushAll();
	}
	finally {
		writer.clearBuffers();
	}
}
 

@Override
public void invoke() throws Exception {
	final RecordWriter<IntValue> writer = new RecordWriterBuilder<IntValue>().build(getEnvironment().getWriter(0));

	try {
		writer.emit(new IntValue(42));
		writer.emit(new IntValue(1337));
		writer.flushAll();
	} finally {
		writer.clearBuffers();
	}
}
 

/**
 * Creates a writer for each output. Creates an OutputCollector which forwards its input to all writers.
 * The output collector applies the configured shipping strategy.
 */
private void initOutputs(ClassLoader cl) throws Exception {
	this.chainedTasks = new ArrayList<ChainedDriver<?, ?>>();
	this.eventualOutputs = new ArrayList<RecordWriter<?>>();

	this.output = BatchTask.initOutputs(this, cl, this.config, this.chainedTasks, this.eventualOutputs,
			getExecutionConfig(), getEnvironment().getAccumulatorRegistry().getUserMap());
}
 

private static <OUT> RecordWriter<SerializationDelegate<StreamRecord<OUT>>> createRecordWriter(
		StreamEdge edge,
		int outputIndex,
		Environment environment,
		String taskName,
		long bufferTimeout) {
	@SuppressWarnings("unchecked")
	StreamPartitioner<OUT> outputPartitioner = (StreamPartitioner<OUT>) edge.getPartitioner();

	LOG.debug("Using partitioner {} for output {} of task {}", outputPartitioner, outputIndex, taskName);

	ResultPartitionWriter bufferWriter = environment.getWriter(outputIndex);

	// we initialize the partitioner here with the number of key groups (aka max. parallelism)
	if (outputPartitioner instanceof ConfigurableStreamPartitioner) {
		int numKeyGroups = bufferWriter.getNumTargetKeyGroups();
		if (0 < numKeyGroups) {
			((ConfigurableStreamPartitioner) outputPartitioner).configure(numKeyGroups);
		}
	}

	RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output = new RecordWriterBuilder<SerializationDelegate<StreamRecord<OUT>>>()
		.setChannelSelector(outputPartitioner)
		.setTimeout(bufferTimeout)
		.setTaskName(taskName)
		.build(bufferWriter);
	output.setMetricGroup(environment.getMetricGroup().getIOMetricGroup());
	return output;
}
 

private static <OUT> RecordWriter<SerializationDelegate<StreamRecord<OUT>>> createRecordWriter(
		StreamEdge edge,
		int outputIndex,
		Environment environment,
		String taskName,
		long bufferTimeout) {
	@SuppressWarnings("unchecked")
	StreamPartitioner<OUT> outputPartitioner = (StreamPartitioner<OUT>) edge.getPartitioner();

	LOG.debug("Using partitioner {} for output {} of task {}", outputPartitioner, outputIndex, taskName);

	ResultPartitionWriter bufferWriter = environment.getWriter(outputIndex);

	// we initialize the partitioner here with the number of key groups (aka max. parallelism)
	if (outputPartitioner instanceof ConfigurableStreamPartitioner) {
		int numKeyGroups = bufferWriter.getNumTargetKeyGroups();
		if (0 < numKeyGroups) {
			((ConfigurableStreamPartitioner) outputPartitioner).configure(numKeyGroups);
		}
	}

	RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output = new RecordWriterBuilder()
		.setChannelSelector(outputPartitioner)
		.setTimeout(bufferTimeout)
		.setTaskName(taskName)
		.build(bufferWriter);
	output.setMetricGroup(environment.getMetricGroup().getIOMetricGroup());
	return output;
}
 

/**
 * Creates a writer for each output. Creates an OutputCollector which forwards its input to all writers.
 * The output collector applies the configured shipping strategy.
 */
private void initOutputs(ClassLoader cl) throws Exception {
	this.chainedTasks = new ArrayList<ChainedDriver<?, ?>>();
	this.eventualOutputs = new ArrayList<RecordWriter<?>>();

	this.output = BatchTask.initOutputs(this, cl, this.config, this.chainedTasks, this.eventualOutputs,
			getExecutionConfig(), getEnvironment().getAccumulatorRegistry().getUserMap());
}
 

@Override
public void invoke() throws Exception {
	List<RecordWriter<IntValue>> writers = Lists.newArrayListWithCapacity(2);

	// The order of intermediate result creation in the job graph specifies which produced
	// result partition is pipelined/blocking.
	final RecordWriter<IntValue> pipelinedWriter = new RecordWriterBuilder().build(getEnvironment().getWriter(0));
	final RecordWriter<IntValue> blockingWriter = new RecordWriterBuilder().build(getEnvironment().getWriter(1));
	writers.add(pipelinedWriter);
	writers.add(blockingWriter);

	final int numberOfTimesToSend = getTaskConfiguration().getInteger(CONFIG_KEY, 0);

	final IntValue subtaskIndex = new IntValue(
			getEnvironment().getTaskInfo().getIndexOfThisSubtask());

	// Produce the first intermediate result and then the second in a serial fashion.
	for (RecordWriter<IntValue> writer : writers) {
		try {
			for (int i = 0; i < numberOfTimesToSend; i++) {
				writer.emit(subtaskIndex);
			}
			writer.flushAll();
		}
		finally {
			writer.clearBuffers();
		}
	}
}
 

@Override
public void invoke() throws Exception {
	final Object o = new Object();
	RecordWriter<IntValue> recordWriter = new RecordWriterBuilder().build(getEnvironment().getWriter(0));
	for (int i = 0; i < 1024; i++) {
		recordWriter.emit(new IntValue(42));
	}

	gotCanceledFuture.get();

}
 

@Override
public void invoke() throws Exception {
	final RecordWriter<IntValue> writer = new RecordWriterBuilder().build(getEnvironment().getWriter(0));

	try {
		writer.emit(new IntValue(42));
		writer.emit(new IntValue(1337));
		writer.flushAll();
	} finally {
		writer.clearBuffers();
	}
}
 

@VisibleForTesting
public static <OUT> RecordWriterDelegate<SerializationDelegate<StreamRecord<OUT>>> createRecordWriterDelegate(
		StreamConfig configuration,
		Environment environment) {
	List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> recordWrites = createRecordWriters(
		configuration,
		environment);
	if (recordWrites.size() == 1) {
		return new SingleRecordWriter<>(recordWrites.get(0));
	} else if (recordWrites.size() == 0) {
		return new NonRecordWriter<>();
	} else {
		return new MultipleRecordWriters<>(recordWrites);
	}
}