Java Code Examples for org.apache.flink.runtime.io.network.api.writer.RecordWriter#emit()

The following examples show how to use org.apache.flink.runtime.io.network.api.writer.RecordWriter#emit() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: NetworkStackThroughputITCase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Override
public void invoke() throws Exception {
	RecordReader<SpeedTestRecord> reader = new RecordReader<>(
			getEnvironment().getInputGate(0),
			SpeedTestRecord.class,
			getEnvironment().getTaskManagerInfo().getTmpDirectories());

	RecordWriter<SpeedTestRecord> writer = new RecordWriter<>(getEnvironment().getWriter(0));

	try {
		SpeedTestRecord record;
		while ((record = reader.next()) != null) {
			writer.emit(record);
		}
	}
	finally {
		reader.clearBuffers();
		writer.clearBuffers();
		writer.flushAll();
	}
}
Example 2
Source File: TaskManagerTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Override
public void invoke() throws Exception {
	final Object o = new Object();
	RecordWriter<IntValue> recordWriter = new RecordWriter<>(getEnvironment().getWriter(0));

	for (int i = 0; i < 1024; i++) {
		recordWriter.emit(new IntValue(42));
	}

	synchronized (o) {
		//noinspection InfiniteLoopStatement
		while (true) {
			o.wait();
		}
	}

}
Example 3
Source File: SlotCountExceedingParallelismTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public void invoke() throws Exception {
	RecordWriter<IntValue> writer = new RecordWriterBuilder<IntValue>().build(getEnvironment().getWriter(0));
	final int numberOfTimesToSend = getTaskConfiguration().getInteger(CONFIG_KEY, 0);

	final IntValue subtaskIndex = new IntValue(
			getEnvironment().getTaskInfo().getIndexOfThisSubtask());

	try {
		for (int i = 0; i < numberOfTimesToSend; i++) {
			writer.emit(subtaskIndex);
		}
		writer.flushAll();
	}
	finally {
		writer.clearBuffers();
	}
}
Example 4
Source File: SlotCountExceedingParallelismTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Override
public void invoke() throws Exception {
	RecordWriter<IntValue> writer = new RecordWriter<>(getEnvironment().getWriter(0));
	final int numberOfTimesToSend = getTaskConfiguration().getInteger(CONFIG_KEY, 0);

	final IntValue subtaskIndex = new IntValue(
			getEnvironment().getTaskInfo().getIndexOfThisSubtask());

	try {
		for (int i = 0; i < numberOfTimesToSend; i++) {
			writer.emit(subtaskIndex);
		}
		writer.flushAll();
	}
	finally {
		writer.clearBuffers();
	}
}
Example 5
Source File: NetworkStackThroughputITCase.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public void invoke() throws Exception {
	RecordReader<SpeedTestRecord> reader = new RecordReader<>(
			getEnvironment().getInputGate(0),
			SpeedTestRecord.class,
			getEnvironment().getTaskManagerInfo().getTmpDirectories());

	RecordWriter<SpeedTestRecord> writer = new RecordWriterBuilder().build(getEnvironment().getWriter(0));

	try {
		SpeedTestRecord record;
		while ((record = reader.next()) != null) {
			writer.emit(record);
		}
	}
	finally {
		reader.clearBuffers();
		writer.clearBuffers();
		writer.flushAll();
	}
}
Example 6
Source File: NetworkStackThroughputITCase.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public void invoke() throws Exception {
	RecordReader<SpeedTestRecord> reader = new RecordReader<>(
			getEnvironment().getInputGate(0),
			SpeedTestRecord.class,
			getEnvironment().getTaskManagerInfo().getTmpDirectories());

	RecordWriter<SpeedTestRecord> writer = new RecordWriterBuilder<SpeedTestRecord>().build(getEnvironment().getWriter(0));

	try {
		SpeedTestRecord record;
		while ((record = reader.next()) != null) {
			writer.emit(record);
		}
	}
	finally {
		reader.clearBuffers();
		writer.clearBuffers();
		writer.flushAll();
	}
}
Example 7
Source File: SlotCountExceedingParallelismTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public void invoke() throws Exception {
	RecordWriter<IntValue> writer = new RecordWriterBuilder().build(getEnvironment().getWriter(0));
	final int numberOfTimesToSend = getTaskConfiguration().getInteger(CONFIG_KEY, 0);

	final IntValue subtaskIndex = new IntValue(
			getEnvironment().getTaskInfo().getIndexOfThisSubtask());

	try {
		for (int i = 0; i < numberOfTimesToSend; i++) {
			writer.emit(subtaskIndex);
		}
		writer.flushAll();
	}
	finally {
		writer.clearBuffers();
	}
}
Example 8
Source File: ScheduleOrUpdateConsumersTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void invoke() throws Exception {
	List<RecordWriter<IntValue>> writers = Lists.newArrayListWithCapacity(2);

	// The order of intermediate result creation in the job graph specifies which produced
	// result partition is pipelined/blocking.
	final RecordWriter<IntValue> pipelinedWriter = new RecordWriterBuilder<IntValue>().build(getEnvironment().getWriter(0));
	final RecordWriter<IntValue> blockingWriter = new RecordWriterBuilder<IntValue>().build(getEnvironment().getWriter(1));
	writers.add(pipelinedWriter);
	writers.add(blockingWriter);

	final int numberOfTimesToSend = getTaskConfiguration().getInteger(CONFIG_KEY, 0);

	final IntValue subtaskIndex = new IntValue(
			getEnvironment().getTaskInfo().getIndexOfThisSubtask());

	// Produce the first intermediate result and then the second in a serial fashion.
	for (RecordWriter<IntValue> writer : writers) {
		try {
			for (int i = 0; i < numberOfTimesToSend; i++) {
				writer.emit(subtaskIndex);
			}
			writer.flushAll();
		}
		finally {
			writer.clearBuffers();
		}
	}
}
Example 9
Source File: NetworkStackThroughputITCase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public void invoke() throws Exception {
	RecordWriter<SpeedTestRecord> writer = new RecordWriter<>(getEnvironment().getWriter(0));

	try {
		// Determine the amount of data to send per subtask
		int dataVolumeGb = getTaskConfiguration().getInteger(NetworkStackThroughputITCase.DATA_VOLUME_GB_CONFIG_KEY, 1);

		long dataMbPerSubtask = (dataVolumeGb * 10) / getCurrentNumberOfSubtasks();
		long numRecordsToEmit = (dataMbPerSubtask * 1024 * 1024) / SpeedTestRecord.RECORD_SIZE;

		LOG.info(String.format("%d/%d: Producing %d records (each record: %d bytes, total: %.2f GB)",
				getIndexInSubtaskGroup() + 1, getCurrentNumberOfSubtasks(), numRecordsToEmit,
				SpeedTestRecord.RECORD_SIZE, dataMbPerSubtask / 1024.0));

		boolean isSlow = getTaskConfiguration().getBoolean(IS_SLOW_SENDER_CONFIG_KEY, false);

		int numRecords = 0;
		SpeedTestRecord record = new SpeedTestRecord();
		for (long i = 0; i < numRecordsToEmit; i++) {
			if (isSlow && (numRecords++ % IS_SLOW_EVERY_NUM_RECORDS) == 0) {
				Thread.sleep(IS_SLOW_SLEEP_MS);
			}

			writer.emit(record);
		}
	}
	finally {
		writer.clearBuffers();
		writer.flushAll();
	}
}
Example 10
Source File: OutputBlockedInvokable.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void invoke() throws Exception {
	final IntValue value = new IntValue(1234);
	final ResultPartitionWriter resultPartitionWriter = getEnvironment().getWriter(0);
	final RecordWriter<IntValue> writer = new RecordWriterBuilder<IntValue>().build(resultPartitionWriter);

	while (true) {
		writer.emit(value);
	}
}
Example 11
Source File: TestingAbstractInvokables.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void invoke() throws Exception {
	final Object o = new Object();
	RecordWriter<IntValue> recordWriter = new RecordWriterBuilder<IntValue>().build(getEnvironment().getWriter(0));
	for (int i = 0; i < 1024; i++) {
		recordWriter.emit(new IntValue(42));
	}

	gotCanceledFuture.get();

}
Example 12
Source File: TestingAbstractInvokables.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void invoke() throws Exception {
	final RecordWriter<IntValue> writer = new RecordWriterBuilder<IntValue>().build(getEnvironment().getWriter(0));

	try {
		writer.emit(new IntValue(42));
		writer.emit(new IntValue(1337));
		writer.flushAll();
	} finally {
		writer.clearBuffers();
	}
}
Example 13
Source File: FileBufferReaderITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void invoke() throws Exception {
	final RecordWriter<ByteArrayType> writer = new RecordWriterBuilder<ByteArrayType>().build(getEnvironment().getWriter(0));

	final ByteArrayType bytes = new ByteArrayType(dataSource);
	int counter = 0;
	while (counter++ < numRecords) {
		try {
			writer.emit(bytes);
			writer.flushAll();
		} finally {
			writer.clearBuffers();
		}
	}
}
Example 14
Source File: NetworkStackThroughputITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void invoke() throws Exception {
	RecordWriter<SpeedTestRecord> writer = new RecordWriterBuilder<SpeedTestRecord>().build(getEnvironment().getWriter(0));

	try {
		// Determine the amount of data to send per subtask
		int dataVolumeGb = getTaskConfiguration().getInteger(NetworkStackThroughputITCase.DATA_VOLUME_GB_CONFIG_KEY, 1);

		long dataMbPerSubtask = (dataVolumeGb * 10) / getCurrentNumberOfSubtasks();
		long numRecordsToEmit = (dataMbPerSubtask * 1024 * 1024) / SpeedTestRecord.RECORD_SIZE;

		LOG.info(String.format("%d/%d: Producing %d records (each record: %d bytes, total: %.2f GB)",
				getIndexInSubtaskGroup() + 1, getCurrentNumberOfSubtasks(), numRecordsToEmit,
				SpeedTestRecord.RECORD_SIZE, dataMbPerSubtask / 1024.0));

		boolean isSlow = getTaskConfiguration().getBoolean(IS_SLOW_SENDER_CONFIG_KEY, false);

		int numRecords = 0;
		SpeedTestRecord record = new SpeedTestRecord();
		for (long i = 0; i < numRecordsToEmit; i++) {
			if (isSlow && (numRecords++ % IS_SLOW_EVERY_NUM_RECORDS) == 0) {
				Thread.sleep(IS_SLOW_SLEEP_MS);
			}

			writer.emit(record);
		}
	}
	finally {
		writer.clearBuffers();
		writer.flushAll();
	}
}
Example 15
Source File: ScheduleOrUpdateConsumersTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void invoke() throws Exception {
	List<RecordWriter<IntValue>> writers = Lists.newArrayListWithCapacity(2);

	// The order of intermediate result creation in the job graph specifies which produced
	// result partition is pipelined/blocking.
	final RecordWriter<IntValue> pipelinedWriter = new RecordWriterBuilder().build(getEnvironment().getWriter(0));
	final RecordWriter<IntValue> blockingWriter = new RecordWriterBuilder().build(getEnvironment().getWriter(1));
	writers.add(pipelinedWriter);
	writers.add(blockingWriter);

	final int numberOfTimesToSend = getTaskConfiguration().getInteger(CONFIG_KEY, 0);

	final IntValue subtaskIndex = new IntValue(
			getEnvironment().getTaskInfo().getIndexOfThisSubtask());

	// Produce the first intermediate result and then the second in a serial fashion.
	for (RecordWriter<IntValue> writer : writers) {
		try {
			for (int i = 0; i < numberOfTimesToSend; i++) {
				writer.emit(subtaskIndex);
			}
			writer.flushAll();
		}
		finally {
			writer.clearBuffers();
		}
	}
}
Example 16
Source File: TestingAbstractInvokables.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void invoke() throws Exception {
	final Object o = new Object();
	RecordWriter<IntValue> recordWriter = new RecordWriterBuilder().build(getEnvironment().getWriter(0));
	for (int i = 0; i < 1024; i++) {
		recordWriter.emit(new IntValue(42));
	}

	gotCanceledFuture.get();

}
Example 17
Source File: TestingAbstractInvokables.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void invoke() throws Exception {
	final RecordWriter<IntValue> writer = new RecordWriterBuilder().build(getEnvironment().getWriter(0));

	try {
		writer.emit(new IntValue(42));
		writer.emit(new IntValue(1337));
		writer.flushAll();
	} finally {
		writer.clearBuffers();
	}
}
Example 18
Source File: FileBufferReaderITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void invoke() throws Exception {
	final RecordWriter<ByteArrayType> writer = new RecordWriterBuilder().build(getEnvironment().getWriter(0));

	final ByteArrayType bytes = new ByteArrayType(dataSource);
	int counter = 0;
	while (counter++ < numRecords) {
		try {
			writer.emit(bytes);
			writer.flushAll();
		} finally {
			writer.clearBuffers();
		}
	}
}
Example 19
Source File: NetworkStackThroughputITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void invoke() throws Exception {
	RecordWriter<SpeedTestRecord> writer = new RecordWriterBuilder().build(getEnvironment().getWriter(0));

	try {
		// Determine the amount of data to send per subtask
		int dataVolumeGb = getTaskConfiguration().getInteger(NetworkStackThroughputITCase.DATA_VOLUME_GB_CONFIG_KEY, 1);

		long dataMbPerSubtask = (dataVolumeGb * 10) / getCurrentNumberOfSubtasks();
		long numRecordsToEmit = (dataMbPerSubtask * 1024 * 1024) / SpeedTestRecord.RECORD_SIZE;

		LOG.info(String.format("%d/%d: Producing %d records (each record: %d bytes, total: %.2f GB)",
				getIndexInSubtaskGroup() + 1, getCurrentNumberOfSubtasks(), numRecordsToEmit,
				SpeedTestRecord.RECORD_SIZE, dataMbPerSubtask / 1024.0));

		boolean isSlow = getTaskConfiguration().getBoolean(IS_SLOW_SENDER_CONFIG_KEY, false);

		int numRecords = 0;
		SpeedTestRecord record = new SpeedTestRecord();
		for (long i = 0; i < numRecordsToEmit; i++) {
			if (isSlow && (numRecords++ % IS_SLOW_EVERY_NUM_RECORDS) == 0) {
				Thread.sleep(IS_SLOW_SLEEP_MS);
			}

			writer.emit(record);
		}
	}
	finally {
		writer.clearBuffers();
		writer.flushAll();
	}
}
Example 20
Source File: TestingAbstractInvokables.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public void invoke() throws Exception {
	final RecordWriter<IntValue> writer = new RecordWriter<>(getEnvironment().getWriter(0));

	try {
		writer.emit(new IntValue(42));
		writer.emit(new IntValue(1337));
		writer.flushAll();
	} finally {
		writer.clearBuffers();
	}
}