Java Code Examples for org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent#isEvent()

The following examples show how to use org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent#isEvent() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MockInputGate.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Override
public Optional<BufferOrEvent> getNextBufferOrEvent() {
	BufferOrEvent next = bufferOrEvents.poll();
	if (next == null) {
		return Optional.empty();
	}

	int channelIdx = next.getChannelIndex();
	if (closed[channelIdx]) {
		throw new RuntimeException("Inconsistent: Channel " + channelIdx
			+ " has data even though it is already closed.");
	}
	if (next.isEvent() && next.getEvent() instanceof EndOfPartitionEvent) {
		closed[channelIdx] = true;
		closedChannels++;
	}
	return Optional.of(next);
}
Example 2
Source File: MockInputGate.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public Optional<BufferOrEvent> getNext() {
	BufferOrEvent next = bufferOrEvents.poll();
	if (!finishAfterLastBuffer && bufferOrEvents.isEmpty()) {
		resetIsAvailable();
	}
	if (next == null) {
		return Optional.empty();
	}

	int channelIdx = next.getChannelIndex();
	if (closed[channelIdx]) {
		throw new RuntimeException("Inconsistent: Channel " + channelIdx
			+ " has data even though it is already closed.");
	}
	if (next.isEvent() && next.getEvent() instanceof EndOfPartitionEvent) {
		closed[channelIdx] = true;
	}
	return Optional.of(next);
}
Example 3
Source File: MockInputGate.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public Optional<BufferOrEvent> getNext() {
	BufferOrEvent next = bufferOrEvents.poll();
	if (!finishAfterLastBuffer && bufferOrEvents.isEmpty()) {
		availabilityHelper.resetUnavailable();
	}
	if (next == null) {
		return Optional.empty();
	}

	int channelIdx = next.getChannelInfo().getInputChannelIdx();
	if (closed[channelIdx]) {
		throw new RuntimeException("Inconsistent: Channel " + channelIdx
			+ " has data even though it is already closed.");
	}
	if (next.isEvent() && next.getEvent() instanceof EndOfPartitionEvent) {
		closed[channelIdx] = true;
	}
	return Optional.of(next);
}
Example 4
Source File: BufferStorageTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up 
private static void assertNextBufferOrEvent(
		ArrayDeque<ArrayDeque<BufferOrEvent>> expectedRolledSequence,
		BufferStorage bufferStorage) throws IOException {
	while (!expectedRolledSequence.isEmpty() && expectedRolledSequence.peekFirst().isEmpty()) {
		expectedRolledSequence.pollFirst();
	}

	Optional<BufferOrEvent> next = bufferStorage.pollNext();
	if (expectedRolledSequence.isEmpty()) {
		assertFalse(next.isPresent());
		return;
	}

	while (!next.isPresent() && !bufferStorage.isEmpty()) {
		next = bufferStorage.pollNext();
	}

	assertTrue(next.isPresent());
	BufferOrEvent actualBufferOrEvent = next.get();
	BufferOrEvent expectedBufferOrEvent = expectedRolledSequence.peekFirst().pollFirst();

	if (expectedBufferOrEvent.isEvent()) {
		assertEquals(expectedBufferOrEvent.getChannelIndex(), actualBufferOrEvent.getChannelIndex());
		assertEquals(expectedBufferOrEvent.getEvent(), actualBufferOrEvent.getEvent());
	} else {
		validateBuffer(
			actualBufferOrEvent,
			expectedBufferOrEvent.getSize(),
			expectedBufferOrEvent.getChannelIndex());
	}
}
Example 5
Source File: BufferBlockerTestBase.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Test
public void testSpillAndRollOverSimple() throws IOException {
	final Random rnd = new Random();
	final Random bufferRnd = new Random();

	final int maxNumEventsAndBuffers = 3000;
	final int maxNumChannels = 1656;

	BufferBlocker bufferBlocker = createBufferBlocker();

	// do multiple spilling / rolling over rounds
	for (int round = 0; round < 5; round++) {

		final long bufferSeed = rnd.nextLong();
		bufferRnd.setSeed(bufferSeed);

		final int numEventsAndBuffers = rnd.nextInt(maxNumEventsAndBuffers) + 1;
		final int numberOfChannels = rnd.nextInt(maxNumChannels) + 1;

		final ArrayList<BufferOrEvent> events = new ArrayList<BufferOrEvent>(128);

		// generate sequence
		for (int i = 0; i < numEventsAndBuffers; i++) {
			boolean isEvent = rnd.nextDouble() < 0.05d;
			BufferOrEvent evt;
			if (isEvent) {
				evt = generateRandomEvent(rnd, numberOfChannels);
				events.add(evt);
			} else {
				evt = generateRandomBuffer(bufferRnd.nextInt(PAGE_SIZE) + 1, bufferRnd.nextInt(numberOfChannels));
			}
			bufferBlocker.add(evt);
		}

		// reset and create reader
		bufferRnd.setSeed(bufferSeed);

		BufferOrEventSequence seq = bufferBlocker.rollOverReusingResources();
		seq.open();

		// read and validate the sequence

		int numEvent = 0;
		for (int i = 0; i < numEventsAndBuffers; i++) {
			BufferOrEvent next = seq.getNext();
			assertNotNull(next);
			if (next.isEvent()) {
				BufferOrEvent expected = events.get(numEvent++);
				assertEquals(expected.getEvent(), next.getEvent());
				assertEquals(expected.getChannelIndex(), next.getChannelIndex());
			} else {
				validateBuffer(next, bufferRnd.nextInt(PAGE_SIZE) + 1, bufferRnd.nextInt(numberOfChannels));
			}
		}

		// no further data
		assertNull(seq.getNext());

		// all events need to be consumed
		assertEquals(events.size(), numEvent);

		seq.cleanup();
	}
}
Example 6
Source File: SpilledBufferOrEventSequenceTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Test
public void testMixedSequence() {
	try {
		final Random rnd = new Random();
		final Random bufferRnd = new Random();

		final long bufferSeed = rnd.nextLong();
		bufferRnd.setSeed(bufferSeed);

		final int numEventsAndBuffers = 3000;
		final int numberOfChannels = 1656;

		final ArrayList<BufferOrEvent> events = new ArrayList<BufferOrEvent>(128);

		// generate sequence

		for (int i = 0; i < numEventsAndBuffers; i++) {
			boolean isEvent = rnd.nextDouble() < 0.05d;
			if (isEvent) {
				events.add(generateAndWriteEvent(fileChannel, rnd, numberOfChannels));
			}
			else {
				writeBuffer(fileChannel, bufferRnd.nextInt(pageSize) + 1, bufferRnd.nextInt(numberOfChannels));
			}
		}

		// reset and create reader

		fileChannel.position(0L);
		bufferRnd.setSeed(bufferSeed);
		SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(tempFile, fileChannel, buffer, pageSize);
		seq.open();

		// read and validate the sequence

		int numEvent = 0;
		for (int i = 0; i < numEventsAndBuffers; i++) {
			BufferOrEvent next = seq.getNext();
			if (next.isEvent()) {
				BufferOrEvent expected = events.get(numEvent++);
				assertEquals(expected.getEvent(), next.getEvent());
				assertEquals(expected.getChannelIndex(), next.getChannelIndex());
			}
			else {
				validateBuffer(next, bufferRnd.nextInt(pageSize) + 1, bufferRnd.nextInt(numberOfChannels));
			}
		}

		// no further data
		assertNull(seq.getNext());

		// all events need to be consumed
		assertEquals(events.size(), numEvent);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
Example 7
Source File: BufferStorageTestBase.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Test
public void testSpillAndRollOverSimple() throws IOException {
	final Random rnd = new Random();
	final Random bufferRnd = new Random();

	final int maxNumEventsAndBuffers = 3000;
	final int maxNumChannels = 1656;

	BufferStorage bufferStorage = createBufferStorage();

	// do multiple spilling / rolling over rounds
	for (int round = 0; round < 5; round++) {

		final long bufferSeed = rnd.nextLong();
		bufferRnd.setSeed(bufferSeed);

		final int numEventsAndBuffers = rnd.nextInt(maxNumEventsAndBuffers) + 1;
		final int numberOfChannels = rnd.nextInt(maxNumChannels) + 1;

		final ArrayList<BufferOrEvent> events = new ArrayList<BufferOrEvent>(128);

		// generate sequence
		for (int i = 0; i < numEventsAndBuffers; i++) {
			boolean isEvent = rnd.nextDouble() < 0.05d;
			BufferOrEvent evt;
			if (isEvent) {
				evt = generateRandomEvent(rnd, numberOfChannels);
				events.add(evt);
			} else {
				evt = generateRandomBuffer(bufferRnd.nextInt(PAGE_SIZE) + 1, bufferRnd.nextInt(numberOfChannels));
			}
			bufferStorage.add(evt);
		}

		// reset and create reader
		bufferRnd.setSeed(bufferSeed);

		bufferStorage.rollOver();

		// read and validate the sequence

		int numEvent = 0;
		for (int i = 0; i < numEventsAndBuffers; i++) {
			assertFalse(bufferStorage.isEmpty());

			Optional<BufferOrEvent> next = bufferStorage.pollNext();
			assertTrue(next.isPresent());
			BufferOrEvent bufferOrEvent = next.get();

			if (bufferOrEvent.isEvent()) {
				BufferOrEvent expected = events.get(numEvent++);
				assertEquals(expected.getEvent(), bufferOrEvent.getEvent());
				assertEquals(expected.getChannelIndex(), bufferOrEvent.getChannelIndex());
			} else {
				validateBuffer(
					bufferOrEvent,
					bufferRnd.nextInt(PAGE_SIZE) + 1, bufferRnd.nextInt(numberOfChannels));
			}
		}

		// no further data
		assertFalse(bufferStorage.pollNext().isPresent());
		assertTrue(bufferStorage.isEmpty());

		// all events need to be consumed
		assertEquals(events.size(), numEvent);
	}
}
Example 8
Source File: SpilledBufferOrEventSequenceTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Test
public void testMixedSequence() {
	try {
		final Random rnd = new Random();
		final Random bufferRnd = new Random();

		final long bufferSeed = rnd.nextLong();
		bufferRnd.setSeed(bufferSeed);

		final int numEventsAndBuffers = 3000;
		final int numberOfChannels = 1656;

		final ArrayList<BufferOrEvent> events = new ArrayList<BufferOrEvent>(128);

		// generate sequence

		for (int i = 0; i < numEventsAndBuffers; i++) {
			boolean isEvent = rnd.nextDouble() < 0.05d;
			if (isEvent) {
				events.add(generateAndWriteEvent(fileChannel, rnd, numberOfChannels));
			}
			else {
				writeBuffer(fileChannel, bufferRnd.nextInt(pageSize) + 1, bufferRnd.nextInt(numberOfChannels));
			}
		}

		// reset and create reader

		fileChannel.position(0L);
		bufferRnd.setSeed(bufferSeed);
		SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(tempFile, fileChannel, buffer, pageSize);
		seq.open();

		// read and validate the sequence

		int numEvent = 0;
		for (int i = 0; i < numEventsAndBuffers; i++) {
			BufferOrEvent next = seq.getNext();
			if (next.isEvent()) {
				BufferOrEvent expected = events.get(numEvent++);
				assertEquals(expected.getEvent(), next.getEvent());
				assertEquals(expected.getChannelIndex(), next.getChannelIndex());
			}
			else {
				validateBuffer(next, bufferRnd.nextInt(pageSize) + 1, bufferRnd.nextInt(numberOfChannels));
			}
		}

		// no further data
		assertNull(seq.getNext());

		// all events need to be consumed
		assertEquals(events.size(), numEvent);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}