Java Code Examples for org.apache.beam.sdk.transforms.windowing.IntervalWindow#getCoder()
The following examples show how to use
org.apache.beam.sdk.transforms.windowing.IntervalWindow#getCoder() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: TimerInternalsTest.java From beam with Apache License 2.0 | 6 votes |
|
@Test
public void testTimerDataCoder() throws Exception {
CoderProperties.coderDecodeEncodeEqual(
TimerDataCoderV2.of(GlobalWindow.Coder.INSTANCE),
TimerData.of(
"arbitrary-id",
StateNamespaces.global(),
new Instant(0),
new Instant(0),
TimeDomain.EVENT_TIME));
Coder<IntervalWindow> windowCoder = IntervalWindow.getCoder();
CoderProperties.coderDecodeEncodeEqual(
TimerDataCoderV2.of(windowCoder),
TimerData.of(
"another-id",
StateNamespaces.window(
windowCoder, new IntervalWindow(new Instant(0), new Instant(100))),
new Instant(99),
new Instant(99),
TimeDomain.PROCESSING_TIME));
}
Example 2
| Source File: TimerInternalsTest.java From beam with Apache License 2.0 | 6 votes |
|
@Test
public void testCompareByNamespace() {
Instant timestamp = new Instant(100);
IntervalWindow firstWindow = new IntervalWindow(new Instant(0), timestamp);
IntervalWindow secondWindow = new IntervalWindow(timestamp, new Instant(200));
Coder<IntervalWindow> windowCoder = IntervalWindow.getCoder();
StateNamespace firstWindowNs = StateNamespaces.window(windowCoder, firstWindow);
StateNamespace secondWindowNs = StateNamespaces.window(windowCoder, secondWindow);
TimerData secondEventTime =
TimerData.of(firstWindowNs, timestamp, timestamp, TimeDomain.EVENT_TIME);
TimerData thirdEventTime =
TimerData.of(secondWindowNs, timestamp, timestamp, TimeDomain.EVENT_TIME);
assertThat(secondEventTime, lessThan(thirdEventTime));
}
Example 3
| Source File: ValidityWindows.java From streamingbook with Apache License 2.0 | 4 votes |
|
@Override
public Coder<IntervalWindow> windowCoder() {
return IntervalWindow.getCoder();
}
Example 4
| Source File: TVFSlidingWindowFn.java From beam with Apache License 2.0 | 4 votes |
|
@Override
public Coder<IntervalWindow> windowCoder() {
return IntervalWindow.getCoder();
}
Example 5
| Source File: BatchViewOverridesTest.java From beam with Apache License 2.0 | 4 votes |
|
@Test
public void testToIsmRecordForMapLikeDoFnWithoutUniqueKeysThrowsException() throws Exception {
TupleTag<KV<Integer, KV<IntervalWindow, Long>>> outputForSizeTag = new TupleTag<>();
TupleTag<KV<Integer, KV<IntervalWindow, Long>>> outputForEntrySetTag = new TupleTag<>();
Coder<Long> keyCoder = VarLongCoder.of();
Coder<IntervalWindow> windowCoder = IntervalWindow.getCoder();
IsmRecordCoder<WindowedValue<Long>> ismCoder =
IsmRecordCoder.of(
1,
2,
ImmutableList.of(
MetadataKeyCoder.of(keyCoder), IntervalWindow.getCoder(), BigEndianLongCoder.of()),
FullWindowedValueCoder.of(VarLongCoder.of(), windowCoder));
DoFnTester<
KV<Integer, Iterable<KV<KV<Long, IntervalWindow>, WindowedValue<Long>>>>,
IsmRecord<WindowedValue<Long>>>
doFnTester =
DoFnTester.of(
new BatchViewOverrides.BatchViewAsMultimap.ToIsmRecordForMapLikeDoFn<>(
outputForSizeTag,
outputForEntrySetTag,
windowCoder,
keyCoder,
ismCoder,
true /* unique keys */));
IntervalWindow windowA = new IntervalWindow(new Instant(0), new Instant(10));
Iterable<KV<Integer, Iterable<KV<KV<Long, IntervalWindow>, WindowedValue<Long>>>>>
inputElements =
ImmutableList.of(
KV.of(
1,
(Iterable<KV<KV<Long, IntervalWindow>, WindowedValue<Long>>>)
ImmutableList.of(
KV.of(
KV.of(1L, windowA),
WindowedValue.of(
110L, new Instant(1), windowA, PaneInfo.NO_FIRING)),
// same window same key as to previous
KV.of(
KV.of(1L, windowA),
WindowedValue.of(
111L, new Instant(2), windowA, PaneInfo.NO_FIRING)))));
thrown.expect(IllegalStateException.class);
thrown.expectMessage("Unique keys are expected but found key");
doFnTester.processBundle(inputElements);
}
Example 6
| Source File: BatchViewOverridesTest.java From beam with Apache License 2.0 | 4 votes |
|
@Test
public void testToIsmMetadataRecordForSizeDoFn() throws Exception {
Coder<Long> keyCoder = VarLongCoder.of();
Coder<IntervalWindow> windowCoder = IntervalWindow.getCoder();
IsmRecordCoder<WindowedValue<Long>> ismCoder =
IsmRecordCoder.of(
1,
2,
ImmutableList.of(
MetadataKeyCoder.of(keyCoder), IntervalWindow.getCoder(), BigEndianLongCoder.of()),
FullWindowedValueCoder.of(VarLongCoder.of(), windowCoder));
DoFnTester<KV<Integer, Iterable<KV<IntervalWindow, Long>>>, IsmRecord<WindowedValue<Long>>>
doFnTester =
DoFnTester.of(
new BatchViewOverrides.BatchViewAsMultimap.ToIsmMetadataRecordForSizeDoFn<
Long, Long, IntervalWindow>(windowCoder));
IntervalWindow windowA = new IntervalWindow(new Instant(0), new Instant(10));
IntervalWindow windowB = new IntervalWindow(new Instant(10), new Instant(20));
IntervalWindow windowC = new IntervalWindow(new Instant(20), new Instant(30));
Iterable<KV<Integer, Iterable<KV<IntervalWindow, Long>>>> inputElements =
ImmutableList.of(
KV.of(
1,
(Iterable<KV<IntervalWindow, Long>>)
ImmutableList.of(KV.of(windowA, 2L), KV.of(windowA, 3L), KV.of(windowB, 7L))),
KV.of(
ismCoder.hash(ImmutableList.of(IsmFormat.getMetadataKey(), windowB)),
(Iterable<KV<IntervalWindow, Long>>) ImmutableList.of(KV.of(windowC, 9L))));
// The order of the output elements is important relative to processing order
assertThat(
doFnTester.processBundle(inputElements),
contains(
IsmRecord.<WindowedValue<Long>>meta(
ImmutableList.of(IsmFormat.getMetadataKey(), windowA, 0L),
CoderUtils.encodeToByteArray(VarLongCoder.of(), 5L)),
IsmRecord.<WindowedValue<Long>>meta(
ImmutableList.of(IsmFormat.getMetadataKey(), windowB, 0L),
CoderUtils.encodeToByteArray(VarLongCoder.of(), 7L)),
IsmRecord.<WindowedValue<Long>>meta(
ImmutableList.of(IsmFormat.getMetadataKey(), windowC, 0L),
CoderUtils.encodeToByteArray(VarLongCoder.of(), 9L))));
}
Example 7
| Source File: BatchViewOverridesTest.java From beam with Apache License 2.0 | 4 votes |
|
@Test
public void testToIsmMetadataRecordForKeyDoFn() throws Exception {
Coder<Long> keyCoder = VarLongCoder.of();
Coder<IntervalWindow> windowCoder = IntervalWindow.getCoder();
IsmRecordCoder<WindowedValue<Long>> ismCoder =
IsmRecordCoder.of(
1,
2,
ImmutableList.of(
MetadataKeyCoder.of(keyCoder), IntervalWindow.getCoder(), BigEndianLongCoder.of()),
FullWindowedValueCoder.of(VarLongCoder.of(), windowCoder));
DoFnTester<KV<Integer, Iterable<KV<IntervalWindow, Long>>>, IsmRecord<WindowedValue<Long>>>
doFnTester =
DoFnTester.of(
new BatchViewOverrides.BatchViewAsMultimap.ToIsmMetadataRecordForKeyDoFn<
Long, Long, IntervalWindow>(keyCoder, windowCoder));
IntervalWindow windowA = new IntervalWindow(new Instant(0), new Instant(10));
IntervalWindow windowB = new IntervalWindow(new Instant(10), new Instant(20));
IntervalWindow windowC = new IntervalWindow(new Instant(20), new Instant(30));
Iterable<KV<Integer, Iterable<KV<IntervalWindow, Long>>>> inputElements =
ImmutableList.of(
KV.of(
1,
(Iterable<KV<IntervalWindow, Long>>)
ImmutableList.of(
KV.of(windowA, 2L),
// same window as previous
KV.of(windowA, 3L),
// different window as previous
KV.of(windowB, 3L))),
KV.of(
ismCoder.hash(ImmutableList.of(IsmFormat.getMetadataKey(), windowB)),
(Iterable<KV<IntervalWindow, Long>>) ImmutableList.of(KV.of(windowC, 3L))));
// The order of the output elements is important relative to processing order
assertThat(
doFnTester.processBundle(inputElements),
contains(
IsmRecord.<WindowedValue<Long>>meta(
ImmutableList.of(IsmFormat.getMetadataKey(), windowA, 1L),
CoderUtils.encodeToByteArray(VarLongCoder.of(), 2L)),
IsmRecord.<WindowedValue<Long>>meta(
ImmutableList.of(IsmFormat.getMetadataKey(), windowA, 2L),
CoderUtils.encodeToByteArray(VarLongCoder.of(), 3L)),
IsmRecord.<WindowedValue<Long>>meta(
ImmutableList.of(IsmFormat.getMetadataKey(), windowB, 1L),
CoderUtils.encodeToByteArray(VarLongCoder.of(), 3L)),
IsmRecord.<WindowedValue<Long>>meta(
ImmutableList.of(IsmFormat.getMetadataKey(), windowC, 1L),
CoderUtils.encodeToByteArray(VarLongCoder.of(), 3L))));
}
Example 8
| Source File: BatchViewOverridesTest.java From beam with Apache License 2.0 | 4 votes |
|
@Test
public void testToMapDoFn() throws Exception {
Coder<IntervalWindow> windowCoder = IntervalWindow.getCoder();
DoFnTester<
KV<Integer, Iterable<KV<IntervalWindow, WindowedValue<KV<Long, Long>>>>>,
IsmRecord<WindowedValue<TransformedMap<Long, WindowedValue<Long>, Long>>>>
doFnTester =
DoFnTester.of(
new BatchViewOverrides.BatchViewAsMap.ToMapDoFn<Long, Long, IntervalWindow>(
windowCoder));
IntervalWindow windowA = new IntervalWindow(new Instant(0), new Instant(10));
IntervalWindow windowB = new IntervalWindow(new Instant(10), new Instant(20));
IntervalWindow windowC = new IntervalWindow(new Instant(20), new Instant(30));
Iterable<KV<Integer, Iterable<KV<IntervalWindow, WindowedValue<KV<Long, Long>>>>>>
inputElements =
ImmutableList.of(
KV.of(
1,
(Iterable<KV<IntervalWindow, WindowedValue<KV<Long, Long>>>>)
ImmutableList.of(
KV.of(
windowA,
WindowedValue.of(
KV.of(1L, 11L), new Instant(3), windowA, PaneInfo.NO_FIRING)),
KV.of(
windowA,
WindowedValue.of(
KV.of(2L, 21L), new Instant(7), windowA, PaneInfo.NO_FIRING)),
KV.of(
windowB,
WindowedValue.of(
KV.of(2L, 21L), new Instant(13), windowB, PaneInfo.NO_FIRING)),
KV.of(
windowB,
WindowedValue.of(
KV.of(3L, 31L),
new Instant(15),
windowB,
PaneInfo.NO_FIRING)))),
KV.of(
2,
(Iterable<KV<IntervalWindow, WindowedValue<KV<Long, Long>>>>)
ImmutableList.of(
KV.of(
windowC,
WindowedValue.of(
KV.of(4L, 41L),
new Instant(25),
windowC,
PaneInfo.NO_FIRING)))));
// The order of the output elements is important relative to processing order
List<IsmRecord<WindowedValue<TransformedMap<Long, WindowedValue<Long>, Long>>>> output =
doFnTester.processBundle(inputElements);
assertEquals(3, output.size());
Map<Long, Long> outputMap;
outputMap = output.get(0).getValue().getValue();
assertEquals(2, outputMap.size());
assertEquals(ImmutableMap.of(1L, 11L, 2L, 21L), outputMap);
outputMap = output.get(1).getValue().getValue();
assertEquals(2, outputMap.size());
assertEquals(ImmutableMap.of(2L, 21L, 3L, 31L), outputMap);
outputMap = output.get(2).getValue().getValue();
assertEquals(1, outputMap.size());
assertEquals(ImmutableMap.of(4L, 41L), outputMap);
}
Example 9
| Source File: BatchViewOverridesTest.java From beam with Apache License 2.0 | 4 votes |
|
@Test
public void testToMultimapDoFn() throws Exception {
Coder<IntervalWindow> windowCoder = IntervalWindow.getCoder();
DoFnTester<
KV<Integer, Iterable<KV<IntervalWindow, WindowedValue<KV<Long, Long>>>>>,
IsmRecord<
WindowedValue<TransformedMap<Long, Iterable<WindowedValue<Long>>, Iterable<Long>>>>>
doFnTester =
DoFnTester.of(
new BatchViewOverrides.BatchViewAsMultimap.ToMultimapDoFn<
Long, Long, IntervalWindow>(windowCoder));
IntervalWindow windowA = new IntervalWindow(new Instant(0), new Instant(10));
IntervalWindow windowB = new IntervalWindow(new Instant(10), new Instant(20));
IntervalWindow windowC = new IntervalWindow(new Instant(20), new Instant(30));
Iterable<KV<Integer, Iterable<KV<IntervalWindow, WindowedValue<KV<Long, Long>>>>>>
inputElements =
ImmutableList.of(
KV.of(
1,
(Iterable<KV<IntervalWindow, WindowedValue<KV<Long, Long>>>>)
ImmutableList.of(
KV.of(
windowA,
WindowedValue.of(
KV.of(1L, 11L), new Instant(3), windowA, PaneInfo.NO_FIRING)),
// [BEAM-5184] Specifically test with a duplicate value to ensure that
// duplicate key/values are not lost.
KV.of(
windowA,
WindowedValue.of(
KV.of(1L, 11L), new Instant(3), windowA, PaneInfo.NO_FIRING)),
KV.of(
windowA,
WindowedValue.of(
KV.of(1L, 12L), new Instant(5), windowA, PaneInfo.NO_FIRING)),
KV.of(
windowA,
WindowedValue.of(
KV.of(2L, 21L), new Instant(7), windowA, PaneInfo.NO_FIRING)),
KV.of(
windowB,
WindowedValue.of(
KV.of(2L, 21L), new Instant(13), windowB, PaneInfo.NO_FIRING)),
KV.of(
windowB,
WindowedValue.of(
KV.of(3L, 31L),
new Instant(15),
windowB,
PaneInfo.NO_FIRING)))),
KV.of(
2,
(Iterable<KV<IntervalWindow, WindowedValue<KV<Long, Long>>>>)
ImmutableList.of(
KV.of(
windowC,
WindowedValue.of(
KV.of(4L, 41L),
new Instant(25),
windowC,
PaneInfo.NO_FIRING)))));
// The order of the output elements is important relative to processing order
List<
IsmRecord<
WindowedValue<TransformedMap<Long, Iterable<WindowedValue<Long>>, Iterable<Long>>>>>
output = doFnTester.processBundle(inputElements);
assertEquals(3, output.size());
Map<Long, Iterable<Long>> outputMap;
outputMap = output.get(0).getValue().getValue();
assertEquals(2, outputMap.size());
assertThat(outputMap.get(1L), containsInAnyOrder(11L, 11L, 12L));
assertThat(outputMap.get(2L), containsInAnyOrder(21L));
outputMap = output.get(1).getValue().getValue();
assertEquals(2, outputMap.size());
assertThat(outputMap.get(2L), containsInAnyOrder(21L));
assertThat(outputMap.get(3L), containsInAnyOrder(31L));
outputMap = output.get(2).getValue().getValue();
assertEquals(1, outputMap.size());
assertThat(outputMap.get(4L), containsInAnyOrder(41L));
}
Example 10
| Source File: UserParDoFnFactoryTest.java From beam with Apache License 2.0 | 4 votes |
|
@Test
public void testCleanupWorks() throws Exception {
PipelineOptions options = PipelineOptionsFactory.create();
CounterSet counters = new CounterSet();
DoFn<?, ?> initialFn = new TestStatefulDoFn();
CloudObject cloudObject =
getCloudObject(initialFn, WindowingStrategy.of(FixedWindows.of(Duration.millis(10))));
StateInternals stateInternals = InMemoryStateInternals.forKey("dummy");
// The overarching step context that only ParDoFn gets
DataflowStepContext stepContext = mock(DataflowStepContext.class);
// The user step context that the DoFnRunner gets a handle on
DataflowStepContext userStepContext = mock(DataflowStepContext.class);
when(stepContext.namespacedToUser()).thenReturn(userStepContext);
when(stepContext.stateInternals()).thenReturn(stateInternals);
when(userStepContext.stateInternals()).thenReturn((StateInternals) stateInternals);
DataflowExecutionContext<DataflowStepContext> executionContext =
mock(DataflowExecutionContext.class);
TestOperationContext operationContext = TestOperationContext.create(counters);
when(executionContext.getStepContext(operationContext)).thenReturn(stepContext);
when(executionContext.getSideInputReader(any(), any(), any()))
.thenReturn(NullSideInputReader.empty());
ParDoFn parDoFn =
factory.create(
options,
cloudObject,
Collections.emptyList(),
MAIN_OUTPUT,
ImmutableMap.of(MAIN_OUTPUT, 0),
executionContext,
operationContext);
Receiver rcvr = new OutputReceiver();
parDoFn.startBundle(rcvr);
IntervalWindow firstWindow = new IntervalWindow(new Instant(0), new Instant(9));
IntervalWindow secondWindow = new IntervalWindow(new Instant(10), new Instant(19));
Coder<IntervalWindow> windowCoder = IntervalWindow.getCoder();
StateNamespace firstWindowNamespace = StateNamespaces.window(windowCoder, firstWindow);
StateNamespace secondWindowNamespace = StateNamespaces.window(windowCoder, secondWindow);
StateTag<ValueState<String>> tag =
StateTags.tagForSpec(TestStatefulDoFn.STATE_ID, StateSpecs.value(StringUtf8Coder.of()));
// Set up non-empty state. We don't mock + verify calls to clear() but instead
// check that state is actually empty. We musn't care how it is accomplished.
stateInternals.state(firstWindowNamespace, tag).write("first");
stateInternals.state(secondWindowNamespace, tag).write("second");
when(userStepContext.getNextFiredTimer(windowCoder)).thenReturn(null);
when(stepContext.getNextFiredTimer(windowCoder))
.thenReturn(
TimerData.of(
SimpleParDoFn.CLEANUP_TIMER_ID,
firstWindowNamespace,
firstWindow.maxTimestamp().plus(1L),
firstWindow.maxTimestamp().plus(1L),
TimeDomain.EVENT_TIME))
.thenReturn(null);
// This should fire the timer to clean up the first window
parDoFn.processTimers();
assertThat(stateInternals.state(firstWindowNamespace, tag).read(), nullValue());
assertThat(stateInternals.state(secondWindowNamespace, tag).read(), equalTo("second"));
when(stepContext.getNextFiredTimer((Coder) windowCoder))
.thenReturn(
TimerData.of(
SimpleParDoFn.CLEANUP_TIMER_ID,
secondWindowNamespace,
secondWindow.maxTimestamp().plus(1L),
secondWindow.maxTimestamp().plus(1L),
TimeDomain.EVENT_TIME))
.thenReturn(null);
// And this should clean up the second window
parDoFn.processTimers();
assertThat(stateInternals.state(firstWindowNamespace, tag).read(), nullValue());
assertThat(stateInternals.state(secondWindowNamespace, tag).read(), nullValue());
}
