Java Code Examples for org.apache.beam.sdk.coders.Coder#NonDeterministicException

private static void validateStateApplicableForInput(DoFn<?, ?> fn, PCollection<?> input) {
  Coder<?> inputCoder = input.getCoder();
  checkArgument(
      inputCoder instanceof KvCoder,
      "%s requires its input to use %s in order to use state and timers.",
      ParDo.class.getSimpleName(),
      KvCoder.class.getSimpleName());

  KvCoder<?, ?> kvCoder = (KvCoder<?, ?>) inputCoder;
  try {
    kvCoder.getKeyCoder().verifyDeterministic();
  } catch (Coder.NonDeterministicException exc) {
    throw new IllegalArgumentException(
        String.format(
            "%s requires a deterministic key coder in order to use state and timers",
            ParDo.class.getSimpleName()));
  }
}
 

@Test
public void testGroupByKeyIterator() throws Coder.NonDeterministicException {
  GroupByKeyIterator<String, Integer, GlobalWindow> iteratorUnderTest = createGbkIterator();

  Assert.assertTrue(iteratorUnderTest.hasNext());
  WindowedValue<KV<String, Iterable<Integer>>> k1Win = iteratorUnderTest.next();
  // testing that calling 2x hasNext doesn't move to next key iterator
  Assert.assertTrue(iteratorUnderTest.hasNext());
  Assert.assertTrue(iteratorUnderTest.hasNext());

  Iterator<Integer> valuesIteratorForK1 = k1Win.getValue().getValue().iterator();

  Assert.assertTrue("Now we expect first value for K1 to pop up.", valuesIteratorForK1.hasNext());
  assertEquals(1L, valuesIteratorForK1.next().longValue());
  Assert.assertTrue(valuesIteratorForK1.hasNext());
  Assert.assertTrue(valuesIteratorForK1.hasNext());
  assertEquals(2L, valuesIteratorForK1.next().longValue());

  WindowedValue<KV<String, Iterable<Integer>>> k2Win = iteratorUnderTest.next();
  Iterator<Integer> valuesIteratorForK2 = k2Win.getValue().getValue().iterator();
  assertEquals(3L, valuesIteratorForK2.next().longValue());
}
 

private <W extends BoundedWindow> GroupByKeyIterator<String, Integer, W> createGbkIterator(
    W window, Coder<W> winCoder, WindowingStrategy<Object, W> winStrategy)
    throws Coder.NonDeterministicException {

  StringUtf8Coder keyCoder = StringUtf8Coder.of();
  final WindowedValue.FullWindowedValueCoder<KV<String, Integer>> winValCoder =
      WindowedValue.getFullCoder(
          KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of()),
          winStrategy.getWindowFn().windowCoder());

  ItemFactory<String, Integer, W> factory =
      ItemFactory.forWindow(keyCoder, winValCoder, winCoder, window);
  List<Tuple2<ByteArray, byte[]>> items =
      Arrays.asList(
          factory.create("k1", 1),
          factory.create("k1", 2),
          factory.create("k2", 3),
          factory.create("k2", 4),
          factory.create("k2", 5));
  return new GroupByKeyIterator<>(items.iterator(), keyCoder, winStrategy, winValCoder);
}
 

/**
 * Returns a {@link CountMinSketchFn} combiner with the given input coder. <br>
 * <b>Warning :</b> the coder must be deterministic.
 *
 * @param coder the coder that encodes the elements' type
 */
public static <InputT> CountMinSketchFn<InputT> create(Coder<InputT> coder) {
  try {
    coder.verifyDeterministic();
  } catch (Coder.NonDeterministicException e) {
    throw new IllegalArgumentException(
        "Coder must be deterministic to perform this sketch." + e.getMessage(), e);
  }
  return new CountMinSketchFn<>(coder, 0.01, 0.999);
}
 

/**
 * Returns an {@link ApproximateDistinctFn} combiner with the given input coder.
 *
 * @param coder the coder that encodes the elements' type
 */
public static <InputT> ApproximateDistinctFn<InputT> create(Coder<InputT> coder) {
  try {
    coder.verifyDeterministic();
  } catch (Coder.NonDeterministicException e) {
    throw new IllegalArgumentException(
        "Coder must be deterministic to perform this sketch." + e.getMessage(), e);
  }
  return new ApproximateDistinctFn<>(12, 0, coder);
}
 

private static boolean isKeyAndWindowCoderConsistentWithEquals(
    Coder<?> keyCoder, Coder<?> windowCoder) {
  try {
    keyCoder.verifyDeterministic();
    windowCoder.verifyDeterministic();
    return keyCoder.consistentWithEquals() && windowCoder.consistentWithEquals();
  } catch (Coder.NonDeterministicException ex) {
    throw new IllegalArgumentException(
        "Coder for both key " + keyCoder + " and " + windowCoder + " must be deterministic", ex);
  }
}
 

GroupByKeyIterator(
    Iterator<Tuple2<ByteArray, byte[]>> inner,
    Coder<K> keyCoder,
    WindowingStrategy<?, W> windowingStrategy,
    WindowedValue.FullWindowedValueCoder<KV<K, V>> windowedValueCoder)
    throws Coder.NonDeterministicException {

  this.inner = Iterators.peekingIterator(inner);
  this.keyCoder = keyCoder;
  this.windowingStrategy = windowingStrategy;
  this.windowedValueCoder = windowedValueCoder;
}
 

@Test
public void testGroupByKeyIteratorOnNonGlobalWindows() throws Coder.NonDeterministicException {
  Instant now = Instant.now();
  GroupByKeyIterator<String, Integer, IntervalWindow> iteratorUnderTest =
      createGbkIterator(
          new IntervalWindow(now, now.plus(1)),
          IntervalWindow.getCoder(),
          WindowingStrategy.of(FixedWindows.of(Duration.millis(1))));

  Assert.assertTrue(iteratorUnderTest.hasNext());
  WindowedValue<KV<String, Iterable<Integer>>> k1Win = iteratorUnderTest.next();
  // testing that calling 2x hasNext doesn't move to next key iterator
  Assert.assertTrue(iteratorUnderTest.hasNext());
  Assert.assertTrue(iteratorUnderTest.hasNext());

  Iterator<Integer> valuesIteratorForK1 = k1Win.getValue().getValue().iterator();

  Assert.assertTrue("Now we expect first value for K1 to pop up.", valuesIteratorForK1.hasNext());
  assertEquals(1L, valuesIteratorForK1.next().longValue());
  Assert.assertTrue(valuesIteratorForK1.hasNext());
  Assert.assertTrue(valuesIteratorForK1.hasNext());
  assertEquals(2L, valuesIteratorForK1.next().longValue());

  WindowedValue<KV<String, Iterable<Integer>>> k2Win = iteratorUnderTest.next();
  Iterator<Integer> valuesIteratorForK2 = k2Win.getValue().getValue().iterator();
  assertEquals(3L, valuesIteratorForK2.next().longValue());
}
 

@Override
public void verifyDeterministic() throws Coder.NonDeterministicException {
  inputCoder.verifyDeterministic();
  accumCoder.verifyDeterministic();
}
 

@Override
public void verifyDeterministic() throws Coder.NonDeterministicException {
  keyCoder.verifyDeterministic();
  timersCoder.verifyDeterministic();
  elemsCoder.verifyDeterministic();
}
 

private GroupByKeyIterator<String, Integer, GlobalWindow> createGbkIterator()
    throws Coder.NonDeterministicException {
  return createGbkIterator(
      GlobalWindow.INSTANCE, GlobalWindow.Coder.INSTANCE, WindowingStrategy.globalDefault());
}
 

@Override
public void verifyDeterministic() throws Coder.NonDeterministicException {
  verifyDeterministic(
      this, "Key component coders expected to be deterministic.", keyComponentCoders);
  verifyDeterministic(this, "Value coder expected to be deterministic.", valueCoder);
}