Java Code Examples for org.apache.beam.sdk.coders.IterableCoder#getElemCoder()

The following examples show how to use org.apache.beam.sdk.coders.IterableCoder#getElemCoder() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SortValues.java    From beam with Apache License 2.0 6 votes vote down vote up 
/** Retrieves the {@link Coder} for the secondary key-value pairs. */
@SuppressWarnings("unchecked")
private static <PrimaryKeyT, SecondaryKeyT, ValueT>
    KvCoder<SecondaryKeyT, ValueT> getSecondaryKeyValueCoder(
        Coder<KV<PrimaryKeyT, Iterable<KV<SecondaryKeyT, ValueT>>>> inputCoder) {
  if (!(inputCoder instanceof KvCoder)) {
    throw new IllegalStateException("SortValues requires its input to use KvCoder");
  }
  @SuppressWarnings("unchecked")
  KvCoder<PrimaryKeyT, Iterable<KV<SecondaryKeyT, ValueT>>> kvCoder =
      (KvCoder<PrimaryKeyT, Iterable<KV<SecondaryKeyT, ValueT>>>) inputCoder;

  if (!(kvCoder.getValueCoder() instanceof IterableCoder)) {
    throw new IllegalStateException(
        "SortValues requires the values be encoded with IterableCoder");
  }
  IterableCoder<KV<SecondaryKeyT, ValueT>> iterableCoder =
      (IterableCoder<KV<SecondaryKeyT, ValueT>>) (kvCoder.getValueCoder());

  if (!(iterableCoder.getElemCoder() instanceof KvCoder)) {
    throw new IllegalStateException(
        "SortValues requires the secondary key-value pairs to use KvCoder");
  }
  return (KvCoder<SecondaryKeyT, ValueT>) (iterableCoder.getElemCoder());
}
Example 2
Source File: Combine.java    From beam with Apache License 2.0 6 votes vote down vote up 
private KvCoder<K, InputT> getKvCoder(
    Coder<? extends KV<K, ? extends Iterable<InputT>>> inputCoder) {
  if (!(inputCoder instanceof KvCoder)) {
    throw new IllegalStateException("Combine.GroupedValues requires its input to use KvCoder");
  }
  @SuppressWarnings({"unchecked", "rawtypes"})
  KvCoder<K, ? extends Iterable<InputT>> kvCoder = (KvCoder) inputCoder;
  Coder<K> keyCoder = kvCoder.getKeyCoder();
  Coder<? extends Iterable<InputT>> kvValueCoder = kvCoder.getValueCoder();
  if (!(kvValueCoder instanceof IterableCoder)) {
    throw new IllegalStateException(
        "Combine.GroupedValues requires its input values to use " + "IterableCoder");
  }
  @SuppressWarnings("unchecked")
  IterableCoder<InputT> inputValuesCoder = (IterableCoder<InputT>) kvValueCoder;
  Coder<InputT> inputValueCoder = inputValuesCoder.getElemCoder();
  return KvCoder.of(keyCoder, inputValueCoder);
}
Example 3
Source File: GroupByKeyViaGroupByKeyOnly.java    From beam with Apache License 2.0 6 votes vote down vote up 
@Override
public PCollection<KV<K, Iterable<V>>> expand(
    PCollection<KV<K, Iterable<WindowedValue<V>>>> input) {
  @SuppressWarnings("unchecked")
  KvCoder<K, Iterable<WindowedValue<V>>> inputKvCoder =
      (KvCoder<K, Iterable<WindowedValue<V>>>) input.getCoder();

  Coder<K> keyCoder = inputKvCoder.getKeyCoder();
  Coder<Iterable<WindowedValue<V>>> inputValueCoder = inputKvCoder.getValueCoder();

  IterableCoder<WindowedValue<V>> inputIterableValueCoder =
      (IterableCoder<WindowedValue<V>>) inputValueCoder;
  Coder<WindowedValue<V>> inputIterableElementCoder = inputIterableValueCoder.getElemCoder();
  WindowedValueCoder<V> inputIterableWindowedValueCoder =
      (WindowedValueCoder<V>) inputIterableElementCoder;

  Coder<V> inputIterableElementValueCoder = inputIterableWindowedValueCoder.getValueCoder();
  Coder<Iterable<V>> outputValueCoder = IterableCoder.of(inputIterableElementValueCoder);
  Coder<KV<K, Iterable<V>>> outputKvCoder = KvCoder.of(keyCoder, outputValueCoder);

  return PCollection.createPrimitiveOutputInternal(
      input.getPipeline(), windowingStrategy, input.isBounded(), outputKvCoder);
}
Example 4
Source File: GroupByKeyViaGroupByKeyOnly.java    From beam with Apache License 2.0 5 votes vote down vote up 
public Coder<V> getValueCoder(Coder<KV<K, Iterable<WindowedValue<V>>>> inputCoder) {
  // Coder<Iterable<...>> --> IterableCoder<...>
  Coder<Iterable<WindowedValue<V>>> iterableWindowedValueCoder =
      getKvCoder(inputCoder).getValueCoder();
  checkArgument(
      iterableWindowedValueCoder instanceof IterableCoder,
      "%s requires a %s<..., %s> but got a %s",
      getClass().getSimpleName(),
      KvCoder.class.getSimpleName(),
      IterableCoder.class.getSimpleName(),
      iterableWindowedValueCoder);
  IterableCoder<WindowedValue<V>> iterableCoder =
      (IterableCoder<WindowedValue<V>>) iterableWindowedValueCoder;

  // Coder<WindowedValue<...>> --> WindowedValueCoder<...>
  Coder<WindowedValue<V>> iterableElementCoder = iterableCoder.getElemCoder();
  checkArgument(
      iterableElementCoder instanceof WindowedValueCoder,
      "%s requires a %s<..., %s<%s>> but got a %s",
      getClass().getSimpleName(),
      KvCoder.class.getSimpleName(),
      IterableCoder.class.getSimpleName(),
      WindowedValueCoder.class.getSimpleName(),
      iterableElementCoder);
  WindowedValueCoder<V> windowedValueCoder = (WindowedValueCoder<V>) iterableElementCoder;

  return windowedValueCoder.getValueCoder();
}
Example 5
Source File: GroupingShuffleReader.java    From beam with Apache License 2.0 5 votes vote down vote up 
private void initCoder(Coder<WindowedValue<KV<K, Iterable<V>>>> coder, boolean valuesAreSorted)
    throws Exception {
  if (!(coder instanceof WindowedValueCoder)) {
    throw new Exception("unexpected kind of coder for WindowedValue: " + coder);
  }
  Coder<KV<K, Iterable<V>>> elemCoder =
      ((WindowedValueCoder<KV<K, Iterable<V>>>) coder).getValueCoder();
  if (!(elemCoder instanceof KvCoder)) {
    throw new Exception(
        "unexpected kind of coder for elements read from "
            + "a key-grouping shuffle: "
            + elemCoder);
  }

  @SuppressWarnings("unchecked")
  KvCoder<K, Iterable<V>> kvCoder = (KvCoder<K, Iterable<V>>) elemCoder;
  this.keyCoder = kvCoder.getKeyCoder();
  Coder<Iterable<V>> kvValueCoder = kvCoder.getValueCoder();
  if (!(kvValueCoder instanceof IterableCoder)) {
    throw new Exception(
        "unexpected kind of coder for values of KVs read from " + "a key-grouping shuffle");
  }
  IterableCoder<V> iterCoder = (IterableCoder<V>) kvValueCoder;
  if (valuesAreSorted) {
    checkState(
        iterCoder.getElemCoder() instanceof KvCoder,
        "unexpected kind of coder for elements read from a "
            + "key-grouping value sorting shuffle: %s",
        iterCoder.getElemCoder());
    @SuppressWarnings("rawtypes")
    KvCoder<?, ?> valueKvCoder = (KvCoder) iterCoder.getElemCoder();
    this.secondaryKeyCoder = valueKvCoder.getKeyCoder();
    this.valueCoder = valueKvCoder.getValueCoder();
  } else {
    this.valueCoder = iterCoder.getElemCoder();
  }
}