Java Code Examples for org.apache.beam.sdk.transforms.Materializations#MultimapView

The following examples show how to use org.apache.beam.sdk.transforms.Materializations#MultimapView . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CreateViewTransformTest.java    From incubator-nemo with Apache License 2.0 5 votes vote down vote up 
@Override
public Integer apply(final Materializations.MultimapView<Void, String> view) {
  int sum = 0;
  // MultimapView.get is Nullable
  for (String s : view.get(null)) {
    sum += 1;
  }
  return sum;
}
Example 2
Source File: DataflowSideInputHandlerFactory.java    From beam with Apache License 2.0 5 votes vote down vote up 
private DataflowMultimapSideInputHandler(
    SideInputReader sideInputReader,
    PCollectionView<Materializations.MultimapView<Object, Object>> view,
    Coder<K> keyCoder,
    Coder<V> valueCoder,
    Coder<W> windowCoder) {
  this.sideInputReader = sideInputReader;
  this.view = view;
  this.keyCoder = keyCoder;
  this.valueCoder = valueCoder;
  this.windowCoder = windowCoder;
}
Example 3
Source File: DataflowSideInputHandlerFactory.java    From beam with Apache License 2.0 5 votes vote down vote up 
@Override
public Iterable<K> get(W window) {
  Materializations.MultimapView<K, V> sideInput =
      (Materializations.MultimapView<K, V>)
          sideInputReader.get(view, (BoundedWindow) windowCoder.structuralValue(window));
  return sideInput.get();
}
Example 4
Source File: DataflowSideInputHandlerFactory.java    From beam with Apache License 2.0 5 votes vote down vote up 
@Override
public Iterable<V> get(K key, W window) {
  Materializations.MultimapView<K, V> sideInput =
      (Materializations.MultimapView<K, V>)
          sideInputReader.get(view, (BoundedWindow) windowCoder.structuralValue(window));

  return sideInput.get(key);
}
Example 5
Source File: CreateViewTransform.java    From incubator-nemo with Apache License 2.0 4 votes vote down vote up 
/**
 * Constructor of CreateViewTransform.
 *
 * @param viewFn the viewFn that materializes data.
 */
public CreateViewTransform(final ViewFn<Materializations.MultimapView<Void, ?>, O> viewFn) {
  this.viewFn = viewFn;
  this.windowListMap = new HashMap<>();
  this.currentOutputWatermark = Long.MIN_VALUE;
}
Example 6
Source File: CreateViewTransformTest.java    From incubator-nemo with Apache License 2.0 4 votes vote down vote up 
@Override
public Materialization<Materializations.MultimapView<Void, String>> getMaterialization() {
  throw new UnsupportedOperationException();
}
Example 7
Source File: DataflowSideInputHandlerFactory.java    From beam with Apache License 2.0 4 votes vote down vote up 
@Override
public <K, V, W extends BoundedWindow> MultimapSideInputHandler<K, V, W> forMultimapSideInput(
    String pTransformId, String sideInputId, KvCoder<K, V> elementCoder, Coder<W> windowCoder) {
  checkArgument(
      pTransformId != null && pTransformId.length() > 0, "Expect a valid PTransform ID.");

  SideInputReader sideInputReader = ptransformIdToSideInputReader.get(pTransformId);
  checkState(sideInputReader != null, String.format("Unknown PTransform '%s'", pTransformId));

  PCollectionView<Materializations.MultimapView<Object, Object>> view =
      (PCollectionView<Materializations.MultimapView<Object, Object>>)
          sideInputIdToPCollectionViewMap.get(
              RunnerApi.ExecutableStagePayload.SideInputId.newBuilder()
                  .setTransformId(pTransformId)
                  .setLocalName(sideInputId)
                  .build());
  checkState(
      view != null,
      String.format("Unknown side input '%s' on PTransform '%s'", sideInputId, pTransformId));

  checkState(
      Materializations.MULTIMAP_MATERIALIZATION_URN.equals(
          view.getViewFn().getMaterialization().getUrn()),
      String.format(
          "Unknown materialization for side input '%s' on PTransform '%s' with urn '%s'",
          sideInputId, pTransformId, view.getViewFn().getMaterialization().getUrn()));

  checkState(
      view.getCoderInternal() instanceof KvCoder,
      String.format(
          "Materialization of side input '%s' on PTransform '%s' expects %s but received %s.",
          sideInputId,
          pTransformId,
          KvCoder.class.getSimpleName(),
          view.getCoderInternal().getClass().getSimpleName()));

  KvCoder<K, V> kvCoder = elementCoder;

  return new DataflowMultimapSideInputHandler<>(
      sideInputReader, view, kvCoder.getKeyCoder(), kvCoder.getValueCoder(), windowCoder);
}