Java Code Examples for org.apache.beam.sdk.transforms.View#CreatePCollectionView

The following examples show how to use org.apache.beam.sdk.transforms.View#CreatePCollectionView . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: PCollectionViewTranslatorBatch.java From twister2 with Apache License 2.0

6 votes

@Override
public void translateNode(
    View.CreatePCollectionView<ET, VT> transform, Twister2BatchTranslationContext context) {
  BatchTSet<WindowedValue<ET>> inputDataSet
      = context.getInputDataSet(context.getInput(transform));
  @SuppressWarnings("unchecked")
  AppliedPTransform<
      PCollection<ET>,
      PCollection<ET>,
      PTransform<PCollection<ET>, PCollection<ET>>>
      application =
      (AppliedPTransform<
          PCollection<ET>,
          PCollection<ET>,
          PTransform<PCollection<ET>, PCollection<ET>>>)
          context.getCurrentTransform();
  org.apache.beam.sdk.values.PCollectionView<VT> input;
  try {
    input = CreatePCollectionViewTranslation.getView(application);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
  context.setSideInputDataSet(input.getTagInternal().getId(), inputDataSet);
}

Example 2

Source File: DataflowPipelineTranslator.java From beam with Apache License 2.0

6 votes

@Override
public void visitValue(PValue value, TransformHierarchy.Node producer) {
  LOG.debug("Checking translation of {}", value);
  // Primitive transforms are the only ones assigned step names.
  if (producer.getTransform() instanceof CreateDataflowView
      && !hasExperiment(options, "beam_fn_api")) {
    // CreateDataflowView produces a dummy output (as it must be a primitive transform)
    // but in the Dataflow Job graph produces only the view and not the output PCollection.
    asOutputReference(
        ((CreateDataflowView) producer.getTransform()).getView(),
        producer.toAppliedPTransform(getPipeline()));
    return;
  } else if (producer.getTransform() instanceof View.CreatePCollectionView
      && hasExperiment(options, "beam_fn_api")) {
    // View.CreatePCollectionView produces a dummy output (as it must be a primitive transform)
    // but in the Dataflow Job graph produces only the view and not the output PCollection.
    asOutputReference(
        ((View.CreatePCollectionView) producer.getTransform()).getView(),
        producer.toAppliedPTransform(getPipeline()));
    return;
  }
  asOutputReference(value, producer.toAppliedPTransform(getPipeline()));
}

Example 3

Source File: DataflowPipelineTranslator.java From beam with Apache License 2.0

6 votes

private <ElemT, ViewT> void translateTyped(
    View.CreatePCollectionView<ElemT, ViewT> transform, TranslationContext context) {
  StepTranslationContext stepContext =
      context.addStep(transform, "CollectionToSingleton");
  PCollection<ElemT> input = context.getInput(transform);
  stepContext.addInput(PropertyNames.PARALLEL_INPUT, input);
  WindowingStrategy<?, ?> windowingStrategy = input.getWindowingStrategy();
  stepContext.addInput(
      PropertyNames.WINDOWING_STRATEGY,
      byteArrayToJsonString(
          serializeWindowingStrategy(windowingStrategy, context.getPipelineOptions())));
  stepContext.addInput(
      PropertyNames.IS_MERGING_WINDOW_FN,
      !windowingStrategy.getWindowFn().isNonMerging());
  stepContext.addCollectionToSingletonOutput(
      input, PropertyNames.OUTPUT, transform.getView());
}

Example 4

Source File: PCollectionViewTranslatorBatch.java From beam with Apache License 2.0

5 votes

@Override
public void translateNode(
    View.CreatePCollectionView<ElemT, ViewT> transform, Twister2BatchTranslationContext context) {
  BatchTSet<WindowedValue<ElemT>> inputDataSet =
      context.getInputDataSet(context.getInput(transform));
  @SuppressWarnings("unchecked")
  AppliedPTransform<
          PCollection<ElemT>,
          PCollection<ElemT>,
          PTransform<PCollection<ElemT>, PCollection<ElemT>>>
      application =
          (AppliedPTransform<
                  PCollection<ElemT>,
                  PCollection<ElemT>,
                  PTransform<PCollection<ElemT>, PCollection<ElemT>>>)
              context.getCurrentTransform();
  org.apache.beam.sdk.values.PCollectionView<ViewT> input;
  PCollection<ElemT> inputPCol = context.getInput(transform);
  final KvCoder coder = (KvCoder) inputPCol.getCoder();
  Coder inputKeyCoder = coder.getKeyCoder();
  WindowingStrategy windowingStrategy = inputPCol.getWindowingStrategy();
  WindowFn windowFn = windowingStrategy.getWindowFn();
  final WindowedValue.WindowedValueCoder wvCoder =
      WindowedValue.FullWindowedValueCoder.of(coder.getValueCoder(), windowFn.windowCoder());
  BatchTSet<WindowedValue<ElemT>> inputGathered =
      inputDataSet
          .direct()
          .map(new MapToTupleFunction<>(inputKeyCoder, wvCoder))
          .allGather()
          .map(new ByteToWindowFunctionPrimitive(inputKeyCoder, wvCoder));
  try {
    input = CreatePCollectionViewTranslation.getView(application);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
  context.setSideInputDataSet(input.getTagInternal().getId(), inputGathered);
}

Example 5

Source File: CreatePCollectionViewTranslation.java From beam with Apache License 2.0

5 votes

@Override
public FunctionSpec translate(
    AppliedPTransform<?, ?, View.CreatePCollectionView<?, ?>> transform,
    SdkComponents components) {
  return FunctionSpec.newBuilder()
      .setUrn(getUrn(transform.getTransform()))
      .setPayload(
          ByteString.copyFrom(
              SerializableUtils.serializeToByteArray(transform.getTransform().getView())))
      .build();
}

Example 6

Source File: TransformTranslator.java From beam with Apache License 2.0

5 votes

private static <ReadT, WriteT>
    TransformEvaluator<View.CreatePCollectionView<ReadT, WriteT>> createPCollView() {
  return new TransformEvaluator<View.CreatePCollectionView<ReadT, WriteT>>() {
    @Override
    public void evaluate(
        View.CreatePCollectionView<ReadT, WriteT> transform, EvaluationContext context) {
      Iterable<? extends WindowedValue<?>> iter =
          context.getWindowedValues(context.getInput(transform));
      PCollectionView<WriteT> output = transform.getView();
      Coder<Iterable<WindowedValue<?>>> coderInternal =
          (Coder)
              IterableCoder.of(
                  WindowedValue.getFullCoder(
                      output.getCoderInternal(),
                      output.getWindowingStrategyInternal().getWindowFn().windowCoder()));

      @SuppressWarnings("unchecked")
      Iterable<WindowedValue<?>> iterCast = (Iterable<WindowedValue<?>>) iter;

      context.putPView(output, iterCast, coderInternal);
    }

    @Override
    public String toNativeString() {
      return "<createPCollectionView>";
    }
  };
}

Example 7

Source File: SamzaPublishViewTransformOverride.java From beam with Apache License 2.0

5 votes

@Override
public PTransformReplacement<PCollection<ElemT>, PCollection<ElemT>> getReplacementTransform(
    AppliedPTransform<
            PCollection<ElemT>, PCollection<ElemT>, View.CreatePCollectionView<ElemT, ViewT>>
        transform) {

  @SuppressWarnings("unchecked")
  PCollection<ElemT> input =
      (PCollection<ElemT>) Iterables.getOnlyElement(transform.getInputs().values());

  return PTransformReplacement.of(
      input, new SamzaCreatePCollectionViewTransform<>(transform.getTransform().getView()));
}

Example 8

Source File: CreatePCollectionViewTranslation.java From beam with Apache License 2.0

4 votes

@Override
public String getUrn(View.CreatePCollectionView<?, ?> transform) {
  return PTransformTranslation.CREATE_VIEW_TRANSFORM_URN;
}

Example 9

Source File: DataflowPipelineTranslator.java From beam with Apache License 2.0

4 votes

@Override
public void translate(View.CreatePCollectionView transform, TranslationContext context) {
  translateTyped(transform, context);
}