Java Code Examples for org.apache.beam.sdk.values.TypeDescriptors#voids()

The following examples show how to use org.apache.beam.sdk.values.TypeDescriptors#voids() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: WithKeys.java From beam with Apache License 2.0

5 votes

/**
 * Returns a {@code PTransform} that takes a {@code PCollection<V>} and returns a {@code
 * PCollection<KV<K, V>>}, where each of the values in the input {@code PCollection} has been
 * paired with the given key.
 */
@SuppressWarnings("unchecked")
public static <K, V> WithKeys<K, V> of(@Nullable final K key) {
  return new WithKeys<>(
      value -> key,
      (TypeDescriptor<K>)
          (key == null ? TypeDescriptors.voids() : TypeDescriptor.of(key.getClass())));
}

Example 2

Source File: ProviderTestUtils.java From beam with Apache License 2.0

4 votes

TestOperator(@Nullable String name) {
  super(name, TypeDescriptors.voids());
}

Example 3

Source File: ProviderTestUtils.java From beam with Apache License 2.0

4 votes

SecondTestOperator(@Nullable String name) {
  super(name, TypeDescriptors.voids());
}

Example 4

Source File: DoFnSignatures.java From beam with Apache License 2.0

4 votes

/**
 * Verifies properties related to methods of splittable {@link DoFn}:
 *
 * <ul>
 *   <li>Must declare the required {@link DoFn.GetInitialRestriction} and {@link DoFn.NewTracker}
 *       methods.
 *   <li>Types of restrictions and trackers must match exactly between {@link
 *       DoFn.ProcessElement}, {@link DoFn.GetInitialRestriction}, {@link DoFn.NewTracker}, {@link
 *       DoFn.GetRestrictionCoder}, {@link DoFn.SplitRestriction}.
 * </ul>
 */
private static void verifySplittableMethods(DoFnSignature signature, ErrorReporter errors) {
  DoFnSignature.ProcessElementMethod processElement = signature.processElement();
  DoFnSignature.GetInitialRestrictionMethod getInitialRestriction =
      signature.getInitialRestriction();
  DoFnSignature.NewTrackerMethod newTracker = signature.newTracker();
  DoFnSignature.GetRestrictionCoderMethod getRestrictionCoder = signature.getRestrictionCoder();
  DoFnSignature.GetInitialWatermarkEstimatorStateMethod getInitialWatermarkEstimatorState =
      signature.getInitialWatermarkEstimatorState();
  DoFnSignature.GetWatermarkEstimatorStateCoderMethod getWatermarkEstimatorStateCoder =
      signature.getWatermarkEstimatorStateCoder();

  ErrorReporter processElementErrors =
      errors.forMethod(DoFn.ProcessElement.class, processElement.targetMethod());

  TypeDescriptor<?> restrictionT = getInitialRestriction.restrictionT();
  TypeDescriptor<?> watermarkEstimatorStateT =
      getInitialWatermarkEstimatorState == null
          ? TypeDescriptors.voids()
          : getInitialWatermarkEstimatorState.watermarkEstimatorStateT();

  if (newTracker == null) {
    ErrorReporter newTrackerErrors = errors.forMethod(DoFn.NewTracker.class, null);
    newTrackerErrors.checkArgument(
        restrictionT.isSubtypeOf(TypeDescriptor.of(HasDefaultTracker.class)),
        "Splittable, but does not define @%s method or %s does not implement %s.",
        format(DoFn.NewTracker.class),
        format(restrictionT),
        format(HasDefaultTracker.class));
  }

  processElementErrors.checkArgument(
      processElement.trackerT().getRawType().equals(RestrictionTracker.class),
      "Has tracker type %s, but the DoFn's tracker type must be of type RestrictionTracker.",
      format(processElement.trackerT()));

  if (processElement.watermarkEstimatorT() != null) {
    processElementErrors.checkArgument(
        processElement.watermarkEstimatorT().getRawType().equals(WatermarkEstimator.class)
            || processElement
                .watermarkEstimatorT()
                .getRawType()
                .equals(ManualWatermarkEstimator.class),
        "Has watermark estimator type %s, but the DoFn's watermark estimator type must be one of [WatermarkEstimator, ManualWatermarkEstimator] types.",
        format(processElement.watermarkEstimatorT()));
  }

  if (getRestrictionCoder != null) {
    ErrorReporter getInitialRestrictionErrors =
        errors.forMethod(DoFn.GetInitialRestriction.class, getInitialRestriction.targetMethod());
    getInitialRestrictionErrors.checkArgument(
        getRestrictionCoder.coderT().isSubtypeOf(coderTypeOf(restrictionT)),
        "Uses restriction type %s, but @%s method %s returns %s "
            + "which is not a subtype of %s",
        format(restrictionT),
        format(DoFn.GetRestrictionCoder.class),
        format(getRestrictionCoder.targetMethod()),
        format(getRestrictionCoder.coderT()),
        format(coderTypeOf(restrictionT)));
  }

  if (getWatermarkEstimatorStateCoder != null) {
    ErrorReporter getInitialWatermarkEstimatorStateReporter =
        errors.forMethod(
            DoFn.GetInitialWatermarkEstimatorState.class,
            getInitialWatermarkEstimatorState == null
                ? null
                : getInitialWatermarkEstimatorState.targetMethod());
    getInitialWatermarkEstimatorStateReporter.checkArgument(
        getWatermarkEstimatorStateCoder
            .coderT()
            .isSubtypeOf(coderTypeOf(watermarkEstimatorStateT)),
        "Uses watermark estimator state type %s, but @%s method %s returns %s "
            + "which is not a subtype of %s",
        format(watermarkEstimatorStateT),
        format(DoFn.GetInitialWatermarkEstimatorState.class),
        format(getWatermarkEstimatorStateCoder.targetMethod()),
        format(getWatermarkEstimatorStateCoder.coderT()),
        format(coderTypeOf(watermarkEstimatorStateT)));
  }
}