org.apache.beam.sdk.runners.AppliedPTransform Java Examples
The following examples show how to use
org.apache.beam.sdk.runners.AppliedPTransform.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: PTransformMatchers.java From beam with Apache License 2.0 | 6 votes |
/** * A {@link PTransformMatcher} that matches a {@link ParDo.SingleOutput} containing a {@link DoFn} * that is splittable, as signified by {@link ProcessElementMethod#isSplittable()}. */ public static PTransformMatcher splittableParDoSingle() { return new PTransformMatcher() { @Override public boolean matches(AppliedPTransform<?, ?, ?> application) { PTransform<?, ?> transform = application.getTransform(); if (transform instanceof ParDo.SingleOutput) { DoFn<?, ?> fn = ((ParDo.SingleOutput<?, ?>) transform).getFn(); DoFnSignature signature = DoFnSignatures.signatureForDoFn(fn); return signature.processElement().isSplittable(); } return false; } @Override public String toString() { return MoreObjects.toStringHelper("SplittableParDoSingleMatcher").toString(); } }; }
Example #2
Source File: CreatePCollectionViewTranslationTest.java From beam with Apache License 2.0 | 6 votes |
@Test public void testExtractionDirectFromTransform() throws Exception { SdkComponents components = SdkComponents.create(); components.registerEnvironment(Environments.createDockerEnvironment("java")); components.registerPCollection(testPCollection); AppliedPTransform<?, ?, ?> appliedPTransform = AppliedPTransform.of( "foo", testPCollection.expand(), createViewTransform.getView().expand(), createViewTransform, p); CreatePCollectionViewTranslation.getView((AppliedPTransform) appliedPTransform); FunctionSpec payload = PTransformTranslation.toProto(appliedPTransform, components).getSpec(); // Checks that the payload is what it should be PCollectionView<?> deserializedView = (PCollectionView<?>) SerializableUtils.deserializeFromByteArray( payload.getPayload().toByteArray(), PCollectionView.class.getSimpleName()); assertThat(deserializedView, Matchers.equalTo(createViewTransform.getView())); }
Example #3
Source File: DirectTransformExecutor.java From beam with Apache License 2.0 | 6 votes |
@VisibleForTesting DirectTransformExecutor( EvaluationContext context, TransformEvaluatorRegistry factory, Iterable<? extends ModelEnforcementFactory> modelEnforcements, CommittedBundle<T> inputBundle, AppliedPTransform<?, ?, ?> transform, CompletionCallback completionCallback, TransformExecutorService transformEvaluationState) { this.evaluatorRegistry = factory; this.modelEnforcements = modelEnforcements; this.inputBundle = inputBundle; this.transform = transform; this.onComplete = completionCallback; this.transformEvaluationState = transformEvaluationState; this.context = context; }
Example #4
Source File: WriteFilesTranslationTest.java From beam with Apache License 2.0 | 6 votes |
@Test public void testExtractionDirectFromTransform() throws Exception { PCollection<String> input = p.apply(Create.of("hello")); WriteFilesResult<Void> output = input.apply(writeFiles); AppliedPTransform<PCollection<String>, WriteFilesResult<Void>, WriteFiles<String, Void, String>> appliedPTransform = AppliedPTransform.of("foo", input.expand(), output.expand(), writeFiles, p); assertThat( WriteFilesTranslation.isRunnerDeterminedSharding(appliedPTransform), equalTo( writeFiles.getNumShardsProvider() == null && writeFiles.getComputeNumShards() == null)); assertThat( WriteFilesTranslation.isWindowedWrites(appliedPTransform), equalTo(writeFiles.getWindowedWrites())); assertThat( WriteFilesTranslation.<String, Void, String>getSink(appliedPTransform), equalTo(writeFiles.getSink())); }
Example #5
Source File: PTransformTranslationTest.java From beam with Apache License 2.0 | 6 votes |
private static AppliedPTransform<?, ?, ?> multiMultiParDo(Pipeline pipeline) { PCollectionView<String> view = pipeline.apply(Create.of("foo")).apply(View.asSingleton()); PCollection<Long> input = pipeline.apply(GenerateSequence.from(0)); ParDo.MultiOutput<Long, KV<Long, String>> parDo = ParDo.of(new TestDoFn()) .withSideInputs(view) .withOutputTags( new TupleTag<KV<Long, String>>() {}, TupleTagList.of(new TupleTag<KV<String, Long>>() {})); PCollectionTuple output = input.apply(parDo); Map<TupleTag<?>, PValue> inputs = new HashMap<>(); inputs.putAll(parDo.getAdditionalInputs()); inputs.putAll(input.expand()); return AppliedPTransform .<PCollection<Long>, PCollectionTuple, ParDo.MultiOutput<Long, KV<Long, String>>>of( "MultiParDoInAndOut", inputs, output.expand(), parDo, pipeline); }
Example #6
Source File: ParDoTranslation.java From beam with Apache License 2.0 | 6 votes |
@Override public RunnerApi.PTransform translate( AppliedPTransform<?, ?, ?> appliedPTransform, List<AppliedPTransform<?, ?, ?>> subtransforms, SdkComponents components) throws IOException { RunnerApi.PTransform.Builder builder = PTransformTranslation.translateAppliedPTransform( appliedPTransform, subtransforms, components); AppliedPTransform<?, ?, ParDo.MultiOutput<?, ?>> appliedParDo = (AppliedPTransform<?, ?, ParDo.MultiOutput<?, ?>>) appliedPTransform; ParDoPayload payload = translateParDo(appliedParDo, components); builder.setSpec( RunnerApi.FunctionSpec.newBuilder() .setUrn(PAR_DO_TRANSFORM_URN) .setPayload(payload.toByteString()) .build()); builder.setEnvironmentId(components.getOnlyEnvironmentId()); return builder.build(); }
Example #7
Source File: ViewOverrideFactory.java From beam with Apache License 2.0 | 6 votes |
@Override public PTransformReplacement<PCollection<ElemT>, PCollection<ElemT>> getReplacementTransform( AppliedPTransform< PCollection<ElemT>, PCollection<ElemT>, PTransform<PCollection<ElemT>, PCollection<ElemT>>> transform) { PCollectionView<ViewT> view; try { view = CreatePCollectionViewTranslation.getView(transform); } catch (IOException exc) { throw new RuntimeException( String.format( "Could not extract %s from transform %s", PCollectionView.class.getSimpleName(), transform), exc); } return PTransformReplacement.of( PTransformReplacements.getSingletonMainInput(transform), new GroupAndWriteView<>(view)); }
Example #8
Source File: DataflowPTransformMatchersTest.java From beam with Apache License 2.0 | 6 votes |
/** * Test significant cases that the matcher should not match against. In this case, this tests that * any {@link Combine.GroupedValues} with side inputs should not match, and that a {@link * Combine.GroupedValues} without an encompassing {@link Combine.PerKey} will not match. */ @Test public void combineValuesWithParentCheckSkipsNonmatching() { PTransformMatcher matcher = new DataflowPTransformMatchers.CombineValuesWithParentCheckPTransformMatcher(); AppliedPTransform<?, ?, ?> groupedValues; groupedValues = getCombineGroupedValuesFrom(createCombineGroupedValuesPipeline()); assertThat(matcher.matches(groupedValues), is(false)); groupedValues = getCombineGroupedValuesFrom(createCombineGroupedValuesWithSideInputsPipeline()); assertThat(matcher.matches(groupedValues), is(false)); groupedValues = getCombineGroupedValuesFrom(createCombinePerKeyWithSideInputsPipeline()); assertThat(matcher.matches(groupedValues), is(false)); }
Example #9
Source File: CombineTranslation.java From beam with Apache License 2.0 | 6 votes |
private static <K, InputT, AccumT> Coder<AccumT> extractAccumulatorCoder( GlobalCombineFn<InputT, AccumT, ?> combineFn, AppliedPTransform<PCollection<KV<K, InputT>>, ?, Combine.PerKey<K, InputT, ?>> transform) throws IOException { try { @SuppressWarnings("unchecked") PCollection<KV<K, InputT>> mainInput = (PCollection<KV<K, InputT>>) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(transform)); return combineFn.getAccumulatorCoder( transform.getPipeline().getCoderRegistry(), ((KvCoder<K, InputT>) mainInput.getCoder()).getValueCoder()); } catch (CannotProvideCoderException e) { throw new IOException("Could not obtain a Coder for the accumulator", e); } }
Example #10
Source File: TestStreamEvaluatorFactory.java From beam with Apache License 2.0 | 6 votes |
@Override public Collection<CommittedBundle<TestStreamIndex<T>>> getInitialInputs( AppliedPTransform<PBegin, PCollection<T>, PTransform<PBegin, PCollection<T>>> transform, int targetParallelism) { // This will always be run on an execution-time transform, so it can be downcast DirectTestStreamFactory.DirectTestStream<T> testStream = (DirectTestStreamFactory.DirectTestStream<T>) transform.getTransform(); CommittedBundle<TestStreamIndex<T>> initialBundle = evaluationContext .<TestStreamIndex<T>>createRootBundle() .add(WindowedValue.valueInGlobalWindow(TestStreamIndex.of(testStream.original))) .commit(BoundedWindow.TIMESTAMP_MAX_VALUE); return Collections.singleton(initialBundle); }
Example #11
Source File: DirectGraphVisitorTest.java From beam with Apache License 2.0 | 6 votes |
@Test public void getValueToConsumersWithDuplicateInputSucceeds() { PCollection<String> created = p.apply(Create.of("1", "2", "3")); PCollection<String> flattened = PCollectionList.of(created).and(created).apply(Flatten.pCollections()); p.traverseTopologically(visitor); DirectGraph graph = visitor.getGraph(); AppliedPTransform<?, ?, ?> flattenedProducer = graph.getProducer(flattened); assertThat( graph.getPerElementConsumers(created), Matchers.containsInAnyOrder(new Object[] {flattenedProducer, flattenedProducer})); assertThat(graph.getPerElementConsumers(flattened), emptyIterable()); }
Example #12
Source File: Utils.java From beam with Apache License 2.0 | 6 votes |
static DoFn<?, ?> getDoFn(AppliedPTransform<?, ?, ?> appliedTransform) { try { DoFn<?, ?> doFn = ParDoTranslation.getDoFn(appliedTransform); if (DoFnSignatures.isSplittable(doFn)) { throw new IllegalStateException( "Not expected to directly translate splittable DoFn, should have been overridden: " + doFn); // todo } if (DoFnSignatures.requiresTimeSortedInput(doFn)) { throw new UnsupportedOperationException( String.format( "%s doesn't current support @RequiresTimeSortedInput annotation.", JetRunner.class.getSimpleName())); } return doFn; } catch (IOException e) { throw new RuntimeException(e); } }
Example #13
Source File: SdkComponentsTest.java From beam with Apache License 2.0 | 6 votes |
/** Tests that trying to register a transform which has unregistered children throws. */ @Test public void registerTransformWithUnregisteredChildren() throws IOException { Create.Values<Long> create = Create.of(1L, 2L, 3L); GenerateSequence createChild = GenerateSequence.from(0); PCollection<Long> pt = pipeline.apply(create); String userName = "my_transform"; String childUserName = "my_transform/my_nesting"; AppliedPTransform<?, ?, ?> transform = AppliedPTransform.of(userName, pipeline.begin().expand(), pt.expand(), create, pipeline); AppliedPTransform<?, ?, ?> childTransform = AppliedPTransform.of( childUserName, pipeline.begin().expand(), pt.expand(), createChild, pipeline); thrown.expect(IllegalArgumentException.class); thrown.expectMessage(childTransform.toString()); components.registerPTransform(transform, Collections.singletonList(childTransform)); }
Example #14
Source File: DataflowRunnerTest.java From beam with Apache License 2.0 | 6 votes |
private void testStreamingWriteOverride(PipelineOptions options, int expectedNumShards) { TestPipeline p = TestPipeline.fromOptions(options); StreamingShardedWriteFactory<Object, Void, Object> factory = new StreamingShardedWriteFactory<>(p.getOptions()); WriteFiles<Object, Void, Object> original = WriteFiles.to(new TestSink(tmpFolder.toString())); PCollection<Object> objs = (PCollection) p.apply(Create.empty(VoidCoder.of())); AppliedPTransform<PCollection<Object>, WriteFilesResult<Void>, WriteFiles<Object, Void, Object>> originalApplication = AppliedPTransform.of("writefiles", objs.expand(), Collections.emptyMap(), original, p); WriteFiles<Object, Void, Object> replacement = (WriteFiles<Object, Void, Object>) factory.getReplacementTransform(originalApplication).getTransform(); assertThat(replacement, not(equalTo((Object) original))); assertThat(replacement.getNumShardsProvider().get(), equalTo(expectedNumShards)); WriteFilesResult<Void> originalResult = objs.apply(original); WriteFilesResult<Void> replacementResult = objs.apply(replacement); Map<PValue, ReplacementOutput> res = factory.mapOutputs(originalResult.expand(), replacementResult); assertEquals(1, res.size()); assertEquals( originalResult.getPerDestinationOutputFilenames(), res.get(replacementResult.getPerDestinationOutputFilenames()).getOriginal().getValue()); }
Example #15
Source File: BoundedReadEvaluatorFactoryTest.java From beam with Apache License 2.0 | 6 votes |
@Test public void boundedSourceEvaluatorNoElementsClosesReader() throws Exception { TestSource<Long> source = new TestSource<>(BigEndianLongCoder.of()); PCollection<Long> pcollection = p.apply(Read.from(source)); AppliedPTransform<?, ?, ?> sourceTransform = DirectGraphs.getProducer(pcollection); UncommittedBundle<Long> output = bundleFactory.createBundle(pcollection); when(context.createBundle(pcollection)).thenReturn(output); TransformEvaluator<BoundedSourceShard<Long>> evaluator = factory.forApplication( sourceTransform, bundleFactory.createRootBundle().commit(Instant.now())); evaluator.processElement(WindowedValue.valueInGlobalWindow(BoundedSourceShard.of(source))); evaluator.finishBundle(); CommittedBundle<Long> committed = output.commit(Instant.now()); assertThat(committed.getElements(), emptyIterable()); assertThat(TestSource.readerClosed, is(true)); }
Example #16
Source File: ParDoEvaluatorFactory.java From beam with Apache License 2.0 | 6 votes |
@Override public <T> TransformEvaluator<T> forApplication( AppliedPTransform<?, ?, ?> application, CommittedBundle<?> inputBundle) throws Exception { @SuppressWarnings({"unchecked", "rawtypes"}) TransformEvaluator<T> evaluator = (TransformEvaluator<T>) createEvaluator( (AppliedPTransform) application, (PCollection<InputT>) inputBundle.getPCollection(), inputBundle.getKey(), ParDoTranslation.getSideInputs(application), (TupleTag<OutputT>) ParDoTranslation.getMainOutputTag(application), ParDoTranslation.getAdditionalOutputTags(application).getAll(), ParDoTranslation.getSchemaInformation(application), ParDoTranslation.getSideInputMapping(application)); return evaluator; }
Example #17
Source File: BoundedReadEvaluatorFactoryTest.java From beam with Apache License 2.0 | 6 votes |
@Test public void boundedSourceEvaluatorClosesReader() throws Exception { TestSource<Long> source = new TestSource<>(BigEndianLongCoder.of(), 1L, 2L, 3L); PCollection<Long> pcollection = p.apply(Read.from(source)); AppliedPTransform<?, ?, ?> sourceTransform = DirectGraphs.getProducer(pcollection); UncommittedBundle<Long> output = bundleFactory.createBundle(pcollection); when(context.createBundle(pcollection)).thenReturn(output); TransformEvaluator<BoundedSourceShard<Long>> evaluator = factory.forApplication( sourceTransform, bundleFactory.createRootBundle().commit(Instant.now())); evaluator.processElement(WindowedValue.valueInGlobalWindow(BoundedSourceShard.of(source))); evaluator.finishBundle(); CommittedBundle<Long> committed = output.commit(Instant.now()); assertThat(committed.getElements(), containsInAnyOrder(gw(2L), gw(3L), gw(1L))); assertThat(TestSource.readerClosed, is(true)); }
Example #18
Source File: PTransformMatchersTest.java From beam with Apache License 2.0 | 5 votes |
@Test public void parDoWithFnTypeWithNoMatch() { DoFn<Object, Object> fn = new DoFn<Object, Object>() { @ProcessElement public void process(ProcessContext ctxt) {} }; AppliedPTransform<?, ?, ?> parDoSingle = getAppliedTransform(ParDo.of(fn)); AppliedPTransform<?, ?, ?> parDoMulti = getAppliedTransform(ParDo.of(fn).withOutputTags(new TupleTag<>(), TupleTagList.empty())); PTransformMatcher matcher = PTransformMatchers.parDoWithFnType(doFnWithState.getClass()); assertThat(matcher.matches(parDoSingle), is(false)); assertThat(matcher.matches(parDoMulti), is(false)); }
Example #19
Source File: CombineTranslationTest.java From beam with Apache License 2.0 | 5 votes |
@Test public void testToProtoWithoutSideInputs() throws Exception { PCollection<Integer> input = pipeline.apply(Create.of(1, 2, 3)); CombineFnWithContext<Integer, int[], Integer> combineFn = new TestCombineFnWithContext(); input.apply(Combine.globally(combineFn).withoutDefaults()); final AtomicReference<AppliedPTransform<?, ?, Combine.Globally<?, ?>>> combine = new AtomicReference<>(); pipeline.traverseTopologically( new PipelineVisitor.Defaults() { @Override public void leaveCompositeTransform(Node node) { if (node.getTransform() instanceof Combine.Globally) { checkState(combine.get() == null); combine.set((AppliedPTransform) node.toAppliedPTransform(getPipeline())); } } }); checkState(combine.get() != null); assertEquals(combineFn, combine.get().getTransform().getFn()); SdkComponents sdkComponents = SdkComponents.create(); sdkComponents.registerEnvironment(Environments.createDockerEnvironment("java")); CombinePayload combineProto = CombineTranslation.CombineGloballyPayloadTranslator.payloadForCombineGlobally( (AppliedPTransform) combine.get(), sdkComponents); RunnerApi.Components componentsProto = sdkComponents.toComponents(); assertEquals( combineFn.getAccumulatorCoder(pipeline.getCoderRegistry(), input.getCoder()), getAccumulatorCoder(combineProto, RehydratedComponents.forComponents(componentsProto))); assertEquals( combineFn, SerializableUtils.deserializeFromByteArray( combineProto.getCombineFn().getPayload().toByteArray(), "CombineFn")); }
Example #20
Source File: DataflowMetricsTest.java From beam with Apache License 2.0 | 5 votes |
@Test public void testIgnoreDistributionButGetCounterUpdates() throws IOException { JobMetrics jobMetrics = new JobMetrics(); DataflowClient dataflowClient = mock(DataflowClient.class); when(dataflowClient.getJobMetrics(JOB_ID)).thenReturn(jobMetrics); DataflowPipelineJob job = mock(DataflowPipelineJob.class); DataflowPipelineOptions options = mock(DataflowPipelineOptions.class); when(options.isStreaming()).thenReturn(false); when(job.getDataflowOptions()).thenReturn(options); when(job.getState()).thenReturn(State.RUNNING); job.jobId = JOB_ID; AppliedPTransform<?, ?, ?> myStep = mock(AppliedPTransform.class); when(myStep.getFullName()).thenReturn("myStepName"); job.transformStepNames = HashBiMap.create(); job.transformStepNames.put(myStep, "s2"); // The parser relies on the fact that one tentative and one committed metric update exist in // the job metrics results. jobMetrics.setMetrics( ImmutableList.of( makeCounterMetricUpdate("counterName", "counterNamespace", "s2", 1233L, false), makeCounterMetricUpdate("counterName", "counterNamespace", "s2", 1233L, true), makeCounterMetricUpdate("otherCounter[MIN]", "otherNamespace", "s2", 0L, false), makeCounterMetricUpdate("otherCounter[MIN]", "otherNamespace", "s2", 0L, true))); DataflowMetrics dataflowMetrics = new DataflowMetrics(job, dataflowClient); MetricQueryResults result = dataflowMetrics.allMetrics(); assertThat( result.getCounters(), containsInAnyOrder( attemptedMetricsResult("counterNamespace", "counterName", "myStepName", 1233L))); assertThat( result.getCounters(), containsInAnyOrder( committedMetricsResult("counterNamespace", "counterName", "myStepName", 1233L))); }
Example #21
Source File: BatchStatefulParDoOverrides.java From beam with Apache License 2.0 | 5 votes |
@Override public PTransformReplacement<PCollection<KV<K, InputT>>, PCollection<OutputT>> getReplacementTransform( AppliedPTransform< PCollection<KV<K, InputT>>, PCollection<OutputT>, SingleOutput<KV<K, InputT>, OutputT>> transform) { return PTransformReplacement.of( PTransformReplacements.getSingletonMainInput(transform), new StatefulSingleOutputParDo<>(transform.getTransform(), isFnApi)); }
Example #22
Source File: Utils.java From beam with Apache License 2.0 | 5 votes |
static boolean usesStateOrTimers(AppliedPTransform<?, ?, ?> appliedTransform) { try { return ParDoTranslation.usesStateOrTimers(appliedTransform); } catch (IOException e) { throw new RuntimeException(e); } }
Example #23
Source File: PrimitiveParDoSingleFactory.java From beam with Apache License 2.0 | 5 votes |
@Override public PTransformReplacement<PCollection<? extends InputT>, PCollection<OutputT>> getReplacementTransform( AppliedPTransform< PCollection<? extends InputT>, PCollection<OutputT>, SingleOutput<InputT, OutputT>> transform) { return PTransformReplacement.of( PTransformReplacements.getSingletonMainInput(transform), new ParDoSingle<>( transform.getTransform(), Iterables.getOnlyElement(transform.getOutputs().keySet()), PTransformReplacements.getSingletonMainOutput(transform).getCoder())); }
Example #24
Source File: TransformInputs.java From beam with Apache License 2.0 | 5 votes |
/** * Gets all inputs of the {@link AppliedPTransform} that are not returned by {@link * PTransform#getAdditionalInputs()}. */ public static Collection<PValue> nonAdditionalInputs(AppliedPTransform<?, ?, ?> application) { ImmutableList.Builder<PValue> mainInputs = ImmutableList.builder(); PTransform<?, ?> transform = application.getTransform(); for (Map.Entry<TupleTag<?>, PValue> input : application.getInputs().entrySet()) { if (!transform.getAdditionalInputs().containsKey(input.getKey())) { mainInputs.add(input.getValue()); } } checkArgument( !mainInputs.build().isEmpty() || application.getInputs().isEmpty(), "Expected at least one main input if any inputs exist"); return mainInputs.build(); }
Example #25
Source File: Utils.java From beam with Apache License 2.0 | 5 votes |
static List<PCollectionView<?>> getSideInputs(AppliedPTransform<?, ?, ?> appliedTransform) { PTransform<?, ?> transform = appliedTransform.getTransform(); if (transform instanceof ParDo.MultiOutput) { ParDo.MultiOutput multiParDo = (ParDo.MultiOutput) transform; return (List) multiParDo.getSideInputs().values().stream().collect(Collectors.toList()); } else if (transform instanceof ParDo.SingleOutput) { ParDo.SingleOutput singleParDo = (ParDo.SingleOutput) transform; return (List) singleParDo.getSideInputs().values().stream().collect(Collectors.toList()); } return Collections.emptyList(); }
Example #26
Source File: DataflowRunner.java From beam with Apache License 2.0 | 5 votes |
@Override public PTransformReplacement<PBegin, PCollection<T>> getReplacementTransform( AppliedPTransform<PBegin, PCollection<T>, Create.Values<T>> transform) { Create.Values<T> original = transform.getTransform(); PCollection<T> output = (PCollection) Iterables.getOnlyElement(transform.getOutputs().values()); return PTransformReplacement.of( transform.getPipeline().begin(), new StreamingFnApiCreate<>(original, output)); }
Example #27
Source File: CommittedResultTest.java From beam with Apache License 2.0 | 5 votes |
@Test public void getUncommittedElementsNull() { CommittedResult<AppliedPTransform<?, ?, ?>> result = CommittedResult.create( StepTransformResult.withoutHold(transform).build(), Optional.empty(), Collections.emptyList(), EnumSet.noneOf(OutputType.class)); assertThat(result.getUnprocessedInputs(), Matchers.equalTo(Optional.empty())); }
Example #28
Source File: PipelineTranslationTest.java From beam with Apache License 2.0 | 5 votes |
private static Optional<CombinePayload> getCombinePayload( AppliedPTransform<?, ?, ?> transform, SdkComponents components) throws IOException { RunnerApi.PTransform proto = PTransformTranslation.toProto(transform, Collections.emptyList(), components); // Even if the proto has no spec, calling getSpec still returns a blank spec, which we want to // avoid. It should be clear to the caller whether or not there was a spec in the transform. if (proto.hasSpec()) { return Optional.of(CombinePayload.parseFrom(proto.getSpec().getPayload())); } else { return Optional.empty(); } }
Example #29
Source File: DataflowRunner.java From beam with Apache License 2.0 | 5 votes |
@Override public PTransformReplacement<PCollection<PubsubMessage>, PDone> getReplacementTransform( AppliedPTransform<PCollection<PubsubMessage>, PDone, PubsubUnboundedSink> transform) { return PTransformReplacement.of( PTransformReplacements.getSingletonMainInput(transform), new StreamingPubsubIOWrite(runner, transform.getTransform())); }
Example #30
Source File: PTransformMatchers.java From beam with Apache License 2.0 | 5 votes |
/** * A {@link PTransformMatcher} that matches a {@link ParDo.MultiOutput} containing a {@link DoFn} * that uses state or timers, as specified by {@link DoFnSignature#usesState()} and {@link * DoFnSignature#usesTimers()}. */ public static PTransformMatcher stateOrTimerParDoMulti() { return new PTransformMatcher() { @Override public boolean matches(AppliedPTransform<?, ?, ?> application) { PTransform<?, ?> transform = application.getTransform(); if (transform instanceof ParDo.MultiOutput) { DoFn<?, ?> fn = ((ParDo.MultiOutput<?, ?>) transform).getFn(); DoFnSignature signature = DoFnSignatures.signatureForDoFn(fn); return signature.usesState() || signature.usesTimers(); } return false; } @Override public String toString() { return MoreObjects.toStringHelper("StateOrTimerParDoMultiMatcher").toString(); } }; }