Java Code Examples for org.apache.beam.sdk.Pipeline#PipelineExecutionException

private static RuntimeException beamExceptionFrom(final Throwable e) {
  // Scala doesn't declare checked exceptions in the bytecode, and the Java compiler
  // won't let you catch something that is not declared, so we can't catch
  // SparkException directly, instead we do an instanceof check.

  if (e instanceof SparkException) {
    if (e.getCause() != null && e.getCause() instanceof UserCodeException) {
      UserCodeException userException = (UserCodeException) e.getCause();
      return new Pipeline.PipelineExecutionException(userException.getCause());
    } else if (e.getCause() != null) {
      return new Pipeline.PipelineExecutionException(e.getCause());
    }
  }

  return runtimeExceptionFrom(e);
}
 

private static RuntimeException beamExceptionFrom(final Throwable e) {
  // Scala doesn't declare checked exceptions in the bytecode, and the Java compiler
  // won't let you catch something that is not declared, so we can't catch
  // SparkException directly, instead we do an instanceof check.

  if (e instanceof SparkException) {
    if (e.getCause() != null && e.getCause() instanceof UserCodeException) {
      UserCodeException userException = (UserCodeException) e.getCause();
      return new Pipeline.PipelineExecutionException(userException.getCause());
    } else if (e.getCause() != null) {
      return new Pipeline.PipelineExecutionException(e.getCause());
    }
  }

  return runtimeExceptionFrom(e);
}
 

@Test
public void testDisabledReIterationThrowsAnException() {
  // If output during closing is not supported, we can not chain DoFns and results
  // are therefore materialized during output serialization.
  Assume.assumeTrue(FlinkCapabilities.supportsOutputDuringClosing());
  final Pipeline p = FlinkTestPipeline.createForBatch();
  p.apply(Create.of(Arrays.asList(KV.of("a", 1), KV.of("b", 2), KV.of("c", 3))))
      .apply(GroupByKey.create())
      .apply(ParDo.of(new ReiterateDoFn<>()));
  Pipeline.PipelineExecutionException resultException = null;
  try {
    p.run().waitUntilFinish();
  } catch (Pipeline.PipelineExecutionException exception) {
    resultException = exception;
  }
  Assert.assertEquals(
      IllegalStateException.class, Objects.requireNonNull(resultException).getCause().getClass());
  Assert.assertTrue(
      resultException.getCause().getMessage().contains("GBK result is not re-iterable."));
}
 

@Test
@SuppressWarnings("MissingFail")
public void shouldThrowIfThrowErrorOptionSetOnInternalException() {
  String testMessage1 = "test1";

  pipeline
      .apply(Create.of(testMessage1))
      .apply(
          SnsIO.<String>writeAsync()
              .withCoder(StringUtf8Coder.of())
              .withPublishRequestFn(createPublishRequestFn())
              .withSnsClientProvider(MockSnsAsyncExceptionClient::create));
  try {
    pipeline.run().waitUntilFinish();
  } catch (final Pipeline.PipelineExecutionException e) {
    assertThrows(IOException.class, () -> e.getCause().getClass());
  }
}
 

@Test
@SuppressWarnings("MissingFail")
public void shouldThrowIfThrowErrorOptionSet() {
  String testMessage1 = "test1";

  pipeline
      .apply(Create.of(testMessage1))
      .apply(
          SnsIO.<String>writeAsync()
              .withCoder(StringUtf8Coder.of())
              .withPublishRequestFn(createPublishRequestFn())
              .withSnsClientProvider(
                  () -> MockSnsAsyncClient.withStatusCode(FAILURE_STATUS_CODE)));
  try {
    pipeline.run().waitUntilFinish();
  } catch (final Pipeline.PipelineExecutionException e) {
    assertThrows(IOException.class, () -> e.getCause().getClass());
  }
}
 

@Test
public void testRetries() throws Throwable {
  thrown.expectMessage("Error writing to SNS");
  final PublishRequest request1 = createSampleMessage("my message that will not be published");
  final TupleTag<PublishResult> results = new TupleTag<>();
  final AmazonSNS amazonSnsErrors = getAmazonSnsMockErrors();
  p.apply(Create.of(request1))
      .apply(
          SnsIO.write()
              .withTopicName(topicName)
              .withRetryConfiguration(
                  SnsIO.RetryConfiguration.create(4, org.joda.time.Duration.standardSeconds(10)))
              .withAWSClientsProvider(new Provider(amazonSnsErrors))
              .withResultOutputTag(results));

  try {
    p.run();
  } catch (final Pipeline.PipelineExecutionException e) {
    // check 3 retries were initiated by inspecting the log before passing on the exception
    expectedLogs.verifyWarn(String.format(SnsIO.Write.SnsWriterFn.RETRY_ATTEMPT_LOG, 1));
    expectedLogs.verifyWarn(String.format(SnsIO.Write.SnsWriterFn.RETRY_ATTEMPT_LOG, 2));
    expectedLogs.verifyWarn(String.format(SnsIO.Write.SnsWriterFn.RETRY_ATTEMPT_LOG, 3));
    throw e.getCause();
  }
  fail("Pipeline is expected to fail because we were unable to write to SNS.");
}
 

/**
 * Return a converter that can any datum to an {@link IndexedRecord} representation as if it were passed in the
 * transformation. This might be an expensive call, so the converter should be cached if possible.
 *
 * @param datum the datum to convert.
 * @return a converter that can turn it into an Avro {@link IndexedRecord}.
 */
public static <DatumT> IndexedRecordConverter<DatumT, IndexedRecord> getConverter(DatumT datum) {
    @SuppressWarnings("unchecked")
    IndexedRecordConverter<DatumT, IndexedRecord> converter = (IndexedRecordConverter<DatumT, IndexedRecord>) new AvroRegistry()
            .createIndexedRecordConverter(datum.getClass());
    if (datum instanceof IndexedRecord) {
        converter.setSchema(((IndexedRecord) datum).getSchema());
    }
    if (converter == null) {
        throw new Pipeline.PipelineExecutionException(
                new RuntimeException("Cannot convert " + datum.getClass() + " to IndexedRecord."));
    }
    return converter;
}
 

@Override
public void getSample(int limit, Consumer<IndexedRecord> consumer) {
    // Create an input runtime based on the properties.
    SimpleFileIOInputRuntime inputRuntime = new SimpleFileIOInputRuntime();
    SimpleFileIOInputProperties inputProperties = new SimpleFileIOInputProperties(null);
    inputProperties.limit.setValue(limit);
    inputProperties.init();
    inputProperties.setDatasetProperties(properties);
    inputRuntime.initialize(null, inputProperties);
    // Create a pipeline using the input component to get records.

    DirectOptions options = BeamLocalRunnerOption.getOptions();
    final Pipeline p = Pipeline.create(options);

    try (DirectConsumerCollector<IndexedRecord> collector = DirectConsumerCollector.of(consumer)) {
        // Collect a sample of the input records.
        p.apply(inputRuntime) //
                .apply(collector);
        try {
            p.run().waitUntilFinish();
        } catch (Pipeline.PipelineExecutionException e) {
            if (e.getCause() instanceof TalendRuntimeException)
                throw (TalendRuntimeException) e.getCause();
            throw e;
        }
    }
}
 

public static PipelineResult waitUntilDone(
    final PipelineResult result, final Duration exportTimeout) {
  // terminal state might be null, such as:
  // {{ @link org.apache.beam.runners.dataflow.DataflowPipelineJob.waitUntilFinish }}
  @Nullable
  final PipelineResult.State terminalState =
      result.waitUntilFinish(org.joda.time.Duration.millis(exportTimeout.toMillis()));
  if (terminalState == null || !terminalState.isTerminal()) {
    try {
      result.cancel();
    } catch (IOException e) {
      throw new Pipeline.PipelineExecutionException(
          new Exception(
              String.format(
                  "Job exceeded timeout of %s, but was not possible to cancel, "
                      + "finished with terminalState %s",
                  exportTimeout.toString(), terminalState),
              e));
    }
    throw new Pipeline.PipelineExecutionException(
        new Exception("Job cancelled after exceeding timeout " + exportTimeout.toString()));
  }
  if (!terminalState.equals(PipelineResult.State.DONE)) {
    throw new Pipeline.PipelineExecutionException(
        new Exception("Job finished with terminalState " + terminalState.toString()));
  }
  return result;
}
 

private boolean safeRun() {
  try {
    return pipeline.run().waitUntilFinish() == PipelineResult.State.DONE;
  } catch (Pipeline.PipelineExecutionException e) {
    return false;
  }
}
 

@Test(expected = Pipeline.PipelineExecutionException.class)
public void failWritingWithoutCustomMapper() {
  pipeline
      .apply(Create.of(EMPTY_BEANS))
      .apply(AsJsons.of(MyEmptyBean.class))
      .setCoder(StringUtf8Coder.of());

  pipeline.run();
}
 

@Test(expected = Pipeline.PipelineExecutionException.class)
public void failParsingWithoutCustomMapper() {
  PCollection<MyPojo> output =
      pipeline
          .apply(Create.of(EXTRA_PROPERTIES_JSONS))
          .apply(ParseJsons.of(MyPojo.class))
          .setCoder(SerializableCoder.of(MyPojo.class));

  PAssert.that(output).empty();

  pipeline.run();
}
 

@Test(expected = Pipeline.PipelineExecutionException.class)
public void failParsingInvalidJsons() {
  PCollection<MyPojo> output =
      pipeline
          .apply(Create.of(Iterables.concat(VALID_JSONS, INVALID_JSONS)))
          .apply(ParseJsons.of(MyPojo.class))
          .setCoder(SerializableCoder.of(MyPojo.class));

  PAssert.that(output).containsInAnyOrder(POJOS);

  pipeline.run();
}
 

/**
 * Converts any datum to an {@link IndexedRecord} representation as if it were passed in the transformation. This
 * might be an expensive call, so it should only be used for sampling data (not in a processing-intensive loop).
 *
 * @param datum the datum to convert.
 * @return its representation as an Avro {@link IndexedRecord}.
 */
public static <DatumT> IndexedRecord convertToAvro(DatumT datum) {
    IndexedRecordConverter<DatumT, IndexedRecord> c = getConverter(datum);
    if (c == null) {
        throw new Pipeline.PipelineExecutionException(
                new RuntimeException("Cannot convert " + datum.getClass() + " to IndexedRecord."));
    }
    return (IndexedRecord) c.convertToAvro(datum);
}
 

@Test
public void shouldThrownExceptionInCaseFailedPipelineResult() {
  PipelineResult mockResult =
      new PipelineResult() {
        @Override
        public State getState() {
          return null;
        }

        @Override
        public State cancel() throws IOException {
          return null;
        }

        @Override
        public State waitUntilFinish(org.joda.time.Duration duration) {
          return PipelineResult.State.FAILED;
        }

        @Override
        public State waitUntilFinish() {
          return null;
        }

        @Override
        public MetricResults metrics() {
          return null;
        }
      };
  try {
    BeamHelper.waitUntilDone(mockResult, Duration.ofMinutes(1));
    Assert.fail("A PipelineExecutionException should be thrown");
  } catch (Pipeline.PipelineExecutionException exception) {
    Assert.assertEquals(
        "java.lang.Exception: Job finished with terminalState FAILED", exception.getMessage());
  }
}
 

/**
 * Basic unit test using all default values (except for the path) on an in-memory DFS cluster.
 */
@Test
public void testTryToOverwrite() throws IOException, URISyntaxException {
    Path parent = new Path(mini.newFolder().toString());
    Path dst = new Path(parent, "output");
    String fileSpec = mini.getLocalFs().getUri().resolve(dst.toUri()).toString();

    // Write something to the file before trying to run.
    try (OutputStream out = mini.getLocalFs().create(new Path(dst, "part-00000"))) {
        out.write(0);
    }

    // Trying to write to an existing destination throws an exception.
    thrown.expect(TalendRuntimeException.class);
    thrown.expect(hasProperty("code", is(SimpleFileIOErrorCode.OUTPUT_ALREADY_EXISTS)));
    thrown.expectMessage("The path " + fileSpec + " already exists. Please remove it manually.");

    // Now try using the component.
    try {
        // Configure the component.
        SimpleFileIOOutputProperties props = SimpleFileIOOutputRuntimeTest.createOutputComponentProperties();
        props.getDatasetProperties().path.setValue(fileSpec);

        // Create the runtime.
        SimpleFileIOOutputRuntime runtime = new SimpleFileIOOutputRuntime();
        runtime.initialize(null, props);

        // Use the runtime in a direct pipeline to test.
        final Pipeline p = beam.createPipeline();
        PCollection<IndexedRecord> input = p.apply( //
                Create.of(ConvertToIndexedRecord.convertToAvro(new String[] { "1", "one" }), //
                        ConvertToIndexedRecord.convertToAvro(new String[] { "2", "two" }))); //
        input.apply(runtime);

        // And run the test.
        p.run().waitUntilFinish();
    } catch (Pipeline.PipelineExecutionException e) {
        if (e.getCause() instanceof TalendRuntimeException)
            throw (TalendRuntimeException) e.getCause();
        throw e;
    }
}
 

/**
 * Basic unit test using all default values (except for the path) on an in-memory DFS cluster.
 */
@Test
public void testTryToOverwrite() throws IOException, URISyntaxException {
    Path parent = new Path(mini.newFolder().toString());
    Path dst = new Path(parent, "output");
    String fileSpec = mini.getLocalFs().getUri().resolve(dst.toUri()).toString();

    // Write something to the file before trying to run.
    try (OutputStream out = mini.getLocalFs().create(new Path(dst, "part-00000"))) {
        out.write(0);
    }

    // Now try using the component.
    try {
        // Configure the component.
        SimpleFileIOOutputProperties props = SimpleFileIOOutputRuntimeTest.createOutputComponentProperties();
        props.getDatasetProperties().path.setValue(fileSpec);
        props.overwrite.setValue(true);

        // Create the runtime.
        SimpleFileIOOutputRuntime runtime = new SimpleFileIOOutputRuntime();
        runtime.initialize(null, props);

        // Use the runtime in a direct pipeline to test.
        final Pipeline p = beam.createPipeline();
        PCollection<IndexedRecord> input = p.apply( //
                Create.of(ConvertToIndexedRecord.convertToAvro(new String[] { "1", "one" }), //
                        ConvertToIndexedRecord.convertToAvro(new String[] { "2", "two" }))); //
        input.apply(runtime);

        // And run the test.
        runtime.runAtDriver(null);
        p.run().waitUntilFinish();
    } catch (Pipeline.PipelineExecutionException e) {
        if (e.getCause() instanceof TalendRuntimeException)
            throw (TalendRuntimeException) e.getCause();
        throw e;
    }

    // Check the expected values, which should be overwritten.
    mini.assertReadFile(mini.getLocalFs(), fileSpec, "1;one", "2;two");
}
 

private StateInfo(State state, Pipeline.PipelineExecutionException error) {
  this.state = state;
  this.error = error;
}
 

@Test(expected = Pipeline.PipelineExecutionException.class)
public void testQueueDeclareWithoutQueueNameFails() throws Exception {
  RabbitMqIO.Read read = RabbitMqIO.read().withQueueDeclare(true);
  doExchangeTest(new ExchangeTestPlan(read, 1));
}
 

/**
 * Basic unit test using all default values (except for the path) on an in-memory DFS cluster.
 */
@Test
public void testUnauthorizedOverwrite() throws IOException, URISyntaxException {
    Path parent = new Path(mini.newFolder().toString());
    Path dst = new Path(parent, "output");
    String fileSpec = mini.getLocalFs().getUri().resolve(dst.toUri()).toString();

    // Write something to the file before trying to run.
    try (OutputStream out = mini.getLocalFs().create(new Path(dst, "part-00000"))) {
        out.write(0);
    }

    // Ensure that the destination is unwritable.
    FileUtil.chmod(dst.toUri().toString(), "000", true);

    // Trying to overwrite an unmodifiable destination throws an exception.
    thrown.expect(TalendRuntimeException.class);
    thrown.expect(hasProperty("code", is(SimpleFileIOErrorCode.OUTPUT_NOT_AUTHORIZED)));
    thrown.expectMessage("Can not write to " + fileSpec
            + ". Please check user permissions or existence of base directory.");

    // Now try using the component.
    try {
        // Configure the component.
        SimpleFileIOOutputProperties props = SimpleFileIOOutputRuntimeTest.createOutputComponentProperties();
        props.getDatasetProperties().path.setValue(fileSpec);
        props.overwrite.setValue(true);

        // Create the runtime.
        SimpleFileIOOutputRuntime runtime = new SimpleFileIOOutputRuntime();
        runtime.initialize(null, props);

        // Use the runtime in a direct pipeline to test.
        final Pipeline p = beam.createPipeline();
        PCollection<IndexedRecord> input = p.apply( //
                Create.of(ConvertToIndexedRecord.convertToAvro(new String[] { "1", "one" }), //
                        ConvertToIndexedRecord.convertToAvro(new String[] { "2", "two" }))); //
        input.apply(runtime);

        // And run the test.
        runtime.runAtDriver(null);
        p.run().waitUntilFinish();
    } catch (Pipeline.PipelineExecutionException e) {
        if (e.getCause() instanceof TalendRuntimeException)
            throw (TalendRuntimeException) e.getCause();
        throw e;
    }
}