org.apache.beam.sdk.PipelineResult#State

Source File: WordCountTimeOut1Sec.java From incubator-nemo with Apache License 2.0

6 votes

/**
 * Main function for the MR BEAM program.
 *
 * @param args arguments.
 */
public static void main(final String[] args) {
  final String inputFilePath = args[0];
  final String outputFilePath = args[1];
  final PipelineOptions options = NemoPipelineOptionsFactory.create();
  options.setJobName("WordCountTimeOut1Sec");

  final Pipeline p = generateWordCountPipeline(options, inputFilePath, outputFilePath);
  final PipelineResult pr = p.run();
  final PipelineResult.State running = pr.waitUntilFinish(org.joda.time.Duration.standardSeconds(1));
  try {
    final PipelineResult.State cancelled = pr.cancel();
  } catch (final IOException e) {
    LOG.info("IOException while cancelling job");
  }
}

Source File: JobFailure.java From beam with Apache License 2.0

6 votes

private static Optional<JobFailure> lookForInvalidState(PipelineResult.State state) {
  switch (state) {
    case RUNNING:
    case UNKNOWN:
      return of(new JobFailure("Job timeout.", true));

    case CANCELLED:
    case FAILED:
    case STOPPED:
    case UPDATED:
      return of(new JobFailure(format("Invalid job state: %s.", state.toString()), false));

    default:
      return empty();
  }
}

Source File: BeamHelper.java From dbeam with Apache License 2.0

5 votes

public static PipelineResult waitUntilDone(
    final PipelineResult result, final Duration exportTimeout) {
  // terminal state might be null, such as:
  // {{ @link org.apache.beam.runners.dataflow.DataflowPipelineJob.waitUntilFinish }}
  @Nullable
  final PipelineResult.State terminalState =
      result.waitUntilFinish(org.joda.time.Duration.millis(exportTimeout.toMillis()));
  if (terminalState == null || !terminalState.isTerminal()) {
    try {
      result.cancel();
    } catch (IOException e) {
      throw new Pipeline.PipelineExecutionException(
          new Exception(
              String.format(
                  "Job exceeded timeout of %s, but was not possible to cancel, "
                      + "finished with terminalState %s",
                  exportTimeout.toString(), terminalState),
              e));
    }
    throw new Pipeline.PipelineExecutionException(
        new Exception("Job cancelled after exceeding timeout " + exportTimeout.toString()));
  }
  if (!terminalState.equals(PipelineResult.State.DONE)) {
    throw new Pipeline.PipelineExecutionException(
        new Exception("Job finished with terminalState " + terminalState.toString()));
  }
  return result;
}

Source File: NonMergingGroupByKeyTest.java From beam with Apache License 2.0

5 votes

@Test
public void testEnabledReIterationDoesNotThrowAnException() {
  final Pipeline p = FlinkTestPipeline.createForBatch();
  p.getOptions().as(FlinkPipelineOptions.class).setReIterableGroupByKeyResult(true);
  p.apply(Create.of(Arrays.asList(KV.of("a", 1), KV.of("b", 2), KV.of("c", 3))))
      .apply(GroupByKey.create())
      .apply(ParDo.of(new ReiterateDoFn<>()));
  final PipelineResult.State state = p.run().waitUntilFinish();
  Assert.assertEquals(PipelineResult.State.DONE, state);
}

Source File: MetricsPusher.java From beam with Apache License 2.0

5 votes

private void run() {
  pushMetrics();
  if (pipelineResult != null) {
    PipelineResult.State pipelineState = pipelineResult.getState();
    if (pipelineState.isTerminal()) {
      tearDown();
    }
  }
}

Source File: KafkaIOIT.java From beam with Apache License 2.0

5 votes

private void cancelIfTimeouted(PipelineResult readResult, PipelineResult.State readState)
    throws IOException {

  // TODO(lgajowy) this solution works for dataflow only - it returns null when
  //  waitUntilFinish(Duration duration) exceeds provided duration.
  if (readState == null) {
    readResult.cancel();
  }
}

Source File: AvroTableProviderTest.java From beam with Apache License 2.0

5 votes

@Test
public void testReadAndWriteAvroTable() {
  File destinationFile = new File(tempFolder.getRoot(), "person-info.avro");

  BeamSqlEnv env = BeamSqlEnv.inMemory(new AvroTableProvider());
  env.executeDdl(
      String.format(
          "CREATE EXTERNAL TABLE PersonInfo %s TYPE avro LOCATION '%s'",
          AVRO_FIELD_NAMES, destinationFile.getAbsolutePath()));

  BeamSqlRelUtils.toPCollection(
      writePipeline,
      env.parseQuery(
          "INSERT INTO PersonInfo VALUES ('Alan', 22, 'England'), ('John', 42, 'USA')"));

  writePipeline.run().waitUntilFinish();

  PCollection<Row> rows =
      BeamSqlRelUtils.toPCollection(
          readPipeline, env.parseQuery("SELECT age, country FROM PersonInfo where age > 25"));

  PAssert.that(rows)
      .containsInAnyOrder(Row.withSchema(OUTPUT_ROW_SCHEMA).addValues(42L, "USA").build());

  PipelineResult.State state = readPipeline.run().waitUntilFinish();
  assertEquals(state, State.DONE);
}

Source File: JobFailure.java From beam with Apache License 2.0

5 votes

private static Optional<JobFailure> lookForFailure(
    PipelineResult pipelineResult, List<NamedTestResult> testResults) {
  PipelineResult.State state = pipelineResult.getState();

  Optional<JobFailure> stateRelatedFailure = lookForInvalidState(state);

  if (stateRelatedFailure.isPresent()) {
    return stateRelatedFailure;
  } else {
    return lookForMetricResultFailure(testResults);
  }
}

Source File: DataStoreReadWriteIT.java From beam with Apache License 2.0

5 votes

@Test
public void testDataStoreV1SqlWriteRead_withoutKey() {
  BeamSqlEnv sqlEnv = BeamSqlEnv.inMemory(new DataStoreV1TableProvider());
  String projectId = options.getProject();

  String createTableStatement =
      "CREATE EXTERNAL TABLE TEST( \n"
          + "   `content` VARCHAR \n"
          + ") \n"
          + "TYPE 'datastoreV1' \n"
          + "LOCATION '"
          + projectId
          + "/"
          + KIND
          + "'";
  sqlEnv.executeDdl(createTableStatement);

  String insertStatement = "INSERT INTO TEST VALUES ( '3000' )";

  BeamSqlRelUtils.toPCollection(writePipeline, sqlEnv.parseQuery(insertStatement));
  writePipeline.run().waitUntilFinish();

  String selectTableStatement = "SELECT * FROM TEST";
  PCollection<Row> output =
      BeamSqlRelUtils.toPCollection(readPipeline, sqlEnv.parseQuery(selectTableStatement));

  assertThat(output.getSchema(), equalTo(SOURCE_SCHEMA_WITHOUT_KEY));

  PipelineResult.State state = readPipeline.run().waitUntilFinish(Duration.standardMinutes(5));
  assertThat(state, equalTo(State.DONE));
}

Source File: SparkStructuredStreamingPipelineResult.java From beam with Apache License 2.0

4 votes

@Override
public PipelineResult.State waitUntilFinish() {
  return waitUntilFinish(Duration.millis(Long.MAX_VALUE));
}

Source File: SparkStructuredStreamingPipelineResult.java From beam with Apache License 2.0

4 votes

@Override
public PipelineResult.State getState() {
  return state;
}

Source File: SparkPipelineResult.java From beam with Apache License 2.0

4 votes

@Override
public PipelineResult.State getState() {
  return state;
}

Source File: DataStoreReadWriteIT.java From beam with Apache License 2.0

4 votes

@Test
public void testReadAllSupportedTypes() {
  BeamSqlEnv sqlEnv = BeamSqlEnv.inMemory(new DataStoreV1TableProvider());
  String projectId = options.getProject();

  final Schema expectedSchema =
      Schema.builder()
          .addNullableField("__key__", VARBINARY)
          .addNullableField("boolean", BOOLEAN)
          .addNullableField("datetime", DATETIME)
          // TODO: flattening of nested fields by Calcite causes some issues.
          /*.addRowField("embeddedentity",
          Schema.builder()
              .addNullableField("property1", STRING)
              .addNullableField("property2", INT64)
              .build())*/
          .addNullableField("floatingnumber", DOUBLE)
          .addNullableField("integer", INT64)
          .addNullableField("primitivearray", FieldType.array(STRING))
          .addNullableField("string", STRING)
          .addNullableField("text", STRING)
          .build();

  String createTableStatement =
      "CREATE EXTERNAL TABLE TEST( \n"
          + "   `__key__` VARBINARY, \n"
          + "   `boolean` BOOLEAN, \n"
          + "   `datetime` TIMESTAMP, \n"
          // + "   `embeddedentity` ROW(`property1` VARCHAR, `property2` BIGINT), \n"
          + "   `floatingnumber` DOUBLE, \n"
          + "   `integer` BIGINT, \n"
          + "   `primitivearray` ARRAY<VARCHAR>, \n"
          + "   `string` VARCHAR, \n"
          + "   `text` VARCHAR"
          + ") \n"
          + "TYPE 'datastoreV1' \n"
          + "LOCATION '"
          + projectId
          + "/"
          + KIND_ALL_TYPES
          + "'";
  sqlEnv.executeDdl(createTableStatement);

  String selectTableStatement = "SELECT * FROM TEST";
  PCollection<Row> output =
      BeamSqlRelUtils.toPCollection(readPipeline, sqlEnv.parseQuery(selectTableStatement));

  assertThat(output.getSchema(), equalTo(expectedSchema));

  PipelineResult.State state = readPipeline.run().waitUntilFinish(Duration.standardMinutes(5));
  assertThat(state, equalTo(State.DONE));
}

Source File: SparkStructuredStreamingPipelineResult.java From beam with Apache License 2.0

4 votes

@Override
public PipelineResult.State cancel() throws IOException {
  offerNewState(PipelineResult.State.CANCELLED);
  return state;
}

Source File: SparkPipelineResult.java From beam with Apache License 2.0

4 votes

@Override
public PipelineResult.State cancel() throws IOException {
  offerNewState(PipelineResult.State.CANCELLED);
  return state;
}