com.google.cloud.bigquery.LoadJobConfiguration Java Examples

@Test
public void shouldRunLoadJobInTestMode() throws Exception {
  final TableId dstTable = TableId.of("foo", "bar", "baz");
  final String srcUri = "gs://foo/bar";

  final Task<TableId> task = Task.named("task")
      .ofType(TableId.class)
      .operator(BigQueryOperator.create())
      .process(bq -> bq.job(
          JobInfo.of(LoadJobConfiguration.of(dstTable, srcUri)))
          .success(response -> dstTable));

  try (TestScope scope = FloTesting.scope()) {

    final TableId result = FloRunner.runTask(task).future()
        .get(30, SECONDS);

    assertThat(result, is(dstTable));
  }
}
 

/** Example of loading a parquet file from GCS to a table. */
public void loadTableGcsParquet(String datasetName) throws InterruptedException {
  // [START bigquery_load_table_gcs_parquet]
  String sourceUri = "gs://cloud-samples-data/bigquery/us-states/us-states.parquet";
  TableId tableId = TableId.of(datasetName, "us_states");
  LoadJobConfiguration configuration =
      LoadJobConfiguration.builder(tableId, sourceUri)
          .setFormatOptions(FormatOptions.parquet())
          .build();
  // Load the table
  Job loadJob = bigquery.create(JobInfo.of(configuration));
  loadJob = loadJob.waitFor();
  // Check the table
  StandardTableDefinition destinationTable = bigquery.getTable(tableId).getDefinition();
  System.out.println("State: " + loadJob.getStatus().getState());
  System.out.printf("Loaded %d rows.\n", destinationTable.getNumRows());
  // [END bigquery_load_table_gcs_parquet]
}
 

@Override
protected CompletableFuture<Void> close() {
  List<String> sourceUris = sourceBlobIds.stream().map(BlobIdToString::apply)
      .collect(Collectors.toList());
  boolean loadSuccess = false;
  try {
    JobStatus status = bigQuery
        .create(JobInfo.of(LoadJobConfiguration.newBuilder(tableId, sourceUris)
            .setCreateDisposition(JobInfo.CreateDisposition.CREATE_NEVER)
            .setWriteDisposition(JobInfo.WriteDisposition.WRITE_APPEND)
            .setFormatOptions(FormatOptions.json()).setIgnoreUnknownValues(true)
            .setAutodetect(false).setMaxBadRecords(0).build()))
        .waitFor().getStatus();
    if (status.getError() != null) {
      throw new BigQueryErrors(ImmutableList.of(status.getError()));
    } else if (status.getExecutionErrors() != null
        && status.getExecutionErrors().size() > 0) {
      throw new BigQueryErrors(status.getExecutionErrors());
    }
    loadSuccess = true;
    return CompletableFuture.completedFuture(null);
  } catch (InterruptedException e) {
    throw new RuntimeException(e);
  } finally {
    if (delete == Delete.always || (delete == Delete.onSuccess && loadSuccess)) {
      try {
        storage.delete(sourceBlobIds);
      } catch (RuntimeException ignore2) {
        // don't fail batch when delete throws
      }
    }
  }
}
 

/** Example of loading a newline-delimited-json file with textual fields from GCS to a table. */
// [TARGET create(JobInfo, JobOption...)]
// [VARIABLE "my_dataset_name"]
// [VARIABLE "my_table_name"]
public Long writeRemoteFileToTable(String datasetName, String tableName)
    throws InterruptedException {
  // [START bigquery_load_table_gcs_json]
  String sourceUri = "gs://cloud-samples-data/bigquery/us-states/us-states.json";
  TableId tableId = TableId.of(datasetName, tableName);
  // Table field definition
  Field[] fields =
      new Field[] {
        Field.of("name", LegacySQLTypeName.STRING),
        Field.of("post_abbr", LegacySQLTypeName.STRING)
      };
  // Table schema definition
  Schema schema = Schema.of(fields);
  LoadJobConfiguration configuration =
      LoadJobConfiguration.builder(tableId, sourceUri)
          .setFormatOptions(FormatOptions.json())
          .setCreateDisposition(CreateDisposition.CREATE_IF_NEEDED)
          .setSchema(schema)
          .build();
  // Load the table
  Job loadJob = bigquery.create(JobInfo.of(configuration));
  loadJob = loadJob.waitFor();
  // Check the table
  System.out.println("State: " + loadJob.getStatus().getState());
  return ((StandardTableDefinition) bigquery.getTable(tableId).getDefinition()).getNumRows();
  // [END bigquery_load_table_gcs_json]
}
 

@Override
JobInfo parse(String... args) throws Exception {
  if (args.length >= 4) {
    String dataset = args[0];
    String table = args[1];
    String format = args[2];
    TableId tableId = TableId.of(dataset, table);
    LoadJobConfiguration configuration =
        LoadJobConfiguration.of(
            tableId, Arrays.asList(args).subList(3, args.length), FormatOptions.of(format));
    return JobInfo.of(configuration);
  }
  throw new IllegalArgumentException("Missing required arguments.");
}