com.google.api.services.bigquery.model.Table Java Examples

/** Ensures the table exists in Bigquery. */
private void ensureTable(Bigquery bigquery, TableReference table, List<TableFieldSchema> schema)
    throws IOException {
  try {
    bigquery.tables().insert(table.getProjectId(), table.getDatasetId(), new Table()
        .setSchema(new TableSchema().setFields(schema))
        .setTableReference(table))
        .execute();
    logger.atInfo().log(
        "Created BigQuery table %s:%s.%s",
        table.getProjectId(), table.getDatasetId(), table.getTableId());
  } catch (IOException e) {
    // Swallow errors about a table that exists, and throw any other ones.
    if (!BigqueryJobFailureException.create(e).getReason().equals("duplicate")) {
      throw e;
    }
  }
}
 

/** Returns a {@link TableDestination} object for the destination. May not return null. */
@Override
public TableDestination getTable(DestinationT destination) {
  TableDestination wrappedDestination = super.getTable(destination);
  Table existingTable = getBigQueryTable(wrappedDestination.getTableReference());

  if (existingTable == null) {
    return wrappedDestination;
  } else {
    return new TableDestination(
        wrappedDestination.getTableSpec(),
        existingTable.getDescription(),
        existingTable.getTimePartitioning(),
        existingTable.getClustering());
  }
}
 

@Override
public synchronized long getEstimatedSizeBytes(PipelineOptions options) throws Exception {
  if (tableSizeBytes.get() == null) {
    BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
    TableReference tableRef = tableDef.getTableReference(bqOptions);
    Table table = bqServices.getDatasetService(bqOptions).getTable(tableRef);
    Long numBytes = table.getNumBytes();
    if (table.getStreamingBuffer() != null
        && table.getStreamingBuffer().getEstimatedBytes() != null) {
      numBytes += table.getStreamingBuffer().getEstimatedBytes().longValue();
    }

    tableSizeBytes.compareAndSet(null, numBytes);
  }
  return tableSizeBytes.get();
}
 

@Override
public Table patchTableDescription(
    TableReference tableReference, @Nullable String tableDescription)
    throws IOException, InterruptedException {
  Table table = new Table();
  table.setDescription(tableDescription);

  return executeWithRetries(
      client
          .tables()
          .patch(
              tableReference.getProjectId(),
              tableReference.getDatasetId(),
              tableReference.getTableId(),
              table),
      String.format(
          "Unable to patch table description: %s, aborting after %d retries.",
          tableReference, MAX_RPC_RETRIES),
      Sleeper.DEFAULT,
      createDefaultBackoff(),
      ALWAYS_RETRY);
}
 

@Test
public void testSuccess_doPost() throws Exception {
  action.run();

  InOrder factoryOrder = inOrder(checkedBigquery);
  // Check that the BigQuery factory was called in such a way that the dataset would be created
  // if it didn't already exist.
  factoryOrder
      .verify(checkedBigquery)
      .ensureDataSetExists("myproject", "latest_datastore_export");

  // Check that we updated both views
  InOrder tableOrder = inOrder(bigqueryTables);
  ArgumentCaptor<Table> tableArg = ArgumentCaptor.forClass(Table.class);
  tableOrder
      .verify(bigqueryTables)
      .update(eq("myproject"), eq("latest_datastore_export"), eq("fookind"), tableArg.capture());
  Iterable<String> actualQueries =
      Iterables.transform(tableArg.getAllValues(), table -> table.getView().getQuery());
  assertThat(actualQueries)
      .containsExactly("#standardSQL\nSELECT * FROM `myproject.some_dataset.12345_fookind`");
}
 

@Override
protected Table getTargetTable(BigQueryOptions options) throws Exception {
  if (cachedTable.get() == null) {
    TableReference tableReference = tableReferenceProvider.get();
    if (Strings.isNullOrEmpty(tableReference.getProjectId())) {
      checkState(
          !Strings.isNullOrEmpty(options.getProject()),
          "No project ID set in %s or %s, cannot construct a complete %s",
          TableReference.class.getSimpleName(),
          BigQueryOptions.class.getSimpleName(),
          TableReference.class.getSimpleName());
      LOG.info(
          "Project ID not set in {}. Using default project from {}.",
          TableReference.class.getSimpleName(),
          BigQueryOptions.class.getSimpleName());
      tableReference.setProjectId(options.getProject());
    }
    Table table = bqServices.getDatasetService(options).getTable(tableReference);
    cachedTable.compareAndSet(null, table);
  }

  return cachedTable.get();
}
 

public NoopFederatedExportToCloudStorage(
    Configuration configuration,
    ExportFileFormat fileFormat,
    BigQueryHelper bigQueryHelper,
    String projectId,
    Table table,
    @Nullable InputFormat<LongWritable, Text> delegateInputFormat) {
  super(
      configuration, getCommaSeparatedGcsPathList(table), fileFormat, bigQueryHelper,
      projectId, table, delegateInputFormat);
  Preconditions.checkNotNull(table.getExternalDataConfiguration());
  String inputType = fileFormat.getFormatIdentifier();
  String tableType = table.getExternalDataConfiguration().getSourceFormat();
  Preconditions.checkArgument(
      inputType.equals(tableType),
      "MapReduce fileFormat '%s' does not match BigQuery sourceFormat '%s'. Use the "
          + "appropriate InputFormat.",
      inputType, tableType);
  gcsPaths = table.getExternalDataConfiguration().getSourceUris();
}
 

/** Returns true if the table exists, or false if not. */
public boolean tableExists(TableReference tableRef) throws IOException {
  try {
    Table fetchedTable =
        service
            .tables()
            .get(tableRef.getProjectId(), tableRef.getDatasetId(), tableRef.getTableId())
            .execute();
    logger.atFine().log(
        "Successfully fetched table '%s' for tableRef '%s'", fetchedTable, tableRef);
    return true;
  } catch (IOException ioe) {
    if (errorExtractor.itemNotFound(ioe)) {
      return false;
    }
    throw new IOException(
        String.format("Unhandled exception trying to get table '%s'", tableRef), ioe);
  }
}
 

void createTable(String projectId, Table table)
        throws IOException
{
    String datasetId = table.getTableReference().getDatasetId();
    try {
        client.tables().insert(projectId, datasetId, table)
                .execute();
    }
    catch (GoogleJsonResponseException e) {
        if (e.getStatusCode() == HttpStatusCodes.STATUS_CODE_CONFLICT) {
            logger.debug("Table already exists: {}:{}.{}", projectId, datasetId, table.getTableReference().getTableId());
        }
        else {
            throw e;
        }
    }
}
 

@Test
public void testGetTableNotFound() throws IOException, InterruptedException {
  when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
  when(response.getStatusCode()).thenReturn(404);

  BigQueryServicesImpl.DatasetServiceImpl datasetService =
      new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());

  TableReference tableRef =
      new TableReference()
          .setProjectId("projectId")
          .setDatasetId("datasetId")
          .setTableId("tableId");
  Table table = datasetService.getTable(tableRef, null, BackOff.ZERO_BACKOFF, Sleeper.DEFAULT);

  assertNull(table);
  verify(response, times(1)).getStatusCode();
  verify(response, times(1)).getContent();
  verify(response, times(1)).getContentType();
}
 

/** Tests that table creation succeeds when the table already exists. */
@Test
public void testCreateTableSucceedsAlreadyExists() throws IOException {
  TableReference ref =
      new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
  TableSchema schema =
      new TableSchema()
          .setFields(
              ImmutableList.of(
                  new TableFieldSchema().setName("column1").setType("String"),
                  new TableFieldSchema().setName("column2").setType("Integer")));
  Table testTable = new Table().setTableReference(ref).setSchema(schema);

  when(response.getStatusCode()).thenReturn(409); // 409 means already exists

  BigQueryServicesImpl.DatasetServiceImpl services =
      new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
  Table ret =
      services.tryCreateTable(
          testTable, new RetryBoundedBackOff(0, BackOff.ZERO_BACKOFF), Sleeper.DEFAULT);

  assertNull(ret);
  verify(response, times(1)).getStatusCode();
  verify(response, times(1)).getContent();
  verify(response, times(1)).getContentType();
}
 

/**
 * Make a new table for use in a test.
 *
 * @return The name of the table
 * @throws Exception if anything goes awry
 */
public String makeTestTable() throws Exception {
  String tableName =
      TEST_TABLE_NAME_BASE + System.currentTimeMillis() + "_" + (new SecureRandom().nextInt(32));

  BQ_CLIENT.createNewTable(
      project,
      BIG_QUERY_DATASET_ID,
      new Table()
          .setSchema(BASE_TABLE_SCHEMA)
          .setTableReference(
              new TableReference()
                  .setTableId(tableName)
                  .setDatasetId(BIG_QUERY_DATASET_ID)
                  .setProjectId(project)));

  return tableName;
}
 

private void doTableSourceEstimatedSizeTest(boolean useStreamingBuffer) throws Exception {
  fakeDatasetService.createDataset("foo.com:project", "dataset", "", "", null);
  TableReference tableRef = BigQueryHelpers.parseTableSpec("foo.com:project:dataset.table");
  Table table = new Table().setTableReference(tableRef).setNumBytes(100L);
  if (useStreamingBuffer) {
    table.setStreamingBuffer(new Streamingbuffer().setEstimatedBytes(BigInteger.TEN));
  }

  fakeDatasetService.createTable(table);

  BigQueryStorageTableSource<TableRow> tableSource =
      BigQueryStorageTableSource.create(
          ValueProvider.StaticValueProvider.of(tableRef),
          null,
          null,
          null,
          new TableRowParser(),
          TableRowJsonCoder.of(),
          new FakeBigQueryServices().withDatasetService(fakeDatasetService));

  assertEquals(100, tableSource.getEstimatedSizeBytes(options));
}
 

@Test
public void testTableSourceEstimatedSize_WithDefaultProject() throws Exception {
  fakeDatasetService.createDataset("project-id", "dataset", "", "", null);
  TableReference tableRef = BigQueryHelpers.parseTableSpec("project-id:dataset.table");
  Table table = new Table().setTableReference(tableRef).setNumBytes(100L);
  fakeDatasetService.createTable(table);

  BigQueryStorageTableSource<TableRow> tableSource =
      BigQueryStorageTableSource.create(
          ValueProvider.StaticValueProvider.of(BigQueryHelpers.parseTableSpec("dataset.table")),
          null,
          null,
          null,
          new TableRowParser(),
          TableRowJsonCoder.of(),
          new FakeBigQueryServices().withDatasetService(fakeDatasetService));

  assertEquals(100, tableSource.getEstimatedSizeBytes(options));
}
 

private Table table(String defaultProjectId, Optional<DatasetReference> defaultDataset, JsonNode node)
{
    if (node.isTextual()) {
        return new Table()
                .setTableReference(Bq.tableReference(defaultProjectId, defaultDataset, node.asText()));
    }
    else {
        TableConfig config;
        try {
            config = objectMapper.readValue(node.traverse(), TableConfig.class);
        }
        catch (IOException e) {
            throw new ConfigException("Invalid table reference or configuration: " + node, e);
        }
        return table(defaultProjectId, defaultDataset, config);
    }
}
 

@Test
public void testCreateTableSucceeds() throws IOException {
  TableReference ref =
      new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
  Table testTable = new Table().setTableReference(ref);
  when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
  when(response.getStatusCode()).thenReturn(200);
  when(response.getContent()).thenReturn(toStream(testTable));

  BigQueryServicesImpl.DatasetServiceImpl services =
      new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
  Table ret =
      services.tryCreateTable(
          testTable, new RetryBoundedBackOff(0, BackOff.ZERO_BACKOFF), Sleeper.DEFAULT);
  assertEquals(testTable, ret);
  verify(response, times(1)).getStatusCode();
  verify(response, times(1)).getContent();
  verify(response, times(1)).getContentType();
}
 

@Test
public void testExecuteWithRetries() throws IOException, InterruptedException {
  Table testTable = new Table();

  when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
  when(response.getStatusCode()).thenReturn(200);
  when(response.getContent()).thenReturn(toStream(testTable));

  Table table =
      BigQueryServicesImpl.executeWithRetries(
          bigquery.tables().get("projectId", "datasetId", "tableId"),
          "Failed to get table.",
          Sleeper.DEFAULT,
          BackOff.STOP_BACKOFF,
          BigQueryServicesImpl.ALWAYS_RETRY);

  assertEquals(testTable, table);
  verify(response, times(1)).getStatusCode();
  verify(response, times(1)).getContent();
  verify(response, times(1)).getContentType();
}
 

/**
 * Updates the specified Bigquery table to reflect the metadata from the input.
 *
 * <p>Returns the input DestinationTable. If the specified table does not already exist, it will
 * be inserted into the dataset.
 *
 * <p>Clients can call this function directly to update a table on demand, or can pass it to
 * Futures.transform() to update a table produced as the asynchronous result of a load or query
 * job (e.g. to add a description to it).
 */
private DestinationTable updateTable(final DestinationTable destinationTable) {
  Table table = destinationTable.getTable();
  TableReference ref = table.getTableReference();
  try {
    if (checkTableExists(ref.getDatasetId(), ref.getTableId())) {
      // Make sure to use patch() rather than update(). The former changes only those properties
      // which are specified, while the latter would change everything, blanking out unspecified
      // properties.
      bigquery
          .tables()
          .patch(ref.getProjectId(), ref.getDatasetId(), ref.getTableId(), table)
          .execute();
    } else {
      bigquery.tables().insert(ref.getProjectId(), ref.getDatasetId(), table).execute();
    }
    return destinationTable;
  } catch (IOException e) {
    throw BigqueryJobFailureException.create(e);
  }
}
 

@Test
public void testE2EBigQueryClustering() throws Exception {
  String tableName = "weather_stations_clustered_" + System.currentTimeMillis();

  Pipeline p = Pipeline.create(options);

  p.apply(BigQueryIO.readTableRows().from(options.getBqcInput()))
      .apply(ParDo.of(new KeepStationNumberAndConvertDate()))
      .apply(
          BigQueryIO.writeTableRows()
              .to(String.format("%s.%s", DATASET_NAME, tableName))
              .withTimePartitioning(TIME_PARTITIONING)
              .withClustering(CLUSTERING)
              .withSchema(SCHEMA)
              .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED)
              .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE));

  p.run().waitUntilFinish();

  Table table = bqClient.tables().get(options.getProject(), DATASET_NAME, tableName).execute();

  Assert.assertEquals(table.getClustering(), CLUSTERING);
}
 

@Test
public void testE2EBigQueryTimePartitioning() throws Exception {
  String tableName = "weather_stations_time_partitioned_" + System.currentTimeMillis();

  Pipeline p = Pipeline.create(options);

  p.apply(BigQueryIO.readTableRows().from(options.getBqcInput()))
      .apply(ParDo.of(new KeepStationNumberAndConvertDate()))
      .apply(
          BigQueryIO.writeTableRows()
              .to(String.format("%s.%s", DATASET_NAME, tableName))
              .withTimePartitioning(TIME_PARTITIONING)
              .withSchema(SCHEMA)
              .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED)
              .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE));

  p.run().waitUntilFinish();

  bqClient = BigqueryClient.getNewBigquerryClient(options.getAppName());
  Table table = bqClient.tables().get(options.getProject(), DATASET_NAME, tableName).execute();

  Assert.assertEquals(table.getTimePartitioning(), TIME_PARTITIONING);
}
 

/** Returns a new copy of the Bigquery API Table object wrapped by this DestinationTable. */
private Table getTable() {
  Table tableCopy = table.clone();
  if (type == TableType.VIEW) {
    tableCopy.setView(new ViewDefinition().setQuery(query));
  }
  return tableCopy;
}
 

@Test
public void testEstimatedSizeWithStreamingBuffer() throws Exception {
  List<TableRow> data =
      ImmutableList.of(
          new TableRow().set("name", "a").set("number", 1L),
          new TableRow().set("name", "b").set("number", 2L),
          new TableRow().set("name", "c").set("number", 3L),
          new TableRow().set("name", "d").set("number", 4L),
          new TableRow().set("name", "e").set("number", 5L),
          new TableRow().set("name", "f").set("number", 6L));

  TableReference table = BigQueryHelpers.parseTableSpec("project:data_set.table_name");
  fakeDatasetService.createDataset("project", "data_set", "", "", null);
  fakeDatasetService.createTable(
      new Table()
          .setTableReference(table)
          .setSchema(
              new TableSchema()
                  .setFields(
                      ImmutableList.of(
                          new TableFieldSchema().setName("name").setType("STRING"),
                          new TableFieldSchema().setName("number").setType("INTEGER"))))
          .setStreamingBuffer(new Streamingbuffer().setEstimatedBytes(BigInteger.valueOf(10))));
  fakeDatasetService.insertAll(table, data, null);

  String stepUuid = "testStepUuid";
  BoundedSource<TableRow> bqSource =
      BigQueryTableSourceDef.create(fakeBqServices, ValueProvider.StaticValueProvider.of(table))
          .toSource(stepUuid, TableRowJsonCoder.of(), BigQueryIO.TableRowParser.INSTANCE);

  PipelineOptions options = PipelineOptionsFactory.create();

  // Each row should have 24 bytes (See StringUtf8Coder in detail):
  //   first 1 byte indicating length and following 23 bytes: {"name":"a","number":1}
  // 10 bytes comes from the estimated bytes of the Streamingbuffer
  long expectedSize = 24L * data.size() + 10;
  assertEquals(expectedSize, bqSource.getEstimatedSizeBytes(options));
}
 

@Test
public void testEstimatedSizeWithoutStreamingBuffer() throws Exception {
  List<TableRow> data =
      ImmutableList.of(
          new TableRow().set("name", "a").set("number", 1L),
          new TableRow().set("name", "b").set("number", 2L),
          new TableRow().set("name", "c").set("number", 3L),
          new TableRow().set("name", "d").set("number", 4L),
          new TableRow().set("name", "e").set("number", 5L),
          new TableRow().set("name", "f").set("number", 6L));

  TableReference table = BigQueryHelpers.parseTableSpec("project:data_set.table_name");
  fakeDatasetService.createDataset("project", "data_set", "", "", null);
  fakeDatasetService.createTable(
      new Table()
          .setTableReference(table)
          .setSchema(
              new TableSchema()
                  .setFields(
                      ImmutableList.of(
                          new TableFieldSchema().setName("name").setType("STRING"),
                          new TableFieldSchema().setName("number").setType("INTEGER")))));
  fakeDatasetService.insertAll(table, data, null);

  String stepUuid = "testStepUuid";
  BoundedSource<TableRow> bqSource =
      BigQueryTableSourceDef.create(fakeBqServices, ValueProvider.StaticValueProvider.of(table))
          .toSource(stepUuid, TableRowJsonCoder.of(), BigQueryIO.TableRowParser.INSTANCE);

  PipelineOptions options = PipelineOptionsFactory.create();

  // Each row should have 24 bytes (See StringUtf8Coder in detail):
  //   first 1 byte indicating length and following 23 bytes: {"name":"a","number":1}
  long expectedSize = 24L * data.size();
  assertEquals(expectedSize, bqSource.getEstimatedSizeBytes(options));
}
 

/** Returns the table schema for the destination. May not return null. */
@Override
public TableSchema getSchema(DestinationT destination) {
  TableDestination wrappedDestination = super.getTable(destination);
  Table existingTable = getBigQueryTable(wrappedDestination.getTableReference());
  if (existingTable == null) {
    return super.getSchema(destination);
  } else {
    return existingTable.getSchema();
  }
}
 

/**
 * Tests query job and table creation with KMS key settings.
 *
 * <p>Verifies table creation with KMS key.
 */
private void testQueryAndWrite(Method method) throws Exception {
  String outputTableId = "testQueryAndWrite_" + method.name();
  String outputTableSpec = project + ":" + BIG_QUERY_DATASET_ID + "." + outputTableId;

  options.setTempLocation(options.getTempRoot() + "/bq_it_temp");
  Pipeline p = Pipeline.create(options);
  // Reading triggers BQ query and extract jobs. Writing triggers either a load job or performs a
  // streaming insert (depending on method).
  p.apply(
          BigQueryIO.readTableRows()
              .fromQuery("SELECT * FROM (SELECT \"foo\" as fruit)")
              .withKmsKey(kmsKey))
      .apply(
          BigQueryIO.writeTableRows()
              .to(outputTableSpec)
              .withSchema(OUTPUT_SCHEMA)
              .withMethod(method)
              .withKmsKey(kmsKey));
  p.run().waitUntilFinish();

  Table table = BQ_CLIENT.getTableResource(project, BIG_QUERY_DATASET_ID, outputTableId);
  assertNotNull(String.format("table not found: %s", outputTableId), table);
  assertNotNull(
      "output table has no EncryptionConfiguration", table.getEncryptionConfiguration());
  assertEquals(table.getEncryptionConfiguration().getKmsKeyName(), kmsKey);
}
 

private Table basicTableSchema() {
  return new Table()
      .setSchema(
          new TableSchema()
              .setFields(
                  Arrays.asList(
                      new TableFieldSchema().setName("name").setType("STRING"),
                      new TableFieldSchema().setName("answer").setType("INTEGER"))));
}
 

/** Tests that {@link BigQueryServicesImpl} retries quota rate limited attempts. */
@Test
public void testCreateTableRetry() throws IOException {
  TableReference ref =
      new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
  Table testTable = new Table().setTableReference(ref);

  // First response is 403 rate limited, second response has valid payload.
  when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
  when(response.getStatusCode()).thenReturn(403).thenReturn(200);
  when(response.getContent())
      .thenReturn(toStream(errorWithReasonAndStatus("rateLimitExceeded", 403)))
      .thenReturn(toStream(testTable));

  BigQueryServicesImpl.DatasetServiceImpl services =
      new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
  Table ret =
      services.tryCreateTable(
          testTable, new RetryBoundedBackOff(3, BackOff.ZERO_BACKOFF), Sleeper.DEFAULT);
  assertEquals(testTable, ret);
  verify(response, times(2)).getStatusCode();
  verify(response, times(2)).getContent();
  verify(response, times(2)).getContentType();
  verifyNotNull(ret.getTableReference());
  expectedLogs.verifyInfo(
      "Quota limit reached when creating table project:dataset.table, "
          + "retrying up to 5.0 minutes");
}
 

/**
 * Tests the cleanupJob method of GsonBigQueryInputFormat with no intermediate delete.
 */
@Test
public void testCleanupJobWithNoIntermediateDelete()
    throws IOException {
  config.setBoolean(BigQueryConfiguration.DELETE_EXPORT_FILES_FROM_GCS.getKey(), true);

  when(mockBigQueryHelper.getTable(any(TableReference.class)))
      .thenReturn(new Table());

  Path tempPath = new Path(BigQueryConfiguration.TEMP_GCS_PATH.get(config, config::get));
  FileSystem fs = tempPath.getFileSystem(config);
  fs.mkdirs(tempPath);
  Path dataFile = new Path(tempPath.toString() + "/data-00000.json");
  fs.createNewFile(dataFile);
  assertThat(fs.exists(tempPath)).isTrue();
  assertThat(fs.exists(dataFile)).isTrue();

  // Run method and verify calls.
  GsonBigQueryInputFormat.cleanupJob(mockBigQueryHelper, config);

  assertThat(!fs.exists(tempPath)).isTrue();
  assertThat(!fs.exists(dataFile)).isTrue();

  verify(mockBigQueryHelper, times(1)).getTable(eq(tableRef));

  verifyNoMoreInteractions(mockBigquery, mockBigqueryTables);
}
 

private static Export constructExport(
    Configuration configuration,
    ExportFileFormat format,
    String exportPath,
    BigQueryHelper bigQueryHelper,
    InputFormat<LongWritable, Text> delegateInputFormat)
    throws IOException {
  logger.atFine().log("constructExport() with export path %s", exportPath);

  // Extract relevant configuration settings.
  Map<String, String> mandatoryConfig =
      getMandatoryConfig(configuration, MANDATORY_CONFIG_PROPERTIES_INPUT);
  String jobProjectId = mandatoryConfig.get(PROJECT_ID.getKey());
  String inputProjectId = mandatoryConfig.get(INPUT_PROJECT_ID.getKey());
  String datasetId = mandatoryConfig.get(INPUT_DATASET_ID.getKey());
  String tableName = mandatoryConfig.get(INPUT_TABLE_ID.getKey());

  TableReference exportTableReference = new TableReference()
      .setDatasetId(datasetId)
      .setProjectId(inputProjectId)
      .setTableId(tableName);
  Table table = bigQueryHelper.getTable(exportTableReference);

  if (EXTERNAL_TABLE_TYPE.equals(table.getType())) {
      logger.atInfo().log("Table is already external, so skipping export");
      return new NoopFederatedExportToCloudStorage(
          configuration, format, bigQueryHelper, jobProjectId, table, delegateInputFormat);
  }

  return new UnshardedExportToCloudStorage(
      configuration,
      exportPath,
      format,
      bigQueryHelper,
      jobProjectId,
      table,
      delegateInputFormat);
}
 

private Table createTable(Description description) throws IOException, InterruptedException {
  TableReference tableReference =
      new TableReference()
          .setProjectId(pipelineOptions.getProject())
          .setDatasetId(pipelineOptions.getTargetDataset())
          .setTableId(createRandomizedName(description));

  table =
      new Table()
          .setTableReference(tableReference)
          .setSchema(BigQueryUtils.toTableSchema(schema))
          .setDescription(
              "Table created for "
                  + description.getDisplayName()
                  + " by TestBigQueryRule. "
                  + "Should be automatically cleaned up after test completion.");

  if (datasetService.getTable(tableReference) != null) {
    throw new IllegalStateException(
        "Table '"
            + tableReference
            + "' already exists. "
            + "It should have been cleaned up by the test rule.");
  }

  datasetService.createTable(table);
  return table;
}