com.google.cloud.bigquery.JobId Java Examples

@Test
public void testJob() throws ExecutionException, InterruptedException {
  Job job1 = bigquerySnippets.createJob(QUERY);
  Job job2 = bigquerySnippets.createJob(QUERY);
  assertNotNull(job1);
  assertNotNull(job2);
  assertEquals(job1.getJobId(), bigquerySnippets.getJob(job1.getJobId().getJob()).getJobId());
  assertEquals(
      job2.getJobId(), bigquerySnippets.getJobFromId(job2.getJobId().getJob()).getJobId());
  Set<JobId> jobs =
      Sets.newHashSet(
          Iterators.transform(
              bigquerySnippets.listJobs().iterateAll().iterator(), TO_JOB_ID_FUNCTION));
  while (!jobs.contains(job1.getJobId()) || !jobs.contains(job2.getJobId())) {
    Thread.sleep(500);
    jobs =
        Sets.newHashSet(
            Iterators.transform(
                bigquerySnippets.listJobs().iterateAll().iterator(), TO_JOB_ID_FUNCTION));
  }
  assertTrue(bigquerySnippets.cancelJob(job1.getJobId().getJob()));
  assertTrue(bigquerySnippets.cancelJobFromId(job2.getJobId().getJob()));
}
 

private Job issueQueryToBQ(String statement) throws InterruptedException {
  QueryJobConfiguration jobConfiguration = QueryJobConfiguration.newBuilder(statement)
      .build();

  String jobId = makeJobId(jobIdPrefix, statement);

  LOG.info("Triggering job {} for statement |{}|", jobId, statement);

  TableResult result = bigQueryClient.query(jobConfiguration, JobId.of(jobId));
  return bigQueryClient.getJob(JobId.of(jobId));
}
 

@Before
public void setUp() throws Exception {
  mockBigquery = mock(BigQuery.class);
  jobId = JobId.of("test-project", "datacollector");
  mockResult = mock(TableResult.class);

  mockStatic(ServiceAccountCredentials.class);
  mockStatic(GoogleCredentials.class);
  ServiceAccountCredentials serviceAccountCredentials = mock(ServiceAccountCredentials.class);
  when(ServiceAccountCredentials.fromStream(any())).thenReturn(serviceAccountCredentials);
}
 

/** Example of running a batch query. */
public void runBatchQuery() throws TimeoutException, InterruptedException {
  // [START bigquery_query_batch]
  // BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService();
  String query = "SELECT corpus FROM `bigquery-public-data.samples.shakespeare` GROUP BY corpus;";
  QueryJobConfiguration queryConfig =
      QueryJobConfiguration.newBuilder(query)
          // Run at batch priority, which won't count toward concurrent rate
          // limit.
          .setPriority(QueryJobConfiguration.Priority.BATCH)
          .build();

  // Location must match that of the dataset(s) referenced in the query.
  JobId jobId = JobId.newBuilder().setRandomJob().setLocation("US").build();
  String jobIdString = jobId.getJob();

  // API request - starts the query.
  bigquery.create(JobInfo.newBuilder(queryConfig).setJobId(jobId).build());

  // Check on the progress by getting the job's updated state. Once the state
  // is `DONE`, the results are ready.
  Job queryJob =
      bigquery.getJob(JobId.newBuilder().setJob(jobIdString).setLocation("US").build());
  System.out.printf(
      "Job %s in location %s currently in state: %s%n",
      queryJob.getJobId().getJob(),
      queryJob.getJobId().getLocation(),
      queryJob.getStatus().getState().toString());
  // [END bigquery_query_batch]
}
 

/** Example of writing a local file to a table. */
// [TARGET writer(WriteChannelConfiguration)]
// [VARIABLE "my_dataset_name"]
// [VARIABLE "my_table_name"]
// [VARIABLE FileSystems.getDefault().getPath(".", "my-data.csv")]
// [VARIABLE "us"]
public long writeFileToTable(String datasetName, String tableName, Path csvPath, String location)
    throws IOException, InterruptedException, TimeoutException {
  // [START bigquery_load_from_file]
  TableId tableId = TableId.of(datasetName, tableName);
  WriteChannelConfiguration writeChannelConfiguration =
      WriteChannelConfiguration.newBuilder(tableId).setFormatOptions(FormatOptions.csv()).build();
  // Generally, location can be inferred based on the location of the referenced dataset.
  // However,
  // it can also be set explicitly to force job execution to be routed to a specific processing
  // location.  See https://cloud.google.com/bigquery/docs/locations for more info.
  JobId jobId = JobId.newBuilder().setLocation(location).build();
  TableDataWriteChannel writer = bigquery.writer(jobId, writeChannelConfiguration);
  // Write data to writer
  try (OutputStream stream = Channels.newOutputStream(writer)) {
    Files.copy(csvPath, stream);
  } finally {
    writer.close();
  }
  // Get load job
  Job job = writer.getJob();
  job = job.waitFor();
  LoadStatistics stats = job.getStatistics();
  return stats.getOutputRows();
  // [END bigquery_load_from_file]
}
 

/** Example of getting a job. */
// [TARGET getJob(JobId, JobOption...)]
// [VARIABLE "my_job_name"]
public Job getJobFromId(String jobName) {
  // [START ]
  JobId jobIdObject = JobId.of(jobName);
  Job job = bigquery.getJob(jobIdObject);
  if (job == null) {
    // job was not found
  }
  // [END ]
  return job;
}
 

/** Example of cancelling a job. */
// [TARGET cancel(JobId)]
// [VARIABLE "my_job_name"]
public boolean cancelJobFromId(String jobName) {
  // [START ]
  JobId jobId = JobId.of(jobName);
  boolean success = bigquery.cancel(jobId);
  if (success) {
    // job was cancelled
  } else {
    // job was not found
  }
  // [END ]
  return success;
}
 

@Override
JobId parse(String... args) throws Exception {
  String message;
  if (args.length == 1) {
    return JobId.of(args[0]);
  } else if (args.length > 1) {
    message = "Too many arguments.";
  } else {
    message = "Missing required query.";
  }
  throw new IllegalArgumentException(message);
}
 

@Override
public void run(BigQuery bigquery, JobId jobId) {
  if (bigquery.cancel(jobId)) {
    System.out.println("Requested cancel for job " + jobId);
  } else {
    System.out.println("Job " + jobId + " not found");
  }
}
 

@NewSpan()
public <T> List<T> processQuery(String queryString, Class<T> t, boolean updateCache) throws Exception {

	QueryJobConfiguration queryConfig = QueryJobConfiguration.newBuilder(queryString).build();
	// Create a job ID so that we can safely retry.
	JobId jobId = JobId.of(UUID.randomUUID().toString());
	Job queryJob = bqInstance.create(JobInfo.newBuilder(queryConfig).setJobId(jobId).build());

	// Wait for the query to complete.
	queryJob = queryJob.waitFor();

	// Check for errors
	if (queryJob == null) {
		throw new RuntimeException("Job no longer exists");
	} else if (queryJob.getStatus().getError() != null) {
		// You can also look at queryJob.getStatus().getExecutionErrors() for all
		// errors, not just the latest one.
		throw new RuntimeException(queryJob.getStatus().getError().toString());
	}

	// Get the results.
	TableResult result = queryJob.getQueryResults();
	// init counters
	long count = 0;
	long total = result.getTotalRows();
	LOGGER.info("Fetched " + total + " products.");
	long start = System.currentTimeMillis();
	// Print all pages of the results.
	List<T> results = new ArrayList<T>();
	// filter
	String filter = "[^A-Za-z0-9 ()-]";
	while (result != null) {
		for (List<FieldValue> row : result.iterateAll()) {
			Object type = t.getConstructor().newInstance();
			String productName = null;
			// query for sku, name
			if (type instanceof Product) {
				productName = row.get(1).getValue() != null
						? row.get(1).getStringValue().replaceAll(filter, "").trim()
						: "";
				if (!updateCache) {
					Product product = new Product();
					product.setSku(row.get(0).getValue().toString());

					product.setName(productName);
					results.add(t.cast(product));
				}
				// query for name
			} else if (type instanceof String) {
				productName = row.get(0).getValue() != null
						? row.get(0).getStringValue().replaceAll(filter, "").trim()
						: "";
				if (!updateCache) {
					results.add(t.cast(productName));
				}
			}

			if (updateCache) {
				getZSetOps().add(productName.toLowerCase() + ":" + productName, 0);
			}
			count++;
		}
		LOGGER.info("Processed " + count + " records..");
		result = result.getNextPage();
	}
	if (updateCache) {
		long actual = getZSetOps().zCard();
		LOGGER.info("Indexing completed for " + count + " products.");
		LOGGER.info("Products in cache: " + actual);
		LOGGER.info("Duplicate product names: " + (count - actual));
		LOGGER.info("Time taken: " + (System.currentTimeMillis() - start) / 1000 + "s.");
	}

	return results;

}
 

public void runQuery() throws InterruptedException {
  QueryJobConfiguration queryConfig =
      QueryJobConfiguration.newBuilder(
              "SELECT "
                  + "CONCAT('https://stackoverflow.com/questions/', CAST(id as STRING)) as url, "
                  + "view_count "
                  + "FROM `bigquery-public-data.stackoverflow.posts_questions` "
                  + "WHERE tags like '%google-bigquery%' "
                  + "ORDER BY favorite_count DESC LIMIT 10")
          // Use standard SQL syntax for queries.
          // See: https://cloud.google.com/bigquery/sql-reference/
          .setUseLegacySql(false)
          .build();

  List<TimeSeries> timeSeriesList = new ArrayList<>();

  long queryStartTime = System.currentTimeMillis();

  // Create a job ID so that we can safely retry.
  JobId jobId = JobId.of(UUID.randomUUID().toString());
  Job queryJob = bigquery.create(JobInfo.newBuilder(queryConfig).setJobId(jobId).build());

  // Wait for the query to complete.
  queryJob = queryJob.waitFor();

  // Check for errors
  if (queryJob == null) {
    throw new RuntimeException("Job no longer exists");
  } else if (queryJob.getStatus().getError() != null) {
    // You can also look at queryJob.getStatus().getExecutionErrors() for all
    // errors, not just the latest one.
    throw new RuntimeException(queryJob.getStatus().getError().toString());
  }

  // Log the result metrics.
  TableResult result = queryJob.getQueryResults();

  long queryEndTime = System.currentTimeMillis();
  // Add query duration metric.
  timeSeriesList.add(prepareMetric(QUERY_DURATION_METRIC, queryEndTime - queryStartTime));

  // Add rows returned metric.
  timeSeriesList.add(prepareMetric(ROWS_RETURNED_METRIC, result.getTotalRows()));

  // Prepares the time series request
  CreateTimeSeriesRequest request =
      CreateTimeSeriesRequest.newBuilder()
          .setName(projectName)
          .addAllTimeSeries(timeSeriesList)
          .build();

  createMetricsIfNeeded();
  client.createTimeSeries(request);
  os.println("Done writing metrics.");

  mostRecentRunResult = result;
}
 

@Before
public void setUp() throws Exception {
  mockBigquery = mock(BigQuery.class);
  jobId = JobId.of("test-project", "datacollector");
}
 

@Override
public void run(BigQuery bigquery, JobId jobId) {
  System.out.println("Job info: " + bigquery.getJob(jobId));
}
 

@Override
public JobId apply(Job job) {
  return job.getJobId();
}
 

private Job issueQueryToBQ(String statement) throws InterruptedException {
  QueryJobConfiguration jobConfiguration = QueryJobConfiguration.newBuilder(statement).build();

  String jobId = makeJobId(JOB_ID_PREFIX, statement);

  LOG.info("Triggering job {} for statement |{}|", jobId, statement);

  TableResult result = bigQueryClient.query(jobConfiguration, JobId.of(jobId));
  return bigQueryClient.getJob(JobId.of(jobId));
}
 

private Job issueQueryToBQ(String statement) throws InterruptedException {
  QueryJobConfiguration jobConfiguration = QueryJobConfiguration.newBuilder(statement).build();

  String jobId = makeJobId(JOB_ID_PREFIX, statement);

  LOG.info("Triggering job {} for statement |{}|", jobId, statement);

  TableResult result = bigQueryClient.query(jobConfiguration, JobId.of(jobId));
  return bigQueryClient.getJob(JobId.of(jobId));
}