Java Code Examples for com.google.cloud.bigquery.JobId#of()

@Before
public void setUp() throws Exception {
  mockBigquery = mock(BigQuery.class);
  jobId = JobId.of("test-project", "datacollector");
  mockResult = mock(TableResult.class);

  mockStatic(ServiceAccountCredentials.class);
  mockStatic(GoogleCredentials.class);
  ServiceAccountCredentials serviceAccountCredentials = mock(ServiceAccountCredentials.class);
  when(ServiceAccountCredentials.fromStream(any())).thenReturn(serviceAccountCredentials);
}
 

/** Example of getting a job. */
// [TARGET getJob(JobId, JobOption...)]
// [VARIABLE "my_job_name"]
public Job getJobFromId(String jobName) {
  // [START ]
  JobId jobIdObject = JobId.of(jobName);
  Job job = bigquery.getJob(jobIdObject);
  if (job == null) {
    // job was not found
  }
  // [END ]
  return job;
}
 

/** Example of cancelling a job. */
// [TARGET cancel(JobId)]
// [VARIABLE "my_job_name"]
public boolean cancelJobFromId(String jobName) {
  // [START ]
  JobId jobId = JobId.of(jobName);
  boolean success = bigquery.cancel(jobId);
  if (success) {
    // job was cancelled
  } else {
    // job was not found
  }
  // [END ]
  return success;
}
 

@Override
JobId parse(String... args) throws Exception {
  String message;
  if (args.length == 1) {
    return JobId.of(args[0]);
  } else if (args.length > 1) {
    message = "Too many arguments.";
  } else {
    message = "Missing required query.";
  }
  throw new IllegalArgumentException(message);
}
 

@NewSpan()
public <T> List<T> processQuery(String queryString, Class<T> t, boolean updateCache) throws Exception {

	QueryJobConfiguration queryConfig = QueryJobConfiguration.newBuilder(queryString).build();
	// Create a job ID so that we can safely retry.
	JobId jobId = JobId.of(UUID.randomUUID().toString());
	Job queryJob = bqInstance.create(JobInfo.newBuilder(queryConfig).setJobId(jobId).build());

	// Wait for the query to complete.
	queryJob = queryJob.waitFor();

	// Check for errors
	if (queryJob == null) {
		throw new RuntimeException("Job no longer exists");
	} else if (queryJob.getStatus().getError() != null) {
		// You can also look at queryJob.getStatus().getExecutionErrors() for all
		// errors, not just the latest one.
		throw new RuntimeException(queryJob.getStatus().getError().toString());
	}

	// Get the results.
	TableResult result = queryJob.getQueryResults();
	// init counters
	long count = 0;
	long total = result.getTotalRows();
	LOGGER.info("Fetched " + total + " products.");
	long start = System.currentTimeMillis();
	// Print all pages of the results.
	List<T> results = new ArrayList<T>();
	// filter
	String filter = "[^A-Za-z0-9 ()-]";
	while (result != null) {
		for (List<FieldValue> row : result.iterateAll()) {
			Object type = t.getConstructor().newInstance();
			String productName = null;
			// query for sku, name
			if (type instanceof Product) {
				productName = row.get(1).getValue() != null
						? row.get(1).getStringValue().replaceAll(filter, "").trim()
						: "";
				if (!updateCache) {
					Product product = new Product();
					product.setSku(row.get(0).getValue().toString());

					product.setName(productName);
					results.add(t.cast(product));
				}
				// query for name
			} else if (type instanceof String) {
				productName = row.get(0).getValue() != null
						? row.get(0).getStringValue().replaceAll(filter, "").trim()
						: "";
				if (!updateCache) {
					results.add(t.cast(productName));
				}
			}

			if (updateCache) {
				getZSetOps().add(productName.toLowerCase() + ":" + productName, 0);
			}
			count++;
		}
		LOGGER.info("Processed " + count + " records..");
		result = result.getNextPage();
	}
	if (updateCache) {
		long actual = getZSetOps().zCard();
		LOGGER.info("Indexing completed for " + count + " products.");
		LOGGER.info("Products in cache: " + actual);
		LOGGER.info("Duplicate product names: " + (count - actual));
		LOGGER.info("Time taken: " + (System.currentTimeMillis() - start) / 1000 + "s.");
	}

	return results;

}
 

public void runQuery() throws InterruptedException {
  QueryJobConfiguration queryConfig =
      QueryJobConfiguration.newBuilder(
              "SELECT "
                  + "CONCAT('https://stackoverflow.com/questions/', CAST(id as STRING)) as url, "
                  + "view_count "
                  + "FROM `bigquery-public-data.stackoverflow.posts_questions` "
                  + "WHERE tags like '%google-bigquery%' "
                  + "ORDER BY favorite_count DESC LIMIT 10")
          // Use standard SQL syntax for queries.
          // See: https://cloud.google.com/bigquery/sql-reference/
          .setUseLegacySql(false)
          .build();

  List<TimeSeries> timeSeriesList = new ArrayList<>();

  long queryStartTime = System.currentTimeMillis();

  // Create a job ID so that we can safely retry.
  JobId jobId = JobId.of(UUID.randomUUID().toString());
  Job queryJob = bigquery.create(JobInfo.newBuilder(queryConfig).setJobId(jobId).build());

  // Wait for the query to complete.
  queryJob = queryJob.waitFor();

  // Check for errors
  if (queryJob == null) {
    throw new RuntimeException("Job no longer exists");
  } else if (queryJob.getStatus().getError() != null) {
    // You can also look at queryJob.getStatus().getExecutionErrors() for all
    // errors, not just the latest one.
    throw new RuntimeException(queryJob.getStatus().getError().toString());
  }

  // Log the result metrics.
  TableResult result = queryJob.getQueryResults();

  long queryEndTime = System.currentTimeMillis();
  // Add query duration metric.
  timeSeriesList.add(prepareMetric(QUERY_DURATION_METRIC, queryEndTime - queryStartTime));

  // Add rows returned metric.
  timeSeriesList.add(prepareMetric(ROWS_RETURNED_METRIC, result.getTotalRows()));

  // Prepares the time series request
  CreateTimeSeriesRequest request =
      CreateTimeSeriesRequest.newBuilder()
          .setName(projectName)
          .addAllTimeSeries(timeSeriesList)
          .build();

  createMetricsIfNeeded();
  client.createTimeSeries(request);
  os.println("Done writing metrics.");

  mostRecentRunResult = result;
}
 

@Before
public void setUp() throws Exception {
  mockBigquery = mock(BigQuery.class);
  jobId = JobId.of("test-project", "datacollector");
}