Java Code Examples for com.google.cloud.bigquery.TableResult#iterateAll()

/** Example of listing table rows, specifying the page size. */
// [TARGET listTableData(String, String, TableDataListOption...)]
// [VARIABLE "my_dataset_name"]
// [VARIABLE "my_table_name"]
public TableResult listTableData(String datasetName, String tableName) {
  // [START ]
  // This example reads the result 100 rows per RPC call. If there's no need to limit the number,
  // simply omit the option.
  TableResult tableData =
      bigquery.listTableData(datasetName, tableName, TableDataListOption.pageSize(100));
  for (FieldValueList row : tableData.iterateAll()) {
    // do something with the row
  }
  // [END ]
  return tableData;
}
 

/** Example of listing table rows, specifying the page size. */
// [TARGET listTableData(TableId, TableDataListOption...)]
// [VARIABLE "my_dataset_name"]
// [VARIABLE "my_table_name"]
public TableResult listTableDataFromId(String datasetName, String tableName) {
  // [START bigquery_browse_table]
  TableId tableIdObject = TableId.of(datasetName, tableName);
  // This example reads the result 100 rows per RPC call. If there's no need to limit the number,
  // simply omit the option.
  TableResult tableData =
      bigquery.listTableData(tableIdObject, TableDataListOption.pageSize(100));
  for (FieldValueList row : tableData.iterateAll()) {
    // do something with the row
  }
  // [END bigquery_browse_table]
  return tableData;
}
 

/** Example of listing table rows with schema. */
// [TARGET listTableData(String, String, Schema, TableDataListOption...)]
// [VARIABLE "my_dataset_name"]
// [VARIABLE "my_table_name"]
// [VARIABLE ...]
// [VARIABLE "field"]
public TableResult listTableDataSchema(
    String datasetName, String tableName, Schema schema, String field) {
  // [START ]
  TableResult tableData = bigquery.listTableData(datasetName, tableName, schema);
  for (FieldValueList row : tableData.iterateAll()) {
    row.get(field);
  }
  // [END ]
  return tableData;
}
 

private static String convertRunToHtmlTable(TableResult result) {
  if (result == null) {
    return "";
  }

  StringBuilder sb = new StringBuilder();
  for (FieldValueList row : result.iterateAll()) {
    sb.append("<tr>");
    String url = row.get("url").getStringValue();
    addColumn(sb, String.format("<a href=\"%s\">%s</a>", url, url));
    addColumn(sb, row.get("view_count").getLongValue());
    sb.append("</tr>");
  }
  return sb.toString();
}
 

@NewSpan()
public <T> List<T> processQuery(String queryString, Class<T> t, boolean updateCache) throws Exception {

	QueryJobConfiguration queryConfig = QueryJobConfiguration.newBuilder(queryString).build();
	// Create a job ID so that we can safely retry.
	JobId jobId = JobId.of(UUID.randomUUID().toString());
	Job queryJob = bqInstance.create(JobInfo.newBuilder(queryConfig).setJobId(jobId).build());

	// Wait for the query to complete.
	queryJob = queryJob.waitFor();

	// Check for errors
	if (queryJob == null) {
		throw new RuntimeException("Job no longer exists");
	} else if (queryJob.getStatus().getError() != null) {
		// You can also look at queryJob.getStatus().getExecutionErrors() for all
		// errors, not just the latest one.
		throw new RuntimeException(queryJob.getStatus().getError().toString());
	}

	// Get the results.
	TableResult result = queryJob.getQueryResults();
	// init counters
	long count = 0;
	long total = result.getTotalRows();
	LOGGER.info("Fetched " + total + " products.");
	long start = System.currentTimeMillis();
	// Print all pages of the results.
	List<T> results = new ArrayList<T>();
	// filter
	String filter = "[^A-Za-z0-9 ()-]";
	while (result != null) {
		for (List<FieldValue> row : result.iterateAll()) {
			Object type = t.getConstructor().newInstance();
			String productName = null;
			// query for sku, name
			if (type instanceof Product) {
				productName = row.get(1).getValue() != null
						? row.get(1).getStringValue().replaceAll(filter, "").trim()
						: "";
				if (!updateCache) {
					Product product = new Product();
					product.setSku(row.get(0).getValue().toString());

					product.setName(productName);
					results.add(t.cast(product));
				}
				// query for name
			} else if (type instanceof String) {
				productName = row.get(0).getValue() != null
						? row.get(0).getStringValue().replaceAll(filter, "").trim()
						: "";
				if (!updateCache) {
					results.add(t.cast(productName));
				}
			}

			if (updateCache) {
				getZSetOps().add(productName.toLowerCase() + ":" + productName, 0);
			}
			count++;
		}
		LOGGER.info("Processed " + count + " records..");
		result = result.getNextPage();
	}
	if (updateCache) {
		long actual = getZSetOps().zCard();
		LOGGER.info("Indexing completed for " + count + " products.");
		LOGGER.info("Products in cache: " + actual);
		LOGGER.info("Duplicate product names: " + (count - actual));
		LOGGER.info("Time taken: " + (System.currentTimeMillis() - start) / 1000 + "s.");
	}

	return results;

}