com.google.cloud.bigquery.TableDefinition Java Examples

The following examples show how to use com.google.cloud.bigquery.TableDefinition. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: PutBigQueryStreamingIT.java    From nifi with Apache License 2.0 6 votes vote down vote up 
private void createTable(String tableName) {
    TableId tableId = TableId.of(dataset.getDatasetId().getDataset(), tableName);

    // Table field definition
    Field id = Field.newBuilder("id", LegacySQLTypeName.INTEGER).setMode(Mode.REQUIRED).build();
    Field name = Field.newBuilder("name", LegacySQLTypeName.STRING).setMode(Mode.NULLABLE).build();
    Field alias = Field.newBuilder("alias", LegacySQLTypeName.STRING).setMode(Mode.REPEATED).build();

    Field zip = Field.newBuilder("zip", LegacySQLTypeName.STRING).setMode(Mode.NULLABLE).build();
    Field city = Field.newBuilder("city", LegacySQLTypeName.STRING).setMode(Mode.NULLABLE).build();
    Field addresses = Field.newBuilder("addresses", LegacySQLTypeName.RECORD, zip, city).setMode(Mode.REPEATED).build();

    Field position = Field.newBuilder("position", LegacySQLTypeName.STRING).setMode(Mode.NULLABLE).build();
    Field company = Field.newBuilder("company", LegacySQLTypeName.STRING).setMode(Mode.NULLABLE).build();
    Field job = Field.newBuilder("job", LegacySQLTypeName.RECORD, position, company).setMode(Mode.NULLABLE).build();

    // Table schema definition
    schema = Schema.of(id, name, alias, addresses, job);
    TableDefinition tableDefinition = StandardTableDefinition.of(schema);
    TableInfo tableInfo = TableInfo.newBuilder(tableId, tableDefinition).build();

    // create table
    bigquery.create(tableInfo);
}
Example #2
Source File: BigQuerySnippets.java    From google-cloud-java with Apache License 2.0 6 votes vote down vote up 
/** Example of creating a table. */
// [TARGET create(TableInfo, TableOption...)]
// [VARIABLE "my_dataset_name"]
// [VARIABLE "my_table_name"]
// [VARIABLE "string_field"]
public Table createTable(String datasetName, String tableName, String fieldName) {
  // [START bigquery_create_table]
  TableId tableId = TableId.of(datasetName, tableName);
  // Table field definition
  Field field = Field.of(fieldName, LegacySQLTypeName.STRING);
  // Table schema definition
  Schema schema = Schema.of(field);
  TableDefinition tableDefinition = StandardTableDefinition.of(schema);
  TableInfo tableInfo = TableInfo.newBuilder(tableId, tableDefinition).build();
  Table table = bigquery.create(tableInfo);
  // [END bigquery_create_table]
  return table;
}
Example #3
Source File: BigQueryStatementIssuingFn.java    From DataflowTemplates with Apache License 2.0 6 votes vote down vote up 
private Table createBigQueryTable(BigQueryAction action) {
  TableDefinition definition = StandardTableDefinition.of(
      BigQuerySchemaUtils.beamSchemaToBigQueryClientSchema(action.tableSchema));

  TableId tableId = TableId.of(action.projectId, action.dataset, action.tableName);
  TableInfo tableInfo = TableInfo.newBuilder(tableId, definition).build();

  LOG.info("Creating a new BigQuery table: {}", tableInfo);

  try {
    return bigQueryClient.create(tableInfo);
  } catch (BigQueryException e) {
    if (e.getMessage().startsWith("Already Exists")) {
      return null;
    } else {
      throw e;
    }
  }
}
Example #4
Source File: BigQueryMetadata.java    From presto with Apache License 2.0 6 votes vote down vote up 
private List<SchemaTableName> listTablesWithTypes(ConnectorSession session, Optional<String> schemaName, TableDefinition.Type... types)
{
    if (schemaName.isPresent() && schemaName.get().equalsIgnoreCase(INFORMATION_SCHEMA)) {
        return ImmutableList.of();
    }
    Set<String> schemaNames = schemaName.map(ImmutableSet::of)
            .orElseGet(() -> ImmutableSet.copyOf(listSchemaNames(session)));

    ImmutableList.Builder<SchemaTableName> tableNames = ImmutableList.builder();
    for (String datasetId : schemaNames) {
        for (Table table : bigQueryClient.listTables(DatasetId.of(projectId, datasetId), types)) {
            tableNames.add(new SchemaTableName(datasetId, table.getTableId().getTable()));
        }
    }
    return tableNames.build();
}
Example #5
Source File: BQClientTest.java    From beast with Apache License 2.0 5 votes vote down vote up 
@Test
public void shouldUpdateTableIfTableAlreadyExistsAndSchemaChanges() {
    when(bqConfig.isBQTablePartitioningEnabled()).thenReturn(false);
    when(bqConfig.getTable()).thenReturn("bq-table");
    when(bqConfig.getDataset()).thenReturn("bq-proto");
    bqClient = new BQClient(bigquery, bqConfig);

    ArrayList<Field> bqSchemaFields = new ArrayList<Field>() {{
        add(Field.newBuilder("test-1", LegacySQLTypeName.INTEGER).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder("test-2", LegacySQLTypeName.STRING).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder(Constants.OFFSET_COLUMN_NAME, LegacySQLTypeName.INTEGER).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder(Constants.TOPIC_COLUMN_NAME, LegacySQLTypeName.STRING).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder(Constants.LOAD_TIME_COLUMN_NAME, LegacySQLTypeName.TIMESTAMP).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder(Constants.TIMESTAMP_COLUMN_NAME, LegacySQLTypeName.TIMESTAMP).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder(Constants.PARTITION_COLUMN_NAME, LegacySQLTypeName.INTEGER).setMode(Field.Mode.NULLABLE).build());
    }};

    TableDefinition tableDefinition = getNonPartitionedTableDefinition(bqSchemaFields);
    ArrayList<Field> updatedBQSchemaFields = new ArrayList<>();
    updatedBQSchemaFields.addAll(bqSchemaFields);
    updatedBQSchemaFields.add(Field.newBuilder("new-field", LegacySQLTypeName.INTEGER).setMode(Field.Mode.NULLABLE).build());
    TableDefinition updatedBQTableDefinition = getNonPartitionedTableDefinition(updatedBQSchemaFields);

    TableId tableId = TableId.of(bqConfig.getDataset(), bqConfig.getTable());
    TableInfo tableInfo = TableInfo.newBuilder(tableId, updatedBQTableDefinition).build();
    when(bigquery.getDataset(tableId.getDataset())).thenReturn(dataset);
    when(dataset.exists()).thenReturn(true);
    when(table.exists()).thenReturn(true);
    when(bigquery.getTable(tableId)).thenReturn(table);
    when(table.getDefinition()).thenReturn(mockTableDefinition);
    when(mockTableDefinition.getSchema()).thenReturn(tableDefinition.getSchema());
    when(bigquery.update(tableInfo)).thenReturn(table);

    bqClient.upsertTable(updatedBQSchemaFields);
    verify(bigquery, never()).create(tableInfo);
    verify(bigquery).update(tableInfo);
}
Example #6
Source File: CreateStore.java    From quetzal with Eclipse Public License 2.0 5 votes vote down vote up 
public Table createTable(String tableName, Field[] fields) {	
	TableId tableId = TableId.of(datasetName, tableName);
	Schema schema = Schema.of(fields);
	TableDefinition tableDefinition = StandardTableDefinition.of(schema);
	TableInfo tableInfo = TableInfo.newBuilder(tableId, tableDefinition).build();
	Table t = bigquery.create(tableInfo);
	System.err.println("created " + t.getTableId());
	return t;
}
Example #7
Source File: BigQueryDatasetRuntimeTestIT.java    From components with Apache License 2.0 5 votes vote down vote up 
@BeforeClass
public static void initDatasetAndTable() throws IOException {
    BigQuery bigquery = BigQueryConnection.createClient(createDatastore());
    for (String dataset : datasets) {
        DatasetId datasetId = DatasetId.of(BigQueryTestConstants.PROJECT, dataset);
        bigquery.create(DatasetInfo.of(datasetId));
    }

    for (String table : tables) {
        TableDefinition tableDefinition =
                StandardTableDefinition.of(Schema.of(Field.of("test", LegacySQLTypeName.STRING)));
        TableId tableId = TableId.of(BigQueryTestConstants.PROJECT, datasets.get(0), table);
        bigquery.create(TableInfo.of(tableId, tableDefinition));
    }
}
Example #8
Source File: BeamBQInputDialog.java    From kettle-beam with Apache License 2.0 5 votes vote down vote up 
public void getFields() {
  try {

    BeamBQInputMeta meta = new BeamBQInputMeta();
    getInfo(meta);

    BigQuery bigQuery = BigQueryOptions.getDefaultInstance().getService();

    if ( StringUtils.isNotEmpty( meta.getDatasetId() ) &&
         StringUtils.isNotEmpty( meta.getTableId() )) {

      Table table = bigQuery.getTable(
        transMeta.environmentSubstitute( meta.getDatasetId()),
        transMeta.environmentSubstitute( meta.getTableId() )
      );

      TableDefinition definition = table.getDefinition();
      Schema schema = definition.getSchema();
      FieldList fieldList = schema.getFields();

      RowMetaInterface rowMeta = new RowMeta();
      for ( int i = 0; i< fieldList.size(); i++) {
        Field field = fieldList.get( i );

        String name = field.getName();
        String type = field.getType().name();

        int kettleType = BQSchemaAndRecordToKettleFn.AvroType.valueOf( type ).getKettleType();
        rowMeta.addValueMeta( ValueMetaFactory.createValueMeta( name, kettleType ) );
      }

      BaseStepDialog.getFieldsFromPrevious( rowMeta, wFields, 1, new int[] { 1 }, new int[] { 3 }, -1, -1, true, null );
    }

  } catch ( Exception e ) {
    new ErrorDialog( shell, "Error", "Error getting BQ fields", e );
  }
}
Example #9
Source File: BQClientTest.java    From beast with Apache License 2.0 5 votes vote down vote up 
private TableDefinition getNonPartitionedTableDefinition(ArrayList<Field> bqSchemaFields) {
    Schema schema = Schema.of(bqSchemaFields);

    TableDefinition tableDefinition = StandardTableDefinition.newBuilder()
            .setSchema(schema)
            .build();
    return tableDefinition;
}
Example #10
Source File: BQClientTest.java    From beast with Apache License 2.0 5 votes vote down vote up 
private TableDefinition getPartitionedTableDefinition(ArrayList<Field> bqSchemaFields) {
    TimePartitioning.Builder timePartitioningBuilder = TimePartitioning.newBuilder(TimePartitioning.Type.DAY);
    timePartitioningBuilder.setField(bqConfig.getBQTablePartitionKey())
            .setRequirePartitionFilter(true);

    Schema schema = Schema.of(bqSchemaFields);

    TableDefinition tableDefinition = StandardTableDefinition.newBuilder()
            .setSchema(schema)
            .setTimePartitioning(timePartitioningBuilder.build())
            .build();
    return tableDefinition;
}
Example #11
Source File: BQClientTest.java    From beast with Apache License 2.0 5 votes vote down vote up 
@Test(expected = BigQueryException.class)
public void shouldThrowExceptionIfUpdateTableFails() {
    when(bqConfig.isBQTablePartitioningEnabled()).thenReturn(false);
    when(bqConfig.getTable()).thenReturn("bq-table");
    when(bqConfig.getDataset()).thenReturn("bq-proto");

    ArrayList<Field> bqSchemaFields = new ArrayList<Field>() {{
        add(Field.newBuilder("test-1", LegacySQLTypeName.INTEGER).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder("test-2", LegacySQLTypeName.STRING).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder(Constants.OFFSET_COLUMN_NAME, LegacySQLTypeName.INTEGER).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder(Constants.TOPIC_COLUMN_NAME, LegacySQLTypeName.STRING).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder(Constants.LOAD_TIME_COLUMN_NAME, LegacySQLTypeName.TIMESTAMP).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder(Constants.TIMESTAMP_COLUMN_NAME, LegacySQLTypeName.TIMESTAMP).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder(Constants.PARTITION_COLUMN_NAME, LegacySQLTypeName.INTEGER).setMode(Field.Mode.NULLABLE).build());
    }};

    TableDefinition tableDefinition = getNonPartitionedTableDefinition(bqSchemaFields);
    ArrayList<Field> updatedBQSchemaFields = new ArrayList<>();
    updatedBQSchemaFields.addAll(bqSchemaFields);
    updatedBQSchemaFields.add(Field.newBuilder("new-field", LegacySQLTypeName.INTEGER).setMode(Field.Mode.NULLABLE).build());
    TableDefinition updatedBQTableDefinition = getNonPartitionedTableDefinition(updatedBQSchemaFields);

    TableId tableId = TableId.of(bqConfig.getDataset(), bqConfig.getTable());
    TableInfo tableInfo = TableInfo.newBuilder(tableId, updatedBQTableDefinition).build();
    when(bigquery.getDataset(tableId.getDataset())).thenReturn(dataset);
    when(dataset.exists()).thenReturn(true);
    when(table.exists()).thenReturn(true);
    when(bigquery.getTable(tableId)).thenReturn(table);
    when(table.getDefinition()).thenReturn(mockTableDefinition);
    when(mockTableDefinition.getSchema()).thenReturn(tableDefinition.getSchema());
    when(bigquery.update(tableInfo)).thenThrow(new BigQueryException(404, "Failed to update"));

    bqClient = new BQClient(bigquery, bqConfig);
    bqClient.upsertTable(updatedBQSchemaFields);
}
Example #12
Source File: BQClientTest.java    From beast with Apache License 2.0 5 votes vote down vote up 
@Test
public void shouldNotUpdateTableIfTableAlreadyExistsWithSameSchema() {
    when(bqConfig.isBQTablePartitioningEnabled()).thenReturn(false);
    when(bqConfig.getTable()).thenReturn("bq-table");
    when(bqConfig.getDataset()).thenReturn("bq-proto");
    bqClient = new BQClient(bigquery, bqConfig);

    ArrayList<Field> bqSchemaFields = new ArrayList<Field>() {{
        add(Field.newBuilder("test-1", LegacySQLTypeName.INTEGER).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder("test-2", LegacySQLTypeName.STRING).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder(Constants.OFFSET_COLUMN_NAME, LegacySQLTypeName.INTEGER).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder(Constants.TOPIC_COLUMN_NAME, LegacySQLTypeName.STRING).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder(Constants.LOAD_TIME_COLUMN_NAME, LegacySQLTypeName.TIMESTAMP).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder(Constants.TIMESTAMP_COLUMN_NAME, LegacySQLTypeName.TIMESTAMP).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder(Constants.PARTITION_COLUMN_NAME, LegacySQLTypeName.INTEGER).setMode(Field.Mode.NULLABLE).build());
    }};

    TableDefinition tableDefinition = getNonPartitionedTableDefinition(bqSchemaFields);

    TableId tableId = TableId.of(bqConfig.getDataset(), bqConfig.getTable());
    TableInfo tableInfo = TableInfo.newBuilder(tableId, tableDefinition).build();
    when(bigquery.getDataset(tableId.getDataset())).thenReturn(dataset);
    when(dataset.exists()).thenReturn(true);
    when(table.exists()).thenReturn(true);
    when(bigquery.getTable(tableId)).thenReturn(table);
    when(table.getDefinition()).thenReturn(mockTableDefinition);
    when(mockTableDefinition.getSchema()).thenReturn(tableDefinition.getSchema());
    when(table.exists()).thenReturn(true);

    bqClient.upsertTable(bqSchemaFields);
    verify(bigquery, never()).create(tableInfo);
    verify(bigquery, never()).update(tableInfo);
}
Example #13
Source File: BQClientTest.java    From beast with Apache License 2.0 5 votes vote down vote up 
@Test
public void shouldCreateBigqueryTableWithoutPartition() {
    when(bqConfig.isBQTablePartitioningEnabled()).thenReturn(false);
    when(bqConfig.getTable()).thenReturn("bq-table");
    when(bqConfig.getDataset()).thenReturn("bq-proto");
    bqClient = new BQClient(bigquery, bqConfig);

    ArrayList<Field> bqSchemaFields = new ArrayList<Field>() {{
        add(Field.newBuilder("test-1", LegacySQLTypeName.INTEGER).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder("test-2", LegacySQLTypeName.STRING).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder(Constants.OFFSET_COLUMN_NAME, LegacySQLTypeName.INTEGER).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder(Constants.TOPIC_COLUMN_NAME, LegacySQLTypeName.STRING).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder(Constants.LOAD_TIME_COLUMN_NAME, LegacySQLTypeName.TIMESTAMP).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder(Constants.TIMESTAMP_COLUMN_NAME, LegacySQLTypeName.TIMESTAMP).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder(Constants.PARTITION_COLUMN_NAME, LegacySQLTypeName.INTEGER).setMode(Field.Mode.NULLABLE).build());
    }};

    TableDefinition tableDefinition = getNonPartitionedTableDefinition(bqSchemaFields);
    TableId tableId = TableId.of(bqConfig.getDataset(), bqConfig.getTable());
    TableInfo tableInfo = TableInfo.newBuilder(tableId, tableDefinition).build();
    when(bigquery.getDataset(tableId.getDataset())).thenReturn(dataset);
    when(dataset.exists()).thenReturn(true);
    when(table.exists()).thenReturn(false);
    when(bigquery.getTable(tableId)).thenReturn(table);
    when(table.exists()).thenReturn(false);
    when(bigquery.create(tableInfo)).thenReturn(table);

    bqClient.upsertTable(bqSchemaFields);

    verify(bigquery).create(tableInfo);
    verify(bigquery, never()).update(tableInfo);
}
Example #14
Source File: BQClientTest.java    From beast with Apache License 2.0 5 votes vote down vote up 
@Test
public void shouldCreateBigqueryTableWithPartition() {
    when(bqConfig.isBQTablePartitioningEnabled()).thenReturn(true);
    when(bqConfig.getBQTablePartitionKey()).thenReturn("partition_column");
    when(bqConfig.getTable()).thenReturn("bq-table");
    when(bqConfig.getDataset()).thenReturn("bq-proto");
    bqClient = new BQClient(bigquery, bqConfig);

    ArrayList<Field> bqSchemaFields = new ArrayList<Field>() {{
        add(Field.newBuilder("test-1", LegacySQLTypeName.INTEGER).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder("partition_column", LegacySQLTypeName.TIMESTAMP).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder(Constants.OFFSET_COLUMN_NAME, LegacySQLTypeName.INTEGER).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder(Constants.TOPIC_COLUMN_NAME, LegacySQLTypeName.STRING).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder(Constants.LOAD_TIME_COLUMN_NAME, LegacySQLTypeName.TIMESTAMP).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder(Constants.TIMESTAMP_COLUMN_NAME, LegacySQLTypeName.TIMESTAMP).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder(Constants.PARTITION_COLUMN_NAME, LegacySQLTypeName.INTEGER).setMode(Field.Mode.NULLABLE).build());
    }};
    TableDefinition tableDefinition = getPartitionedTableDefinition(bqSchemaFields);
    TableId tableId = TableId.of(bqConfig.getDataset(), bqConfig.getTable());
    TableInfo tableInfo = TableInfo.newBuilder(tableId, tableDefinition).build();
    when(bigquery.getDataset(tableId.getDataset())).thenReturn(dataset);
    when(dataset.exists()).thenReturn(true);
    when(table.exists()).thenReturn(false);
    when(bigquery.getTable(tableId)).thenReturn(table);
    when(bigquery.create(tableInfo)).thenReturn(table);

    bqClient.upsertTable(bqSchemaFields);
    verify(bigquery).create(tableInfo);
    verify(bigquery, never()).update(tableInfo);
}
Example #15
Source File: BQClientTest.java    From beast with Apache License 2.0 5 votes vote down vote up 
@Test
public void shouldIgnoreExceptionIfDatasetAlreadyExists() {
    when(bqConfig.isBQTablePartitioningEnabled()).thenReturn(true);
    when(bqConfig.getBQTablePartitionKey()).thenReturn("partition_column");
    when(bqConfig.getTable()).thenReturn("bq-table");
    when(bqConfig.getDataset()).thenReturn("bq-proto");
    bqClient = new BQClient(bigquery, bqConfig);

    ArrayList<Field> bqSchemaFields = new ArrayList<Field>() {{
        add(Field.newBuilder("test-1", LegacySQLTypeName.INTEGER).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder("partition_column", LegacySQLTypeName.TIMESTAMP).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder(Constants.OFFSET_COLUMN_NAME, LegacySQLTypeName.INTEGER).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder(Constants.TOPIC_COLUMN_NAME, LegacySQLTypeName.STRING).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder(Constants.LOAD_TIME_COLUMN_NAME, LegacySQLTypeName.TIMESTAMP).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder(Constants.TIMESTAMP_COLUMN_NAME, LegacySQLTypeName.TIMESTAMP).setMode(Field.Mode.NULLABLE).build());
        add(Field.newBuilder(Constants.PARTITION_COLUMN_NAME, LegacySQLTypeName.INTEGER).setMode(Field.Mode.NULLABLE).build());
    }};

    TableDefinition tableDefinition = getPartitionedTableDefinition(bqSchemaFields);
    TableId tableId = TableId.of(bqConfig.getDataset(), bqConfig.getTable());
    TableInfo tableInfo = TableInfo.newBuilder(tableId, tableDefinition).build();

    when(bigquery.getDataset(tableId.getDataset())).thenReturn(dataset);
    when(dataset.exists()).thenReturn(false);
    when(table.exists()).thenReturn(false);
    when(bigquery.getTable(tableId)).thenReturn(table);
    when(bigquery.create(tableInfo)).thenReturn(table);

    bqClient.upsertTable(bqSchemaFields);
    verify(bigquery).create(DatasetInfo.of(tableId.getDataset()));
    verify(bigquery).create(tableInfo);
    verify(bigquery, never()).update(tableInfo);
}
Example #16
Source File: BQClient.java    From beast with Apache License 2.0 5 votes vote down vote up 
public void upsertTable(List<Field> bqSchemaFields) throws BigQueryException {
    Schema schema = Schema.of(bqSchemaFields);
    TableDefinition tableDefinition = getTableDefinition(schema);
    TableInfo tableInfo = TableInfo.newBuilder(tableID, tableDefinition)
            .setLabels(bqConfig.getTableLabels())
            .build();
    upsertDatasetAndTable(tableInfo);
}
Example #17
Source File: ReadSessionCreator.java    From presto with Apache License 2.0 5 votes vote down vote up 
private TableInfo getActualTable(
        TableInfo table,
        List<String> requiredColumns)
{
    TableDefinition tableDefinition = table.getDefinition();
    TableDefinition.Type tableType = tableDefinition.getType();
    if (TableDefinition.Type.TABLE == tableType) {
        return table;
    }
    if (TableDefinition.Type.VIEW == tableType) {
        if (!config.viewsEnabled) {
            throw new PrestoException(NOT_SUPPORTED, format(
                    "Views are not enabled. You can enable views by setting '%s' to true. Notice additional cost may occur.",
                    BigQueryConfig.VIEWS_ENABLED));
        }
        // get it from the view
        String query = bigQueryClient.selectSql(table.getTableId(), requiredColumns);
        log.debug("query is %s", query);
        try {
            return destinationTableCache.get(query, new DestinationTableBuilder(bigQueryClient, config, query, table.getTableId()));
        }
        catch (ExecutionException e) {
            throw new PrestoException(BIGQUERY_VIEW_DESTINATION_TABLE_CREATION_FAILED, "Error creating destination table", e);
        }
    }
    else {
        // not regular table or a view
        throw new PrestoException(NOT_SUPPORTED, format("Table type '%s' of table '%s.%s' is not supported",
                tableType, table.getTableId().getDataset(), table.getTableId().getTable()));
    }
}
Example #18
Source File: BigQueryClient.java    From presto with Apache License 2.0 5 votes vote down vote up 
Iterable<Table> listTables(DatasetId datasetId, TableDefinition.Type... types)
{
    Set<TableDefinition.Type> allowedTypes = ImmutableSet.copyOf(types);
    DatasetId bigQueryDatasetId = datasetIds.getOrDefault(datasetId, datasetId);
    Iterable<Table> allTables = bigQuery.listTables(bigQueryDatasetId).iterateAll();
    return StreamSupport.stream(allTables.spliterator(), false)
            .filter(table -> allowedTypes.contains(table.getDefinition().getType()))
            .collect(toImmutableList());
}
Example #19
Source File: BigQueryMetadata.java    From presto with Apache License 2.0 5 votes vote down vote up 
private List<BigQueryColumnHandle> getTableColumns(TableInfo table)
{
    ImmutableList.Builder<BigQueryColumnHandle> columns = ImmutableList.builder();
    TableDefinition tableDefinition = table.getDefinition();
    Schema schema = tableDefinition.getSchema();
    if (schema != null) {
        for (Field field : schema.getFields()) {
            columns.add(Conversions.toColumnHandle(field));
        }
    }
    return columns.build();
}
Example #20
Source File: BQClient.java    From beast with Apache License 2.0 4 votes vote down vote up 
private TableDefinition getTableDefinition(Schema schema) throws BQPartitionKeyNotSpecified {
    return bqTableDefinition.getTableDefinition(schema);
}