Java Code Examples for com.google.api.services.bigquery.model.TableFieldSchema#setFields()

The following examples show how to use com.google.api.services.bigquery.model.TableFieldSchema#setFields() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: LoadReadsToBigQuery.java    From dataflow-java with Apache License 2.0 6 votes vote down vote up 
static TableFieldSchema getAlignmentFieldSchema(String fieldName) {
  TableFieldSchema fieldSchema =
      new TableFieldSchema().setName(fieldName).setType(RECORD);
  fieldSchema.setFields(new ArrayList<TableFieldSchema>() {
    private static final long serialVersionUID = 0;
    {
      // Position record.
      add(getPositionFieldSchema(POSITION));
      add(new TableFieldSchema().setName(MAPPING_QUALITY).setType(INT64));
      // CIGAR record.
      TableFieldSchema cigarSchema =
          new TableFieldSchema().setName(CIGAR).setType(RECORD).setMode(REPEATED);
      cigarSchema.setFields(new ArrayList<TableFieldSchema>() {
        private static final long serialVersionUID = 0;
        {
          // Operation is of type enum, but BQ doesn't accept enum. Hence, converting to INT64.
          add(new TableFieldSchema().setName(OPERATION).setType(INT64));
          add(new TableFieldSchema().setName(OPERATION_LENGTH).setType(INT64));
          add(new TableFieldSchema().setName(REFERENCE_SEQUENCE).setType(STRING));
        }
      });
      add(cigarSchema);
    }
  });
  return fieldSchema;
}
Example 2
Source File: FieldSchemaListBuilder.java    From bigquery-etl-dataflow-sample with Apache License 2.0 5 votes vote down vote up 
/**
 * Creates a record TableSchemaField out of the given list and returns it so that it may be added to another
 * FieldSchemaListBuilder
 */
public TableFieldSchema fieldSchema(FieldSchemaListBuilder list) {
  TableFieldSchema tfs = new TableFieldSchema();
  tfs.setType("RECORD");
  tfs.setFields(list.schemaFields);
  return tfs;
}
Example 3
Source File: BigQueryUtils.java    From beam with Apache License 2.0 5 votes vote down vote up 
private static List<TableFieldSchema> toTableFieldSchema(Schema schema) {
  List<TableFieldSchema> fields = new ArrayList<>(schema.getFieldCount());
  for (Field schemaField : schema.getFields()) {
    FieldType type = schemaField.getType();

    TableFieldSchema field = new TableFieldSchema().setName(schemaField.getName());
    if (schemaField.getDescription() != null && !"".equals(schemaField.getDescription())) {
      field.setDescription(schemaField.getDescription());
    }

    if (!schemaField.getType().getNullable()) {
      field.setMode(Mode.REQUIRED.toString());
    }
    if (type.getTypeName().isCollectionType()) {
      type = type.getCollectionElementType();
      if (type.getTypeName().isCollectionType() || type.getTypeName().isMapType()) {
        throw new IllegalArgumentException("Array of collection is not supported in BigQuery.");
      }
      field.setMode(Mode.REPEATED.toString());
    }
    if (TypeName.ROW == type.getTypeName()) {
      Schema subType = type.getRowSchema();
      field.setFields(toTableFieldSchema(subType));
    }
    if (TypeName.MAP == type.getTypeName()) {
      throw new IllegalArgumentException("Maps are not supported in BigQuery.");
    }
    field.setType(toStandardSQLTypeName(type).toString());

    fields.add(field);
  }
  return fields;
}
Example 4
Source File: BigQueryAvroRegistry.java    From components with Apache License 2.0 5 votes vote down vote up 
private TableFieldSchema tryFieldSchema(TableFieldSchema fieldSchema, org.apache.avro.Schema avroSchema) {
    fieldSchema = fieldSchema.setType(getBQFieldType(avroSchema));

    if (avroSchema.getType() == org.apache.avro.Schema.Type.RECORD) {
        List<TableFieldSchema> childFields = new ArrayList<>();
        List<org.apache.avro.Schema.Field> avroChildFields = avroSchema.getFields();
        for (org.apache.avro.Schema.Field avroChildField : avroChildFields) {
            childFields.add(tryArrayFieldSchema(avroChildField));
        }
        fieldSchema.setFields(childFields);
    }
    return fieldSchema;
}
Example 5
Source File: LoadReadsToBigQuery.java    From dataflow-java with Apache License 2.0 5 votes vote down vote up 
static TableFieldSchema getPositionFieldSchema(String fieldName) {
  TableFieldSchema fieldSchema =
      new TableFieldSchema().setName(fieldName).setType(RECORD);
  fieldSchema.setFields(new ArrayList<TableFieldSchema>() {
    private static final long serialVersionUID = 0;
    {
      add(new TableFieldSchema().setName(REFERENCE_NAME).setType(STRING));
      add(new TableFieldSchema().setName(POSITION).setType(INT64));
      add(new TableFieldSchema().setName(REVERSE_STRAND).setType(BOOL));
    }
  });
  return fieldSchema;
}
Example 6
Source File: BigQueryUtils.java    From hadoop-connectors with Apache License 2.0 4 votes vote down vote up 
/**
 * Parses the given JSON string and returns the extracted schema.
 *
 * @param fields a string to read the TableSchema from.
 * @return the List of TableFieldSchema described by the string fields.
 */
public static List<TableFieldSchema> getSchemaFromString(String fields) {
  logger.atFine().log("getSchemaFromString('%s')", fields);

  // Parse the output schema for Json from fields.
  JsonParser jsonParser = new JsonParser();
  JsonArray json = jsonParser.parse(fields).getAsJsonArray();
  List<TableFieldSchema> fieldsList = new ArrayList<>();

  // For each item in the list of fields.
  for (JsonElement jsonElement : json) {
    checkArgument(
        jsonElement.isJsonObject(), "Expected JsonObject for element, got '%s'.", jsonElement);
    JsonObject jsonObject = jsonElement.getAsJsonObject();

    // Set the name and type.
    checkArgument(
        jsonObject.get("name") != null,
        "Expected non-null entry for key 'name' in JsonObject '%s'", jsonObject);
    checkArgument(
        jsonObject.get("type") != null,
        "Expected non-null entry for key 'type' in JsonObject '%s'", jsonObject);
    TableFieldSchema fieldDef = new TableFieldSchema();
    fieldDef.setName(jsonObject.get("name").getAsString());
    fieldDef.setType(jsonObject.get("type").getAsString());

    // If mode is not null, set mode.
    if (jsonObject.get("mode") != null) {
      fieldDef.setMode(jsonObject.get("mode").getAsString());
    }

    // If the type is RECORD set the fields.
    if (jsonObject.get("type").getAsString().equals("RECORD")) {
      checkArgument(
          jsonObject.get("fields") != null,
          "Expected non-null entry for key 'fields' in JsonObject of type RECORD: '%s'",
          jsonObject);
      fieldDef.setFields(getSchemaFromString(jsonObject.get("fields").toString()));
    }

    fieldsList.add(fieldDef);
  }
  // Return list of TableFieldSchema.
  return fieldsList;
}