Java Code Examples for com.google.api.services.bigquery.model.TableFieldSchema#getType()
The following examples show how to use
com.google.api.services.bigquery.model.TableFieldSchema#getType() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: BigQueryAvroUtils.java From beam with Apache License 2.0 | 5 votes |
private static Field convertField(TableFieldSchema bigQueryField) { ImmutableCollection<Type> avroTypes = BIG_QUERY_TO_AVRO_TYPES.get(bigQueryField.getType()); if (avroTypes.isEmpty()) { throw new IllegalArgumentException( "Unable to map BigQuery field type " + bigQueryField.getType() + " to avro type."); } Type avroType = avroTypes.iterator().next(); Schema elementSchema; if (avroType == Type.RECORD) { elementSchema = toGenericAvroSchema(bigQueryField.getName(), bigQueryField.getFields()); } else { elementSchema = Schema.create(avroType); } Schema fieldSchema; if (bigQueryField.getMode() == null || "NULLABLE".equals(bigQueryField.getMode())) { fieldSchema = Schema.createUnion(Schema.create(Type.NULL), elementSchema); } else if ("REQUIRED".equals(bigQueryField.getMode())) { fieldSchema = elementSchema; } else if ("REPEATED".equals(bigQueryField.getMode())) { fieldSchema = Schema.createArray(elementSchema); } else { throw new IllegalArgumentException( String.format("Unknown BigQuery Field Mode: %s", bigQueryField.getMode())); } return new Field( bigQueryField.getName(), fieldSchema, bigQueryField.getDescription(), (Object) null /* Cast to avoid deprecated JsonNode constructor. */); }
Example 2
Source File: BigQueryConverters.java From DataflowTemplates with Apache License 2.0 | 4 votes |
/** * Converts the value of a BigQuery column in Avro format into the value of a Datastore Entity * property. */ public static Value columnToValue(TableFieldSchema column, Object columnValue) throws IllegalArgumentException { String columnName = column.getName(); if (columnValue == null) { return Value.newBuilder().setNullValue(NullValue.NULL_VALUE).build(); } else { Value.Builder valueBuilder = Value.newBuilder(); switch (column.getType()) { case "STRING": // Datastore string properties greater than 1500 bytes will not be indexed in order to // respect the limit imposed on the maximum size of index-able string properties. See // https://cloud.google.com/datastore/docs/concepts/limits String strValue = columnValue.toString(); valueBuilder.setStringValue(strValue); boolean excludeFromIndexes = strValue.getBytes().length > MAX_STRING_SIZE_BYTES; valueBuilder.setExcludeFromIndexes(excludeFromIndexes); break; case "INTEGER": case "INT64": valueBuilder.setIntegerValue((long) columnValue); break; case "FLOAT": case "FLOAT64": valueBuilder.setDoubleValue((double) columnValue); break; case "BOOLEAN": case "BOOL": valueBuilder.setBooleanValue((boolean) columnValue); break; case "TIMESTAMP": // Convert into milliseconds from the BigQuery timestamp, which is in micro seconds long timeInMillis = ((long) columnValue) / 1000; valueBuilder.setTimestampValue(Timestamps.fromMillis(timeInMillis)); break; case "DATE": case "TIME": case "DATETIME": // Handle these types as STRING by default, no dedicated type exists in Datastore valueBuilder.setStringValue(columnValue.toString()); break; default: // TODO: handle nested fields (types "RECORD" or "STRUCT") throw new IllegalArgumentException( String.format( "Column [%s] of type [%s] not supported.", column.getName(), column.getType())); } return valueBuilder.build(); } }
Example 3
Source File: BigQueryAvroUtils.java From beam with Apache License 2.0 | 4 votes |
private static Object convertRequiredField( Type avroType, LogicalType avroLogicalType, TableFieldSchema fieldSchema, Object v) { // REQUIRED fields are represented as the corresponding Avro types. For example, a BigQuery // INTEGER type maps to an Avro LONG type. checkNotNull(v, "REQUIRED field %s should not be null", fieldSchema.getName()); // Per https://cloud.google.com/bigquery/docs/reference/v2/tables#schema, the type field // is required, so it may not be null. String bqType = fieldSchema.getType(); ImmutableCollection<Type> expectedAvroTypes = BIG_QUERY_TO_AVRO_TYPES.get(bqType); verifyNotNull(expectedAvroTypes, "Unsupported BigQuery type: %s", bqType); verify( expectedAvroTypes.contains(avroType), "Expected Avro schema types %s for BigQuery %s field %s, but received %s", expectedAvroTypes, bqType, fieldSchema.getName(), avroType); // For historical reasons, don't validate avroLogicalType except for with NUMERIC. // BigQuery represents NUMERIC in Avro format as BYTES with a DECIMAL logical type. switch (bqType) { case "STRING": case "DATETIME": case "GEOGRAPHY": // Avro will use a CharSequence to represent String objects, but it may not always use // java.lang.String; for example, it may prefer org.apache.avro.util.Utf8. verify(v instanceof CharSequence, "Expected CharSequence (String), got %s", v.getClass()); return v.toString(); case "DATE": if (avroType == Type.INT) { verify(v instanceof Integer, "Expected Integer, got %s", v.getClass()); verifyNotNull(avroLogicalType, "Expected Date logical type"); verify(avroLogicalType instanceof LogicalTypes.Date, "Expected Date logical type"); return formatDate((Integer) v); } else { verify(v instanceof CharSequence, "Expected CharSequence (String), got %s", v.getClass()); return v.toString(); } case "TIME": if (avroType == Type.LONG) { verify(v instanceof Long, "Expected Long, got %s", v.getClass()); verifyNotNull(avroLogicalType, "Expected TimeMicros logical type"); verify( avroLogicalType instanceof LogicalTypes.TimeMicros, "Expected TimeMicros logical type"); return formatTime((Long) v); } else { verify(v instanceof CharSequence, "Expected CharSequence (String), got %s", v.getClass()); return v.toString(); } case "INTEGER": verify(v instanceof Long, "Expected Long, got %s", v.getClass()); return ((Long) v).toString(); case "FLOAT": verify(v instanceof Double, "Expected Double, got %s", v.getClass()); return v; case "NUMERIC": // NUMERIC data types are represented as BYTES with the DECIMAL logical type. They are // converted back to Strings with precision and scale determined by the logical type. verify(v instanceof ByteBuffer, "Expected ByteBuffer, got %s", v.getClass()); verifyNotNull(avroLogicalType, "Expected Decimal logical type"); verify(avroLogicalType instanceof LogicalTypes.Decimal, "Expected Decimal logical type"); BigDecimal numericValue = new Conversions.DecimalConversion() .fromBytes((ByteBuffer) v, Schema.create(avroType), avroLogicalType); return numericValue.toString(); case "BOOLEAN": verify(v instanceof Boolean, "Expected Boolean, got %s", v.getClass()); return v; case "TIMESTAMP": // TIMESTAMP data types are represented as Avro LONG types, microseconds since the epoch. // Values may be negative since BigQuery timestamps start at 0001-01-01 00:00:00 UTC. verify(v instanceof Long, "Expected Long, got %s", v.getClass()); return formatTimestamp((Long) v); case "RECORD": verify(v instanceof GenericRecord, "Expected GenericRecord, got %s", v.getClass()); return convertGenericRecordToTableRow((GenericRecord) v, fieldSchema.getFields()); case "BYTES": verify(v instanceof ByteBuffer, "Expected ByteBuffer, got %s", v.getClass()); ByteBuffer byteBuffer = (ByteBuffer) v; byte[] bytes = new byte[byteBuffer.limit()]; byteBuffer.get(bytes); return BaseEncoding.base64().encode(bytes); default: throw new UnsupportedOperationException( String.format( "Unexpected BigQuery field schema type %s for field named %s", fieldSchema.getType(), fieldSchema.getName())); } }