Java Code Examples for org.apache.avro.Schema.Type#LONG

private Type toAvroType(String columnName, int sqlType) {
  Properties mapping = options.getMapColumnJava();

  if (mapping.containsKey(columnName)) {
    String type = mapping.getProperty(columnName);
    if (LOG.isDebugEnabled()) {
      LOG.info("Overriding type of column " + columnName + " to " + type);
    }

    if (type.equalsIgnoreCase("INTEGER")) { return Type.INT; }
    if (type.equalsIgnoreCase("LONG")) { return Type.LONG; }
    if (type.equalsIgnoreCase("BOOLEAN")) { return Type.BOOLEAN; }
    if (type.equalsIgnoreCase("FLOAT")) { return Type.FLOAT; }
    if (type.equalsIgnoreCase("DOUBLE")) { return Type.DOUBLE; }
    if (type.equalsIgnoreCase("STRING")) { return Type.STRING; }
    if (type.equalsIgnoreCase("BYTES")) { return Type.BYTES; }

    // Mapping was not found
    throw new IllegalArgumentException("Cannot convert to AVRO type " + type);
  }

  return connManager.toAvroType(tableName, columnName, sqlType);
}
 

/**
 * Returns an Avro Decoder. The implementation it chooses will depend on the
 * schema of the field.
 * 
 * @param in
 *          InputStream to decode bytes from
 * @return The avro decoder.
 */
private Decoder getColumnDecoder(Schema writtenFieldAvroSchema, InputStream in) {
  // Use a special Avro decoder that has special handling for int, long,
  // and String types. See ColumnDecoder for more information.
  if (writtenFieldAvroSchema.getType() == Type.INT
      || writtenFieldAvroSchema.getType() == Type.LONG
      || writtenFieldAvroSchema.getType() == Type.STRING) {
    return new ColumnDecoder(in);
  } else {
    return DecoderFactory.get().binaryDecoder(in, null);
  }
}
 

/**
 * Returns an Avro Encoder. The implementation it chooses will depend on the
 * schema of the field.
 * 
 * @param out
 *          Output stream to encode bytes to
 * @return The avro encoder
 */
private Encoder getColumnEncoder(Schema fieldAvroSchema, OutputStream out) {
  // Use a special Avro encoder that has special handling for int, long,
  // and String types. See ColumnEncoder for more information.
  if (fieldAvroSchema.getType() == Type.INT
      || fieldAvroSchema.getType() == Type.LONG
      || fieldAvroSchema.getType() == Type.STRING) {
    return new ColumnEncoder(out);
  } else {
    return EncoderFactory.get().binaryEncoder(out, null);
  }
}
 

/**
 *
 */
private static void updateRecord(Field field, Type type, String providedValue, Record avroRecord) {
    if (Type.NULL != type) {
        Object value;
        if (Type.INT == type) {
            value = null == providedValue ? possiblyGetDefaultValue(field, IntNode.class).getIntValue()
                    : Integer.parseInt(providedValue);
            avroRecord.put(field.name(), value);
        } else if (Type.BOOLEAN == type) {
            value = null == providedValue
                    ? possiblyGetDefaultValue(field, BooleanNode.class).getBooleanValue()
                    : Boolean.parseBoolean(providedValue);
            avroRecord.put(field.name(), value);
        } else if (Type.DOUBLE == type) {
            value = null == providedValue ? possiblyGetDefaultValue(field, DoubleNode.class).getDoubleValue()
                    : Double.parseDouble(providedValue);
            avroRecord.put(field.name(), value);
        } else if (Type.FLOAT == type) {
            value = null == providedValue ? possiblyGetDefaultValue(field, DoubleNode.class).getDoubleValue()
                    : Float.parseFloat(providedValue);
            avroRecord.put(field.name(), value);
        } else if (Type.LONG == type) {
            value = null == providedValue ? possiblyGetDefaultValue(field, LongNode.class).getLongValue()
                    : Long.parseLong(providedValue);
            avroRecord.put(field.name(), value);
        } else if (Type.STRING == type) {
            value = null == providedValue ? possiblyGetDefaultValue(field, TextNode.class).getTextValue()
                    : providedValue;
            avroRecord.put(field.name(), value);
        } else if (Type.BYTES == type) {
            value = encodeLogicalType(field, providedValue);
            avroRecord.put(field.name(), value);
        } else if (Type.UNION == type) {
            field.schema().getTypes()
                    .forEach(schema -> updateRecord(field, schema.getType(), providedValue, avroRecord));
        } else if (Type.ARRAY == type || Type.ENUM == type || Type.FIXED == type || Type.MAP == type
                || Type.NULL == type || Type.RECORD == type) {
            throw new IllegalArgumentException("The field type '" + type + "' is not supported at the moment");
        } else {
            avroRecord.put(field.name(), providedValue);
        }
    }
}
 

/**
 * Resolve a database-specific type to Avro data type.
 * @param sqlType     sql type
 * @return            avro type
 */
public Type toAvroType(int sqlType) {
  switch (sqlType) {
  case Types.TINYINT:
  case Types.SMALLINT:
  case Types.INTEGER:
    return Type.INT;
  case Types.BIGINT:
    return Type.LONG;
  case Types.BIT:
  case Types.BOOLEAN:
    return Type.BOOLEAN;
  case Types.REAL:
    return Type.FLOAT;
  case Types.FLOAT:
  case Types.DOUBLE:
    return Type.DOUBLE;
  case Types.NUMERIC:
  case Types.DECIMAL:
    return Type.STRING;
  case Types.CHAR:
  case Types.VARCHAR:
  case Types.LONGVARCHAR:
  case Types.LONGNVARCHAR:
  case Types.NVARCHAR:
  case Types.NCHAR:
    return Type.STRING;
  case Types.DATE:
  case Types.TIME:
  case Types.TIMESTAMP:
    return Type.LONG;
  case Types.BLOB:
  case Types.BINARY:
  case Types.VARBINARY:
  case Types.LONGVARBINARY:
    return Type.BYTES;
  default:
    throw new IllegalArgumentException("Cannot convert SQL type "
        + sqlType);
  }
}
 

private static Object convertRequiredField(
    Type avroType, LogicalType avroLogicalType, TableFieldSchema fieldSchema, Object v) {
  // REQUIRED fields are represented as the corresponding Avro types. For example, a BigQuery
  // INTEGER type maps to an Avro LONG type.
  checkNotNull(v, "REQUIRED field %s should not be null", fieldSchema.getName());
  // Per https://cloud.google.com/bigquery/docs/reference/v2/tables#schema, the type field
  // is required, so it may not be null.
  String bqType = fieldSchema.getType();
  ImmutableCollection<Type> expectedAvroTypes = BIG_QUERY_TO_AVRO_TYPES.get(bqType);
  verifyNotNull(expectedAvroTypes, "Unsupported BigQuery type: %s", bqType);
  verify(
      expectedAvroTypes.contains(avroType),
      "Expected Avro schema types %s for BigQuery %s field %s, but received %s",
      expectedAvroTypes,
      bqType,
      fieldSchema.getName(),
      avroType);
  // For historical reasons, don't validate avroLogicalType except for with NUMERIC.
  // BigQuery represents NUMERIC in Avro format as BYTES with a DECIMAL logical type.
  switch (bqType) {
    case "STRING":
    case "DATETIME":
    case "GEOGRAPHY":
      // Avro will use a CharSequence to represent String objects, but it may not always use
      // java.lang.String; for example, it may prefer org.apache.avro.util.Utf8.
      verify(v instanceof CharSequence, "Expected CharSequence (String), got %s", v.getClass());
      return v.toString();
    case "DATE":
      if (avroType == Type.INT) {
        verify(v instanceof Integer, "Expected Integer, got %s", v.getClass());
        verifyNotNull(avroLogicalType, "Expected Date logical type");
        verify(avroLogicalType instanceof LogicalTypes.Date, "Expected Date logical type");
        return formatDate((Integer) v);
      } else {
        verify(v instanceof CharSequence, "Expected CharSequence (String), got %s", v.getClass());
        return v.toString();
      }
    case "TIME":
      if (avroType == Type.LONG) {
        verify(v instanceof Long, "Expected Long, got %s", v.getClass());
        verifyNotNull(avroLogicalType, "Expected TimeMicros logical type");
        verify(
            avroLogicalType instanceof LogicalTypes.TimeMicros,
            "Expected TimeMicros logical type");
        return formatTime((Long) v);
      } else {
        verify(v instanceof CharSequence, "Expected CharSequence (String), got %s", v.getClass());
        return v.toString();
      }
    case "INTEGER":
      verify(v instanceof Long, "Expected Long, got %s", v.getClass());
      return ((Long) v).toString();
    case "FLOAT":
      verify(v instanceof Double, "Expected Double, got %s", v.getClass());
      return v;
    case "NUMERIC":
      // NUMERIC data types are represented as BYTES with the DECIMAL logical type. They are
      // converted back to Strings with precision and scale determined by the logical type.
      verify(v instanceof ByteBuffer, "Expected ByteBuffer, got %s", v.getClass());
      verifyNotNull(avroLogicalType, "Expected Decimal logical type");
      verify(avroLogicalType instanceof LogicalTypes.Decimal, "Expected Decimal logical type");
      BigDecimal numericValue =
          new Conversions.DecimalConversion()
              .fromBytes((ByteBuffer) v, Schema.create(avroType), avroLogicalType);
      return numericValue.toString();
    case "BOOLEAN":
      verify(v instanceof Boolean, "Expected Boolean, got %s", v.getClass());
      return v;
    case "TIMESTAMP":
      // TIMESTAMP data types are represented as Avro LONG types, microseconds since the epoch.
      // Values may be negative since BigQuery timestamps start at 0001-01-01 00:00:00 UTC.
      verify(v instanceof Long, "Expected Long, got %s", v.getClass());
      return formatTimestamp((Long) v);
    case "RECORD":
      verify(v instanceof GenericRecord, "Expected GenericRecord, got %s", v.getClass());
      return convertGenericRecordToTableRow((GenericRecord) v, fieldSchema.getFields());
    case "BYTES":
      verify(v instanceof ByteBuffer, "Expected ByteBuffer, got %s", v.getClass());
      ByteBuffer byteBuffer = (ByteBuffer) v;
      byte[] bytes = new byte[byteBuffer.limit()];
      byteBuffer.get(bytes);
      return BaseEncoding.base64().encode(bytes);
    default:
      throw new UnsupportedOperationException(
          String.format(
              "Unexpected BigQuery field schema type %s for field named %s",
              fieldSchema.getType(), fieldSchema.getName()));
  }
}