Java Code Examples for org.apache.iceberg.types.Type#typeId()

@Override
@SuppressWarnings("unchecked")
public <T> Literal<T> to(Type type) {
  switch (type.typeId()) {
    case FIXED:
      Types.FixedType fixed = (Types.FixedType) type;
      if (value().remaining() == fixed.length()) {
        return (Literal<T>) new FixedLiteral(value());
      }
      return null;
    case BINARY:
      return (Literal<T>) this;
    default:
      return null;
  }
}
 

@Override
@SuppressWarnings("unchecked")
public <T> Literal<T> to(Type type) {
  switch (type.typeId()) {
    case FLOAT:
      return (Literal<T>) this;
    case DOUBLE:
      return (Literal<T>) new DoubleLiteral(value().doubleValue());
    case DECIMAL:
      int scale = ((Types.DecimalType) type).scale();
      return (Literal<T>) new DecimalLiteral(
          BigDecimal.valueOf(value()).setScale(scale, RoundingMode.HALF_UP));
    default:
      return null;
  }
}
 

private static Function<Object, Object> converter(Type type) {
  switch (type.typeId()) {
    case DATE:
      return date -> DateTimeUtil.daysFromDate((LocalDate) date);
    case TIME:
      return time -> DateTimeUtil.microsFromTime((LocalTime) time);
    case TIMESTAMP:
      if (((Types.TimestampType) type).shouldAdjustToUTC()) {
        return timestamp -> DateTimeUtil.microsFromTimestamptz((OffsetDateTime) timestamp);
      } else {
        return timestamp -> DateTimeUtil.microsFromTimestamp((LocalDateTime) timestamp);
      }
    case FIXED:
      return bytes -> ByteBuffer.wrap((byte[]) bytes);
    case STRUCT:
      InternalRecordWrapper wrapper = new InternalRecordWrapper(type.asStructType());
      return struct -> wrapper.wrap((StructLike) struct);
    default:
  }
  return null;
}
 

@SuppressWarnings("unchecked")
static <T> Bucket<T> get(Type type, int numBuckets) {
  switch (type.typeId()) {
    case DATE:
    case INTEGER:
      return (Bucket<T>) new BucketInteger(numBuckets);
    case TIME:
    case TIMESTAMP:
    case LONG:
      return (Bucket<T>) new BucketLong(numBuckets);
    case DECIMAL:
      return (Bucket<T>) new BucketDecimal(numBuckets);
    case STRING:
      return (Bucket<T>) new BucketString(numBuckets);
    case FIXED:
    case BINARY:
      return (Bucket<T>) new BucketByteBuffer(numBuckets);
    case UUID:
      return (Bucket<T>) new BucketUUID(numBuckets);
    default:
      throw new IllegalArgumentException("Cannot bucket by type: " + type);
  }
}
 

@Override
public boolean canTransform(Type type) {
  return type.typeId() == TypeID.LONG ||
      type.typeId() == TypeID.TIME ||
      type.typeId() == TypeID.TIMESTAMP;

}
 

public static Transform<?, ?> fromString(Type type, String transform) {
  Matcher widthMatcher = HAS_WIDTH.matcher(transform);
  if (widthMatcher.matches()) {
    String name = widthMatcher.group(1);
    int parsedWidth = Integer.parseInt(widthMatcher.group(2));
    if (name.equalsIgnoreCase("truncate")) {
      return Truncate.get(type, parsedWidth);
    } else if (name.equals("bucket")) {
      return Bucket.get(type, parsedWidth);
    }
  }

  if (transform.equalsIgnoreCase("identity")) {
    return Identity.get(type);
  }

  try {
    if (type.typeId() == Type.TypeID.TIMESTAMP) {
      return Timestamps.valueOf(transform.toUpperCase(Locale.ENGLISH));
    } else if (type.typeId() == Type.TypeID.DATE) {
      return Dates.valueOf(transform.toUpperCase(Locale.ENGLISH));
    }
  } catch (IllegalArgumentException ignored) {
    // fall through to return unknown transform
  }

  if (transform.equalsIgnoreCase("void")) {
    return VoidTransform.get();
  }

  return new UnknownTransform<>(type, transform);
}
 

private Object convertPartitionValue(Type type, Object value) {
  if (type.typeId() == Types.BinaryType.get().typeId()) {
    ByteBuffer buffer = (ByteBuffer) value;
    return new DataByteArray(buffer.get(new byte[buffer.remaining()]).array());
  }

  return value;
}
 

private static Object convertConstant(Type type, Object value) {
  if (value == null) {
    return null;
  }

  switch (type.typeId()) {
    case DECIMAL:
      return Decimal.apply((BigDecimal) value);
    case STRING:
      if (value instanceof Utf8) {
        Utf8 utf8 = (Utf8) value;
        return UTF8String.fromBytes(utf8.getBytes(), 0, utf8.getByteLength());
      }
      return UTF8String.fromString(value.toString());
    case FIXED:
      if (value instanceof byte[]) {
        return value;
      } else if (value instanceof GenericData.Fixed) {
        return ((GenericData.Fixed) value).bytes();
      }
      return ByteBuffers.toByteArray((ByteBuffer) value);
    case BINARY:
      return ByteBuffers.toByteArray((ByteBuffer) value);
    default:
  }
  return value;
}
 

@Override
@SuppressWarnings("unchecked")
public <T> Literal<T> to(Type type) {
  if (type.typeId() == Type.TypeID.TIME) {
    return (Literal<T>) this;
  }
  return null;
}
 

public static Object getValue(JsonNode partitionValue, Type type)
{
    if (partitionValue.isNull()) {
        return null;
    }
    switch (type.typeId()) {
        case BOOLEAN:
            return partitionValue.asBoolean();
        case INTEGER:
        case DATE:
            return partitionValue.asInt();
        case LONG:
        case TIMESTAMP:
            return partitionValue.asLong();
        case FLOAT:
            return partitionValue.floatValue();
        case DOUBLE:
            return partitionValue.doubleValue();
        case STRING:
        case UUID:
            return partitionValue.asText();
        case FIXED:
        case BINARY:
            try {
                return partitionValue.binaryValue();
            }
            catch (IOException e) {
                throw new UncheckedIOException("Failed during JSON conversion of " + partitionValue, e);
            }
        case DECIMAL:
            return partitionValue.decimalValue();
    }
    throw new UnsupportedOperationException("Type not supported as partition column: " + type);
}
 

/**
 * Conversions from internal representations to Iceberg generic values.
 */
public static Object convertConstant(Type type, Object value) {
  if (value == null) {
    return null;
  }

  switch (type.typeId()) {
    case STRING:
      return value.toString();
    case TIME:
      return DateTimeUtil.timeFromMicros((Long) value);
    case DATE:
      return DateTimeUtil.dateFromDays((Integer) value);
    case TIMESTAMP:
      if (((Types.TimestampType) type).shouldAdjustToUTC()) {
        return DateTimeUtil.timestamptzFromMicros((Long) value);
      } else {
        return DateTimeUtil.timestampFromMicros((Long) value);
      }
    case FIXED:
      if (value instanceof GenericData.Fixed) {
        return ((GenericData.Fixed) value).bytes();
      }
      return value;
    default:
  }
  return value;
}
 

private static Object getPrimitiveValue(Row row, int ord, Type type) {
  if (row.isNullAt(ord)) {
    return null;
  }
  switch (type.typeId()) {
    case BOOLEAN:
      return row.getBoolean(ord);
    case INTEGER:
      return row.getInt(ord);
    case LONG:
      return row.getLong(ord);
    case FLOAT:
      return row.getFloat(ord);
    case DOUBLE:
      return row.getDouble(ord);
    case STRING:
      return row.getString(ord);
    case BINARY:
    case FIXED:
    case UUID:
      return row.get(ord);
    case DATE:
      return row.getDate(ord);
    case TIMESTAMP:
      return row.getTimestamp(ord);
    case DECIMAL:
      return row.getDecimal(ord);
    default:
      throw new IllegalArgumentException("Unhandled type " + type);
  }
}
 

@Override
public boolean canTransform(Type type) {
  return type.typeId() == Type.TypeID.INTEGER;
}
 

@SuppressWarnings("unchecked")
private static void assertEqualsSafe(Type type, Object expected, Object actual) {
  if (expected == null && actual == null) {
    return;
  }

  switch (type.typeId()) {
    case BOOLEAN:
    case INTEGER:
    case LONG:
    case FLOAT:
    case DOUBLE:
      Assert.assertEquals("Primitive value should be equal to expected", expected, actual);
      break;
    case DATE:
      Assert.assertTrue("Should be an int", expected instanceof Integer);
      Assert.assertTrue("Should be a Date", actual instanceof Date);
      int daysFromEpoch = (Integer) expected;
      LocalDate date = ChronoUnit.DAYS.addTo(EPOCH_DAY, daysFromEpoch);
      Assert.assertEquals("ISO-8601 date should be equal", date.toString(), actual.toString());
      break;
    case TIMESTAMP:
      Assert.assertTrue("Should be a long", expected instanceof Long);
      Assert.assertTrue("Should be a Timestamp", actual instanceof Timestamp);
      Timestamp ts = (Timestamp) actual;
      // milliseconds from nanos has already been added by getTime
      long tsMicros = (ts.getTime() * 1000) + ((ts.getNanos() / 1000) % 1000);
      Assert.assertEquals("Timestamp micros should be equal", expected, tsMicros);
      break;
    case STRING:
      Assert.assertTrue("Should be a String", actual instanceof String);
      Assert.assertEquals("Strings should be equal", String.valueOf(expected), actual);
      break;
    case UUID:
      Assert.assertTrue("Should expect a UUID", expected instanceof UUID);
      Assert.assertTrue("Should be a String", actual instanceof String);
      Assert.assertEquals("UUID string representation should match",
          expected.toString(), actual);
      break;
    case FIXED:
      Assert.assertTrue("Should expect a Fixed", expected instanceof GenericData.Fixed);
      Assert.assertTrue("Should be a byte[]", actual instanceof byte[]);
      Assert.assertArrayEquals("Bytes should match",
          ((GenericData.Fixed) expected).bytes(), (byte[]) actual);
      break;
    case BINARY:
      Assert.assertTrue("Should expect a ByteBuffer", expected instanceof ByteBuffer);
      Assert.assertTrue("Should be a byte[]", actual instanceof byte[]);
      Assert.assertArrayEquals("Bytes should match",
          ((ByteBuffer) expected).array(), (byte[]) actual);
      break;
    case DECIMAL:
      Assert.assertTrue("Should expect a BigDecimal", expected instanceof BigDecimal);
      Assert.assertTrue("Should be a BigDecimal", actual instanceof BigDecimal);
      Assert.assertEquals("BigDecimals should be equal", expected, actual);
      break;
    case STRUCT:
      Assert.assertTrue("Should expect a Record", expected instanceof Record);
      Assert.assertTrue("Should be a Row", actual instanceof Row);
      assertEqualsSafe(type.asNestedType().asStructType(), (Record) expected, (Row) actual);
      break;
    case LIST:
      Assert.assertTrue("Should expect a Collection", expected instanceof Collection);
      Assert.assertTrue("Should be a Seq", actual instanceof Seq);
      List<?> asList = seqAsJavaListConverter((Seq<?>) actual).asJava();
      assertEqualsSafe(type.asNestedType().asListType(), (Collection) expected, asList);
      break;
    case MAP:
      Assert.assertTrue("Should expect a Collection", expected instanceof Map);
      Assert.assertTrue("Should be a Map", actual instanceof scala.collection.Map);
      Map<String, ?> asMap = mapAsJavaMapConverter(
          (scala.collection.Map<String, ?>) actual).asJava();
      assertEqualsSafe(type.asNestedType().asMapType(), (Map<String, ?>) expected, asMap);
      break;
    case TIME:
    default:
      throw new IllegalArgumentException("Not a supported type: " + type);
  }
}
 

/**
 * Check that the given InternalRow is equivalent to the Row.
 * @param prefix context for error messages
 * @param type the type of the row
 * @param expected the expected value of the row
 * @param actual the actual value of the row
 */
public static void assertEquals(String prefix, Types.StructType type,
                                InternalRow expected, Row actual) {
  if (expected == null || actual == null) {
    Assert.assertEquals(prefix, expected, actual);
  } else {
    List<Types.NestedField> fields = type.fields();
    for (int c = 0; c < fields.size(); ++c) {
      String fieldName = fields.get(c).name();
      Type childType = fields.get(c).type();
      switch (childType.typeId()) {
        case BOOLEAN:
        case INTEGER:
        case LONG:
        case FLOAT:
        case DOUBLE:
        case STRING:
        case DECIMAL:
        case DATE:
        case TIMESTAMP:
          Assert.assertEquals(prefix + "." + fieldName + " - " + childType,
              getValue(expected, c, childType),
              getPrimitiveValue(actual, c, childType));
          break;
        case UUID:
        case FIXED:
        case BINARY:
          assertEqualBytes(prefix + "." + fieldName,
              (byte[]) getValue(expected, c, childType),
              (byte[]) actual.get(c));
          break;
        case STRUCT: {
          Types.StructType st = (Types.StructType) childType;
          assertEquals(prefix + "." + fieldName, st,
              expected.getStruct(c, st.fields().size()), actual.getStruct(c));
          break;
        }
        case LIST:
          assertEqualsLists(prefix + "." + fieldName, childType.asListType(),
              expected.getArray(c),
              toList((Seq<?>) actual.get(c)));
          break;
        case MAP:
          assertEqualsMaps(prefix + "." + fieldName, childType.asMapType(), expected.getMap(c),
              toJavaMap((scala.collection.Map<?, ?>) actual.getMap(c)));
          break;
        default:
          throw new IllegalArgumentException("Unhandled type " + childType);
      }
    }
  }
}
 

@Override
public boolean canTransform(Type type) {
  return type.typeId() == Type.TypeID.BINARY;
}
 

private static void assertEqualsLists(String prefix, Types.ListType type,
                                      ArrayData expected, List actual) {
  if (expected == null || actual == null) {
    Assert.assertEquals(prefix, expected, actual);
  } else {
    Assert.assertEquals(prefix + " length", expected.numElements(), actual.size());
    Type childType = type.elementType();
    for (int e = 0; e < expected.numElements(); ++e) {
      switch (childType.typeId()) {
        case BOOLEAN:
        case INTEGER:
        case LONG:
        case FLOAT:
        case DOUBLE:
        case STRING:
        case DECIMAL:
        case DATE:
        case TIMESTAMP:
          Assert.assertEquals(prefix + ".elem " + e + " - " + childType,
              getValue(expected, e, childType),
              actual.get(e));
          break;
        case UUID:
        case FIXED:
        case BINARY:
          assertEqualBytes(prefix + ".elem " + e,
              (byte[]) getValue(expected, e, childType),
              (byte[]) actual.get(e));
          break;
        case STRUCT: {
          Types.StructType st = (Types.StructType) childType;
          assertEquals(prefix + ".elem " + e, st,
              expected.getStruct(e, st.fields().size()), (Row) actual.get(e));
          break;
        }
        case LIST:
          assertEqualsLists(prefix + ".elem " + e, childType.asListType(),
              expected.getArray(e),
              toList((Seq<?>) actual.get(e)));
          break;
        case MAP:
          assertEqualsMaps(prefix + ".elem " + e, childType.asMapType(),
              expected.getMap(e), toJavaMap((scala.collection.Map<?, ?>) actual.get(e)));
          break;
        default:
          throw new IllegalArgumentException("Unhandled type " + childType);
      }
    }
  }
}
 

@Override
public boolean canTransform(Type type) {
  return type.typeId() == TypeID.DECIMAL;
}
 

private static TypeDescription convert(Integer fieldId, Type type, boolean isRequired) {
  final TypeDescription orcType;

  switch (type.typeId()) {
    case BOOLEAN:
      orcType = TypeDescription.createBoolean();
      break;
    case INTEGER:
      orcType = TypeDescription.createInt();
      break;
    case TIME:
      orcType = TypeDescription.createLong();
      orcType.setAttribute(ICEBERG_LONG_TYPE_ATTRIBUTE, LongType.TIME.toString());
      break;
    case LONG:
      orcType = TypeDescription.createLong();
      orcType.setAttribute(ICEBERG_LONG_TYPE_ATTRIBUTE, LongType.LONG.toString());
      break;
    case FLOAT:
      orcType = TypeDescription.createFloat();
      break;
    case DOUBLE:
      orcType = TypeDescription.createDouble();
      break;
    case DATE:
      orcType = TypeDescription.createDate();
      break;
    case TIMESTAMP:
      Types.TimestampType tsType = (Types.TimestampType) type;
      if (tsType.shouldAdjustToUTC()) {
        orcType = TypeDescription.createTimestampInstant();
      } else {
        orcType = TypeDescription.createTimestamp();
      }
      break;
    case STRING:
      orcType = TypeDescription.createString();
      break;
    case UUID:
      orcType = TypeDescription.createBinary();
      orcType.setAttribute(ICEBERG_BINARY_TYPE_ATTRIBUTE, BinaryType.UUID.toString());
      break;
    case FIXED:
      orcType = TypeDescription.createBinary();
      orcType.setAttribute(ICEBERG_BINARY_TYPE_ATTRIBUTE, BinaryType.FIXED.toString());
      orcType.setAttribute(ICEBERG_FIELD_LENGTH, Integer.toString(((Types.FixedType) type).length()));
      break;
    case BINARY:
      orcType = TypeDescription.createBinary();
      orcType.setAttribute(ICEBERG_BINARY_TYPE_ATTRIBUTE, BinaryType.BINARY.toString());
      break;
    case DECIMAL: {
      Types.DecimalType decimal = (Types.DecimalType) type;
      orcType = TypeDescription.createDecimal()
          .withScale(decimal.scale())
          .withPrecision(decimal.precision());
      break;
    }
    case STRUCT: {
      orcType = TypeDescription.createStruct();
      for (Types.NestedField field : type.asStructType().fields()) {
        TypeDescription childType = convert(field.fieldId(), field.type(), field.isRequired());
        orcType.addField(field.name(), childType);
      }
      break;
    }
    case LIST: {
      Types.ListType list = (Types.ListType) type;
      TypeDescription elementType = convert(list.elementId(), list.elementType(),
          list.isElementRequired());
      orcType = TypeDescription.createList(elementType);
      break;
    }
    case MAP: {
      Types.MapType map = (Types.MapType) type;
      TypeDescription keyType = convert(map.keyId(), map.keyType(), true);
      TypeDescription valueType = convert(map.valueId(), map.valueType(), map.isValueRequired());
      orcType = TypeDescription.createMap(keyType, valueType);
      break;
    }
    default:
      throw new IllegalArgumentException("Unhandled type " + type.typeId());
  }

  // Set Iceberg column attributes for mapping
  orcType.setAttribute(ICEBERG_ID_ATTRIBUTE, String.valueOf(fieldId));
  orcType.setAttribute(ICEBERG_REQUIRED_ATTRIBUTE, String.valueOf(isRequired));
  return orcType;
}
 

public static String convert(Type type) {
  switch (type.typeId()) {
    case BOOLEAN:
      return "boolean";
    case INTEGER:
      return "int";
    case LONG:
      return "bigint";
    case FLOAT:
      return "float";
    case DOUBLE:
      return "double";
    case DATE:
      return "date";
    case TIME:
      return "string";
    case TIMESTAMP:
      return "timestamp";
    case STRING:
    case UUID:
      return "string";
    case FIXED:
      return "binary";
    case BINARY:
      return "binary";
    case DECIMAL:
      final Types.DecimalType decimalType = (Types.DecimalType) type;
      // TODO may be just decimal?
      return String.format("decimal(%s,%s)", decimalType.precision(), decimalType.scale());
    case STRUCT:
      final Types.StructType structType = type.asStructType();
      final String nameToType = structType.fields().stream()
          .map(f -> String.format("%s:%s", f.name(), convert(f.type())))
          .collect(Collectors.joining(","));
      return String.format("struct<%s>", nameToType);
    case LIST:
      final Types.ListType listType = type.asListType();
      return String.format("array<%s>", convert(listType.elementType()));
    case MAP:
      final Types.MapType mapType = type.asMapType();
      return String.format("map<%s,%s>", convert(mapType.keyType()), convert(mapType.valueType()));
    default:
      throw new UnsupportedOperationException(type + " is not supported");
  }
}