Java Code Examples for org.apache.kafka.connect.data.Schema#Type

private Schema resolveArithmaticType(final Schema leftSchema,
                                          final Schema rightSchema) {
  Schema.Type leftType = leftSchema.type();
  Schema.Type rightType = rightSchema.type();

  if (leftType == rightType) {
    return leftSchema;
  } else if (((leftType == Schema.Type.STRING) || (rightType == Schema.Type.STRING))
      || ((leftType == Schema.Type.BOOLEAN) || (rightType == Schema.Type.BOOLEAN))) {
    throw new PlanException("Incompatible types.");
  } else if ((leftType == Schema.Type.FLOAT64) || (rightType == Schema.Type.FLOAT64)) {
    return Schema.FLOAT64_SCHEMA;
  } else if ((leftType == Schema.Type.INT64) || (rightType == Schema.Type.INT64)) {
    return Schema.INT64_SCHEMA;
  } else if ((leftType == Schema.Type.INT32) || (rightType == Schema.Type.INT32)) {
    return Schema.INT32_SCHEMA;
  }
  throw new PlanException("Unsupported types.");
}
 

@Override
protected Map<String, Schema.Type> determineFieldTypes(InputStream inputStream) throws IOException {
  Map<String, Schema.Type> typeMap = new LinkedHashMap<>();

  JsonFactory factory = new JsonFactory();
  try (JsonParser parser = factory.createParser(inputStream)) {
    Iterator<JsonNode> iterator = ObjectMapperFactory.INSTANCE.readValues(parser, JsonNode.class);
    while (iterator.hasNext()) {
      JsonNode node = iterator.next();
      if (node.isObject()) {
        Iterator<String> fieldNames = node.fieldNames();
        while (fieldNames.hasNext()) {
          typeMap.put(fieldNames.next(), Schema.Type.STRING);
        }
        break;
      }
    }
  }

  return typeMap;
}
 

public static Struct struct(
    String name,
    String f1,
    Schema.Type t1,
    boolean o1,
    Object v1,
    String f2,
    Schema.Type t2,
    boolean o2,
    Object v2
) {
  return struct(
      name,
      Arrays.asList(
          FieldState.of(f1, t1, o1, v1),
          FieldState.of(f2, t2, o2, v2)
      )
  );
}
 

private static Schema.Type schemaTypeForSchemalessJavaType(Object value) {
    if (value == null) {
        return null;
    } else if (value instanceof Byte) {
        return Schema.Type.INT8;
    } else if (value instanceof Short) {
        return Schema.Type.INT16;
    } else if (value instanceof Integer) {
        return Schema.Type.INT32;
    } else if (value instanceof Long) {
        return Schema.Type.INT64;
    } else if (value instanceof Float) {
        return Schema.Type.FLOAT32;
    } else if (value instanceof Double) {
        return Schema.Type.FLOAT64;
    } else if (value instanceof Boolean) {
        return Schema.Type.BOOLEAN;
    } else if (value instanceof String) {
        return Schema.Type.STRING;
    } else if (value instanceof Collection) {
        return Schema.Type.ARRAY;
    } else if (value instanceof Map) {
        return Schema.Type.MAP;
    } else {
        throw new DataException("Unknown Java type for schemaless data: " + value.getClass());
    }
}
 

public static Schema keySchema(Schema schema) {
    Schema.Type type = schema.type();
    if (Schema.Type.MAP.equals(type)) {
        return schema.keySchema();
    } else {
        return null;
    }
}
 

public static TimestampExtractionPolicy create(
    final Schema schema,
    final String timestampColumnName,
    final String timestampFormat) {
  if (timestampColumnName == null) {
    return new MetadataTimestampExtractionPolicy();
  }

  final String fieldName = StringUtil.cleanQuotes(timestampColumnName.toUpperCase());
  final Field timestampField = SchemaUtil.getFieldByName(schema,
      fieldName)
      .orElseThrow(() -> new KsqlException(String.format(
          "No column with the provided timestamp column name in the "
              + "WITH clause, %s, exists in the defined schema.",
          fieldName
      )));

  final Schema.Type timestampFieldType = timestampField.schema().type();
  if (timestampFieldType == Schema.Type.STRING) {
    if (timestampFormat == null) {
      throw new KsqlException("A String timestamp field has been specified without"
          + " also specifying the "
          + DdlConfig.TIMESTAMP_FORMAT_PROPERTY.toLowerCase());
    }
    return new StringTimestampExtractionPolicy(
        fieldName,
        StringUtil.cleanQuotes(timestampFormat));
  }

  if (timestampFieldType == Schema.Type.INT64) {
    return new LongColumnTimestampExtractionPolicy(fieldName);
  }

  throw new KsqlException(
      "Timestamp column, " + timestampColumnName + ", should be LONG(INT64)"
          + " or a String with a "
          + DdlConfig.TIMESTAMP_FORMAT_PROPERTY.toLowerCase()
          + " specified");
}
 

private Object mapDatatype(Schema.Type type, Record record, int fieldIndex, boolean dataTypeMappingError) {
    try {
        switch (type) {
            case INT8:
                return record.getByte(fieldIndex);
            case INT16:
                return record.getShort(fieldIndex);
            case INT32:
                return record.getInt(fieldIndex);
            case INT64:
                return record.getLong(fieldIndex);
            case FLOAT32:
                return record.getFloat(fieldIndex);
            case FLOAT64:
                return record.getDouble(fieldIndex);
            case BOOLEAN:
                return record.getBoolean(fieldIndex);
            case BYTES:
                return record.getString(fieldIndex).getBytes();
            case ARRAY:
            case MAP:
            case STRUCT:
            case STRING:
            default:
                return record.getString(fieldIndex);
        }
    } catch (RuntimeException re) {
        if (dataTypeMappingError) {
            throw re;
        }
        return null;
    }
}
 

public static Struct struct(
    String name,
    String f1,
    Schema.Type t1,
    boolean o1,
    Object v1,
    String f2,
    Schema.Type t2,
    boolean o2,
    Object v2,
    String f3,
    Schema.Type t3,
    boolean o3,
    Object v3,
    String f4,
    Schema.Type t4,
    boolean o4,
    Object v4,
    String f5,
    Schema.Type t5,
    boolean o5,
    Object v5
) {
  return struct(
      name,
      Arrays.asList(
          FieldState.of(f1, t1, o1, v1),
          FieldState.of(f2, t2, o2, v2),
          FieldState.of(f3, t3, o3, v3),
          FieldState.of(f4, t4, o4, v4),
          FieldState.of(f5, t5, o5, v5)
      )
  );
}
 

public static Map<String, Object> toJsonMap(Struct struct) {
    Map<String, Object> jsonMap = new HashMap<String, Object>(0);
    List<Field> fields = struct.schema().fields();
    for (Field field : fields) {
        String fieldName = field.name();
        Schema.Type fieldType = field.schema().type();
        String schemaName=field.schema().name();
        switch (fieldType) {
            case STRING:
                jsonMap.put(fieldName, struct.getString(fieldName));
                break;
            case INT32:
            	if (Date.LOGICAL_NAME.equals(schemaName) 
            			|| Time.LOGICAL_NAME.equals(schemaName)) {
            		jsonMap.put(fieldName, (java.util.Date) struct.get(fieldName));
            	} else {
            		jsonMap.put(fieldName, struct.getInt32(fieldName));
            	}
                break;
            case INT16:
                jsonMap.put(fieldName, struct.getInt16(fieldName));
                break;
            case INT64:
            	if (Timestamp.LOGICAL_NAME.equals(schemaName)) {
            		jsonMap.put(fieldName, (java.util.Date) struct.get(fieldName));
            	} else {
            		jsonMap.put(fieldName, struct.getInt64(fieldName));
            	}
                break;
            case FLOAT32:
                jsonMap.put(fieldName, struct.getFloat32(fieldName));
                break;
            case STRUCT:
                jsonMap.put(fieldName, toJsonMap(struct.getStruct(fieldName)));
                break;
        }
    }
    return jsonMap;
}
 

public static Struct struct(
    String name,
    String f1,
    Schema.Type t1,
    boolean o1,
    Object v1,
    String f2,
    Schema.Type t2,
    boolean o2,
    Object v2,
    String f3,
    Schema.Type t3,
    boolean o3,
    Object v3,
    String f4,
    Schema.Type t4,
    boolean o4,
    Object v4
) {
  return struct(
      name,
      Arrays.asList(
          FieldState.of(f1, t1, o1, v1),
          FieldState.of(f2, t2, o2, v2),
          FieldState.of(f3, t3, o3, v3),
          FieldState.of(f4, t4, o4, v4)
      )
  );
}
 

public static Struct struct(
    String name,
    String f1,
    Schema.Type t1,
    boolean o1,
    Object v1,
    String f2,
    Schema.Type t2,
    boolean o2,
    Object v2,
    String f3,
    Schema.Type t3,
    boolean o3,
    Object v3,
    String f4,
    Schema.Type t4,
    boolean o4,
    Object v4,
    String f5,
    Schema.Type t5,
    boolean o5,
    Object v5,
    String f6,
    Schema.Type t6,
    boolean o6,
    Object v6,
    String f7,
    Schema.Type t7,
    boolean o7,
    Object v7,
    String f8,
    Schema.Type t8,
    boolean o8,
    Object v8
) {
  return struct(
      name,
      Arrays.asList(
          FieldState.of(f1, t1, o1, v1),
          FieldState.of(f2, t2, o2, v2),
          FieldState.of(f3, t3, o3, v3),
          FieldState.of(f4, t4, o4, v4),
          FieldState.of(f5, t5, o5, v5),
          FieldState.of(f6, t6, o6, v6),
          FieldState.of(f7, t7, o7, v7),
          FieldState.of(f8, t8, o8, v8)
      )
  );
}
 

public static Struct struct(
    String name,
    String f1,
    Schema.Type t1,
    boolean o1,
    Object v1,
    String f2,
    Schema.Type t2,
    boolean o2,
    Object v2,
    String f3,
    Schema.Type t3,
    boolean o3,
    Object v3,
    String f4,
    Schema.Type t4,
    boolean o4,
    Object v4,
    String f5,
    Schema.Type t5,
    boolean o5,
    Object v5,
    String f6,
    Schema.Type t6,
    boolean o6,
    Object v6,
    String f7,
    Schema.Type t7,
    boolean o7,
    Object v7
) {
  return struct(
      name,
      Arrays.asList(
          FieldState.of(f1, t1, o1, v1),
          FieldState.of(f2, t2, o2, v2),
          FieldState.of(f3, t3, o3, v3),
          FieldState.of(f4, t4, o4, v4),
          FieldState.of(f5, t5, o5, v5),
          FieldState.of(f6, t6, o6, v6),
          FieldState.of(f7, t7, o7, v7)
      )
  );
}
 

public static Struct struct(
    String name,
    String f1,
    Schema.Type t1,
    boolean o1,
    Object v1,
    String f2,
    Schema.Type t2,
    boolean o2,
    Object v2,
    String f3,
    Schema.Type t3,
    boolean o3,
    Object v3,
    String f4,
    Schema.Type t4,
    boolean o4,
    Object v4,
    String f5,
    Schema.Type t5,
    boolean o5,
    Object v5,
    String f6,
    Schema.Type t6,
    boolean o6,
    Object v6,
    String f7,
    Schema.Type t7,
    boolean o7,
    Object v7,
    String f8,
    Schema.Type t8,
    boolean o8,
    Object v8,
    String f9,
    Schema.Type t9,
    boolean o9,
    Object v9,
    String f10,
    Schema.Type t10,
    boolean o10,
    Object v10
) {
  return struct(
      name,
      Arrays.asList(
          FieldState.of(f1, t1, o1, v1),
          FieldState.of(f2, t2, o2, v2),
          FieldState.of(f3, t3, o3, v3),
          FieldState.of(f4, t4, o4, v4),
          FieldState.of(f5, t5, o5, v5),
          FieldState.of(f6, t6, o6, v6),
          FieldState.of(f7, t7, o7, v7),
          FieldState.of(f8, t8, o8, v8),
          FieldState.of(f9, t9, o9, v9),
          FieldState.of(f10, t10, o10, v10)
      )
  );
}
 

ParserKey(Schema.Type type, String logicalName) {
  this.type = type;
  this.logicalName = logicalName == null ? "" : logicalName;
}
 

@BeforeEach
  public void before() {
    tests = new ArrayList<>();
    final ImmutableSet<Schema.Type> nonPrimitives = ImmutableSet.of(
        Schema.Type.ARRAY,
        Schema.Type.STRUCT,
        Schema.Type.MAP
    );

    tests.addAll(
        Arrays.stream(Schema.Type.values())
            .filter(type -> !nonPrimitives.contains(type))
            .map(type -> test(
                SchemaBuilder.type(type).build(),
                SchemaBuilder.type(type).build(),
                true
                )
            ).collect(Collectors.toList())
    );

    tests.addAll(
        Arrays.stream(Schema.Type.values())
            .filter(type -> !nonPrimitives.contains(type))
            .map(type -> test(
                SchemaBuilder.array(SchemaBuilder.type(type).build()).build(),
                SchemaBuilder.array(SchemaBuilder.type(type).build()).build(),
                true
                )
            ).collect(Collectors.toList())
    );


//    tests.addAll(
//        Arrays.stream(Schema.Type.values())
//            .filter(type -> !nonPrimitives.contains(type))
//            .map(type -> test(
//                SchemaBuilder.type(type).build(),
//                SchemaBuilder.type(type).optional().build(),
//                false
//                )
//            ).collect(Collectors.toList())
//    );

    tests.addAll(
        Arrays.asList(
            test(Schema.STRING_SCHEMA, Schema.STRING_SCHEMA, true),
            test(Schema.STRING_SCHEMA, Schema.BOOLEAN_SCHEMA, false),
            test(Timestamp.SCHEMA, Timestamp.SCHEMA, true),
            test(Timestamp.SCHEMA, Schema.INT64_SCHEMA, false)
        )
    );
  }
 

public static Struct struct(
    String name,
    String f1,
    Schema.Type t1,
    boolean o1,
    Object v1,
    String f2,
    Schema.Type t2,
    boolean o2,
    Object v2,
    String f3,
    Schema.Type t3,
    boolean o3,
    Object v3,
    String f4,
    Schema.Type t4,
    boolean o4,
    Object v4,
    String f5,
    Schema.Type t5,
    boolean o5,
    Object v5,
    String f6,
    Schema.Type t6,
    boolean o6,
    Object v6,
    String f7,
    Schema.Type t7,
    boolean o7,
    Object v7,
    String f8,
    Schema.Type t8,
    boolean o8,
    Object v8,
    String f9,
    Schema.Type t9,
    boolean o9,
    Object v9
) {
  return struct(
      name,
      Arrays.asList(
          FieldState.of(f1, t1, o1, v1),
          FieldState.of(f2, t2, o2, v2),
          FieldState.of(f3, t3, o3, v3),
          FieldState.of(f4, t4, o4, v4),
          FieldState.of(f5, t5, o5, v5),
          FieldState.of(f6, t6, o6, v6),
          FieldState.of(f7, t7, o7, v7),
          FieldState.of(f8, t8, o8, v8),
          FieldState.of(f9, t9, o9, v9)
      )
  );
}
 

private SchemaKey(String schemaName, Schema.Type type) {
  this.schemaName = schemaName;
  this.type = type;
}
 

/**
 * Parses Kafka Values
 * 
 * @param schema
 *            - Schema of passed message as per
 *            https://kafka.apache.org/0100/javadoc/org/apache/kafka/connect/data/Schema.html
 * @param value
 *            - Value of the message
 * @return Parsed bytebuffer as per schema type
 */
public static ByteBuffer parseValue(Schema schema, Object value) {
	Schema.Type t = schema.type();
	switch (t) {
	case INT8:
		ByteBuffer byteBuffer = ByteBuffer.allocate(1);
		byteBuffer.put((Byte) value);
		return byteBuffer;
	case INT16:
		ByteBuffer shortBuf = ByteBuffer.allocate(2);
		shortBuf.putShort((Short) value);
		return shortBuf;
	case INT32:
		ByteBuffer intBuf = ByteBuffer.allocate(4);
		intBuf.putInt((Integer) value);
		return intBuf;
	case INT64:
		ByteBuffer longBuf = ByteBuffer.allocate(8);
		longBuf.putLong((Long) value);
		return longBuf;
	case FLOAT32:
		ByteBuffer floatBuf = ByteBuffer.allocate(4);
		floatBuf.putFloat((Float) value);
		return floatBuf;
	case FLOAT64:
		ByteBuffer doubleBuf = ByteBuffer.allocate(8);
		doubleBuf.putDouble((Double) value);
		return doubleBuf;
	case BOOLEAN:
		ByteBuffer boolBuffer = ByteBuffer.allocate(1);
		boolBuffer.put((byte) ((Boolean) value ? 1 : 0));
		return boolBuffer;
	case STRING:
		try {
			return ByteBuffer.wrap(((String) value).getBytes("UTF-8"));
		} catch (UnsupportedEncodingException e) {
			// TODO Auto-generated catch block
			System.out.println("Message cannot be translated:" + e.getLocalizedMessage());
		}
	case ARRAY:
		Schema sch = schema.valueSchema();
		if (sch.type() == Type.MAP || sch.type() == Type.STRUCT) {
			throw new DataException("Invalid schema type.");
		}
		Object[] objs = (Object[]) value;
		ByteBuffer[] byteBuffers = new ByteBuffer[objs.length];
		int noOfByteBuffer = 0;

		for (Object obj : objs) {
			byteBuffers[noOfByteBuffer++] = parseValue(sch, obj);
		}

		ByteBuffer result = ByteBuffer.allocate(Arrays.stream(byteBuffers).mapToInt(Buffer::remaining).sum());
		Arrays.stream(byteBuffers).forEach(bb -> result.put(bb.duplicate()));
		return result;
	case BYTES:
		if (value instanceof byte[])
			return ByteBuffer.wrap((byte[]) value);
		else if (value instanceof ByteBuffer)
			return (ByteBuffer) value;
	case MAP:
		// TO BE IMPLEMENTED
		return ByteBuffer.wrap(null);
	case STRUCT:

		List<ByteBuffer> fieldList = new LinkedList<ByteBuffer>();
		// Parsing each field of structure
		schema.fields().forEach(field -> fieldList.add(parseValue(field.schema(), ((Struct) value).get(field))));
		// Initialize ByteBuffer
		ByteBuffer processedValue = ByteBuffer.allocate(fieldList.stream().mapToInt(Buffer::remaining).sum());
		// Combine bytebuffer of all fields
		fieldList.forEach(buffer -> processedValue.put(buffer.duplicate()));

		return processedValue;

	}
	return null;
}
 

@TestFactory
Stream<DynamicTest> roundTrip() {
  List<Schema> schemas = new ArrayList<>();
  schemas.add(
      SchemaBuilder.struct()
          .name("test")
          .doc("This is a test")
          .version(16)
          .field("first_name", Schema.OPTIONAL_STRING_SCHEMA)
          .field("last_name", Schema.OPTIONAL_STRING_SCHEMA)
          .field("defaultValueField", SchemaBuilder.int64().defaultValue(1234L).optional().build())
          .build()
  );

  for (Schema.Type schemaType : Arrays.asList(
      Schema.Type.INT8,
      Schema.Type.INT16,
      Schema.Type.INT32,
      Schema.Type.INT64,
      Schema.Type.FLOAT32,
      Schema.Type.FLOAT64,
      Schema.Type.STRING,
      Schema.Type.BOOLEAN)) {

    schemas.add(
        SchemaBuilder.type(schemaType)
            .name(String.format("%s", schemaType))
            .build()
    );
    schemas.add(
        SchemaBuilder.type(schemaType)
            .name(String.format("Optional%s", schemaType))
            .optional()
            .build()
    );
    schemas.add(
        SchemaBuilder.array(SchemaBuilder.type(schemaType).build())
            .name(String.format("Array%s", schemaType))
            .build()
    );
  }


  return schemas.stream().map(expected -> dynamicTest(
      String.format("%s:%s", expected.type(), null == expected.name() ? "" : expected.name()),
      () -> {
        String input = this.objectMapper.writeValueAsString(expected);
        log.trace("Serialized to \n{}", input);
        Schema actual = this.objectMapper.readValue(input, Schema.class);
        assertSchema(expected, actual);
      }
  ));

}
 

protected abstract Map<String, Schema.Type> determineFieldTypes(InputStream inputStream) throws IOException;