com.facebook.presto.spi.type.BigintType Java Examples

@Test
public void testStreamHasData()
        throws Exception
{
    MaterializedResult result = queryRunner.execute("Select count(1) from " + streamName);

    MaterializedResult expected = MaterializedResult.resultBuilder(SESSION, BigintType.BIGINT)
            .row(0)
            .build();

    assertEquals(result, expected);

    int count = 500;
    createMessages(streamName, count);

    result = queryRunner.execute("SELECT count(1) from " + streamName);

    expected = MaterializedResult.resultBuilder(SESSION, BigintType.BIGINT)
            .row(count)
            .build();

    assertEquals(result, expected);
}
 

@Test
public void testStringNumber()
        throws Exception
{
    byte[] json = "{\"a_number\":481516,\"a_string\":\"2342\"}".getBytes(StandardCharsets.UTF_8);

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", VarcharType.VARCHAR, "a_number", null, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", BigintType.BIGINT, "a_number", null, null, false, false);
    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", VarcharType.VARCHAR, "a_string", null, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", null, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    DecoderTestUtil.checkValue(providers, row1, "481516");
    DecoderTestUtil.checkValue(providers, row2, 481516);
    DecoderTestUtil.checkValue(providers, row3, "2342");
    DecoderTestUtil.checkValue(providers, row4, 2342);
}
 

@Test
public void testNonExistent()
        throws Exception
{
    byte[] json = "{}".getBytes(StandardCharsets.UTF_8);

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", VarcharType.VARCHAR, "very/deep/varchar", null, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", BigintType.BIGINT, "no_bigint", null, null, false, false);
    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", DoubleType.DOUBLE, "double/is_missing", null, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BooleanType.BOOLEAN, "hello", null, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    DecoderTestUtil.checkIsNull(providers, row1);
    DecoderTestUtil.checkIsNull(providers, row2);
    DecoderTestUtil.checkIsNull(providers, row3);
    DecoderTestUtil.checkIsNull(providers, row4);
}
 

public void handleQuantity(Map<String, Quantity>  quantityMap, String prefix, Map<String, Object> data) {
    if (quantityMap != null){
        for (String key : quantityMap.keySet()) {
            String columnName = (prefix + key).toLowerCase();
            if (key.endsWith("cpu")) {
                data.put(columnName, quantityMap.get(key).getNumber().doubleValue());
                if (!getKubeColumn().containsKey(columnName)) {
                    UpdateColumns(columnName, new KubeColumn<Object>(columnName, DoubleType.DOUBLE, key));
                }
            } else {
                data.put(columnName, quantityMap.get(key).getNumber().longValue());
                if (!getKubeColumn().containsKey(columnName)) {
                    UpdateColumns(columnName, new KubeColumn<Object>(columnName, BigintType.BIGINT, key));
                }
            }

        }
    }
}
 

public static long getLong(Type type, RowResult row, int field) {
    if (type == TimestampType.TIMESTAMP) {
        return row.getLong(field) / 1000;
    } else if (type == BigintType.BIGINT) {
        return row.getLong(field);
    } else if (type == IntegerType.INTEGER) {
        return row.getInt(field);
    } else if (type == SmallintType.SMALLINT) {
        return row.getShort(field);
    } else if (type == TinyintType.TINYINT) {
        return row.getByte(field);
    } else if (type == RealType.REAL) {
        return floatToRawIntBits(row.getFloat(field));
    } else if (type instanceof DecimalType) {
        DecimalType dtype = (DecimalType) type;
        if (dtype.isShort()) {
            return row.getDecimal(field).unscaledValue().longValue();
        } else {
            throw new IllegalStateException("getLong not supported for long decimal: " + type);
        }
    } else {
        throw new IllegalStateException("getLong not implemented for " + type);
    }
}
 

@Test
public void testStreamHasData()
        throws Exception
{
    MaterializedResult result = queryRunner.execute("Select count(1) from " + dummyStreamName);
    MaterializedResult expected = MaterializedResult.resultBuilder(SESSION, BigintType.BIGINT)
            .row(0)
            .build();

    assertEquals(result.getRowCount(), expected.getRowCount());

    int count = 500;
    createDummyMessages(dummyStreamName, count);

    result = queryRunner.execute("SELECT count(1) from " + dummyStreamName);

    expected = MaterializedResult.resultBuilder(SESSION, BigintType.BIGINT)
            .row(count)
            .build();

    assertEquals(result.getRowCount(), expected.getRowCount());
    log.info("Completed second test (select counts)");
}
 

@Test
public void testSimple()
        throws Exception
{
    byte[] json = ByteStreams.toByteArray(TestJsonDecoder.class.getResourceAsStream("/decoder/json/message.json"));

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", VarcharType.VARCHAR, "source", null, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "user/screen_name", null, null, false, false);
    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "id", null, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "user/statuses_count", null, null, false, false);
    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", BooleanType.BOOLEAN, "user/geo_enabled", null, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    DecoderTestUtil.checkValue(providers, row1, "<a href=\"http://twitterfeed.com\" rel=\"nofollow\">twitterfeed</a>");
    DecoderTestUtil.checkValue(providers, row2, "EKentuckyNews");
    DecoderTestUtil.checkValue(providers, row3, 493857959588286460L);
    DecoderTestUtil.checkValue(providers, row4, 7630);
    DecoderTestUtil.checkValue(providers, row5, true);
}
 

@Test
public void testSimple()
{
    String csv = "\"row 1\",row2,\"row3\",100,\"200\",300,4.5";

    CsvKinesisRowDecoder rowDecoder = new CsvKinesisRowDecoder();
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", VarcharType.VARCHAR, "0", null, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "1", null, null, false, false);
    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", VarcharType.VARCHAR, "2", null, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "3", null, null, false, false);
    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", BigintType.BIGINT, "4", null, null, false, false);
    KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", BigintType.BIGINT, "5", null, null, false, false);
    KinesisColumnHandle row7 = new KinesisColumnHandle("", 6, "row7", DoubleType.DOUBLE, "6", null, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6, row7);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(csv.getBytes(StandardCharsets.UTF_8), providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    DecoderTestUtil.checkValue(providers, row1, "row 1");
    DecoderTestUtil.checkValue(providers, row2, "row2");
    DecoderTestUtil.checkValue(providers, row3, "row3");
    DecoderTestUtil.checkValue(providers, row4, 100);
    DecoderTestUtil.checkValue(providers, row5, 200);
    DecoderTestUtil.checkValue(providers, row6, 300);
    DecoderTestUtil.checkValue(providers, row7, 4.5d);
}
 

@Test
public void testSimple()
{
    ByteBuffer buf = ByteBuffer.allocate(100);
    buf.putLong(4815162342L); // 0 - 7
    buf.putInt(12345678); // 8 - 11
    buf.putShort((short) 4567); // 12 - 13
    buf.put((byte) 123); // 14
    buf.put("Ich bin zwei Oeltanks".getBytes(StandardCharsets.UTF_8)); // 15+

    byte[] row = new byte[buf.position()];
    System.arraycopy(buf.array(), 0, row, 0, buf.position());

    RawKinesisRowDecoder rowDecoder = new RawKinesisRowDecoder();
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "0", "LONG", null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", BigintType.BIGINT, "8", "INT", null, false, false);
    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "12", "SHORT", null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "14", "BYTE", null, false, false);
    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "15", null, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(row, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    DecoderTestUtil.checkValue(providers, row1, 4815162342L);
    DecoderTestUtil.checkValue(providers, row2, 12345678);
    DecoderTestUtil.checkValue(providers, row3, 4567);
    DecoderTestUtil.checkValue(providers, row4, 123);
    DecoderTestUtil.checkValue(providers, row5, "Ich bin zwei Oeltanks");
}
 

@Test
public void testNullValues()
        throws Exception
{
    byte[] json = "{}".getBytes(StandardCharsets.UTF_8);

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);

    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", RFC2822JsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", RFC2822JsonKinesisFieldDecoder.NAME, null, false, false);

    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", RFC2822JsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", RFC2822JsonKinesisFieldDecoder.NAME, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(json, providers, columns, map(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    // sanity checks
    checkIsNull(providers, row1);
    checkIsNull(providers, row2);
    checkIsNull(providers, row3);
    checkIsNull(providers, row4);
    checkIsNull(providers, row5);
    checkIsNull(providers, row6);
}
 

@Test
public void testNullValues()
        throws Exception
{
    byte[] json = "{}".getBytes(StandardCharsets.UTF_8);

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);

    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false);

    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    // sanity checks
    DecoderTestUtil.checkIsNull(providers, row1);
    DecoderTestUtil.checkIsNull(providers, row2);
    DecoderTestUtil.checkIsNull(providers, row3);
    DecoderTestUtil.checkIsNull(providers, row4);
    DecoderTestUtil.checkIsNull(providers, row5);
    DecoderTestUtil.checkIsNull(providers, row6);
}
 

@Test
public void testNullValues()
        throws Exception
{
    byte[] json = "{}".getBytes(StandardCharsets.UTF_8);

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);

    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);

    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    // sanity checks
    DecoderTestUtil.checkIsNull(providers, row1);
    DecoderTestUtil.checkIsNull(providers, row2);
    DecoderTestUtil.checkIsNull(providers, row3);
    DecoderTestUtil.checkIsNull(providers, row4);
    DecoderTestUtil.checkIsNull(providers, row5);
    DecoderTestUtil.checkIsNull(providers, row6);
}
 

@Test
public void testNullValues()
        throws Exception
{
    byte[] json = "{}".getBytes(StandardCharsets.UTF_8);

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);

    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", SecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", SecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);

    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", SecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", SecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    // sanity checks
    DecoderTestUtil.checkIsNull(providers, row1);
    DecoderTestUtil.checkIsNull(providers, row2);
    DecoderTestUtil.checkIsNull(providers, row3);
    DecoderTestUtil.checkIsNull(providers, row4);
    DecoderTestUtil.checkIsNull(providers, row5);
    DecoderTestUtil.checkIsNull(providers, row6);
}
 

@Test
public void testOtherExtracts()
        throws Exception
{
    // Test other scenarios: deeper dive into object, get JSON constructs as strings, etc.
    byte[] json = ByteStreams.toByteArray(TestJsonDecoder.class.getResourceAsStream("/decoder/json/event.json"));

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "event_source", VarcharType.VARCHAR, "source", null, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "user", VarcharType.VARCHAR, "user/handle", null, null, false, false);
    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "user_string", VarcharType.VARCHAR, "user", null, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "timestamp", BigintType.BIGINT, "timestamp", null, null, false, false);
    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "browser_name", VarcharType.VARCHAR, "environment/browser/name", null, null, false, false);
    KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "tags_array", VarcharType.VARCHAR, "tags", null, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    log.info("Decoding row from event JSON file");
    boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    DecoderTestUtil.checkValue(providers, row1, "otherworld");
    DecoderTestUtil.checkValue(providers, row2, "joeblow");
    DecoderTestUtil.checkValue(providers, row3, "{\"email\":\"[email protected]\",\"handle\":\"joeblow\"}");
    KinesisFieldValueProvider provider = DecoderTestUtil.findValueProvider(providers, row6);
    assertNotNull(provider);
    log.info(new String(provider.getSlice().getBytes(), StandardCharsets.UTF_8));

    DecoderTestUtil.checkValue(providers, row4, 1450214872847L);
    DecoderTestUtil.checkValue(providers, row5, "Chrome");
    DecoderTestUtil.checkValue(providers, row6, "[\"tag1\",\"tag2\",\"tag3\"]");
    log.info("DONE");
}
 

public static Object getObject(Type type, RowResult row, int field) {
    if (row.isNull(field)) {
        return null;
    } else {
        if (type instanceof VarcharType) {
            return row.getString(field);
        } else if (type == TimestampType.TIMESTAMP) {
            return row.getLong(field) / 1000;
        } else if (type == BigintType.BIGINT) {
            return row.getLong(field);
        } else if (type == IntegerType.INTEGER) {
            return row.getInt(field);
        } else if (type == SmallintType.SMALLINT) {
            return row.getShort(field);
        } else if (type == TinyintType.TINYINT) {
            return row.getByte(field);
        } else if (type == DoubleType.DOUBLE) {
            return row.getDouble(field);
        } else if (type == RealType.REAL) {
            return row.getFloat(field);
        } else if (type == BooleanType.BOOLEAN) {
            return row.getBoolean(field);
        } else if (type instanceof VarbinaryType) {
            return Slices.wrappedBuffer(row.getBinary(field));
        } else if (type instanceof DecimalType) {
            return row.getDecimal(field);
        } else {
            throw new IllegalStateException("getObject not implemented for " + type);
        }
    }
}
 

public static Object getJavaValue(Type type, Object nativeValue) {
    if (type instanceof VarcharType) {
        return ((Slice) nativeValue).toStringUtf8();
    } else if (type == TimestampType.TIMESTAMP) {
        return ((Long) nativeValue) * 1000;
    } else if (type == BigintType.BIGINT) {
        return nativeValue;
    } else if (type == IntegerType.INTEGER) {
        return ((Long) nativeValue).intValue();
    } else if (type == SmallintType.SMALLINT) {
        return ((Long) nativeValue).shortValue();
    } else if (type == TinyintType.TINYINT) {
        return ((Long) nativeValue).byteValue();
    } else if (type == DoubleType.DOUBLE) {
        return nativeValue;
    } else if (type == RealType.REAL) {
        // conversion can result in precision lost
        return intBitsToFloat(((Long) nativeValue).intValue());
    } else if (type == BooleanType.BOOLEAN) {
        return nativeValue;
    } else if (type instanceof VarbinaryType) {
        return ((Slice) nativeValue).toByteBuffer();
    } else if (type instanceof DecimalType) {
        return nativeValue;
    } else {
        throw new IllegalStateException("Back conversion not implemented for " + type);
    }
}
 

public static NullableValue getColumnValue(Type type, PartialRow row, int i) {
    if (row.isNull(i)) {
        return NullableValue.asNull(type);
    } else {
        if (type instanceof VarcharType) {
            return NullableValue.of(type, utf8Slice(row.getString(i)));
        } else if (type == TimestampType.TIMESTAMP) {
            return NullableValue.of(type, row.getLong(i) / 1000);
        } else if (type == BigintType.BIGINT) {
            return NullableValue.of(type, row.getLong(i));
        } else if (type == IntegerType.INTEGER) {
            return NullableValue.of(type, row.getInt(i));
        } else if (type == SmallintType.SMALLINT) {
            return NullableValue.of(type, row.getShort(i));
        } else if (type == TinyintType.TINYINT) {
            return NullableValue.of(type, row.getByte(i));
        } else if (type == DoubleType.DOUBLE) {
            return NullableValue.of(type, row.getDouble(i));
        } else if (type == RealType.REAL) {
            return NullableValue.of(type, (long) floatToRawIntBits(row.getFloat(i)));
        } else if (type == BooleanType.BOOLEAN) {
            return NullableValue.of(type, row.getBoolean(i));
        } else if (type instanceof VarbinaryType) {
            return NullableValue.of(type, wrappedBuffer(row.getBinary(i)));
        } else if (type instanceof DecimalType) {
            return NullableValue.of(type, row.getDecimal(i));
        } else {
            throw new IllegalStateException("Handling of type " + type + " is not implemented");
        }
    }
}
 

private static Type fromKuduClientType(org.apache.kudu.Type ktype, ColumnTypeAttributes attributes) {
    switch (ktype) {
        case STRING:
            return VarcharType.VARCHAR;
        case UNIXTIME_MICROS:
            return TimestampType.TIMESTAMP;
        case INT64:
            return BigintType.BIGINT;
        case INT32:
            return IntegerType.INTEGER;
        case INT16:
            return SmallintType.SMALLINT;
        case INT8:
            return TinyintType.TINYINT;
        case FLOAT:
            return RealType.REAL;
        case DOUBLE:
            return DoubleType.DOUBLE;
        case BOOL:
            return BooleanType.BOOLEAN;
        case BINARY:
            return VarbinaryType.VARBINARY;
        case DECIMAL:
            return DecimalType.createDecimalType(attributes.getPrecision(), attributes.getScale());
        default:
            throw new IllegalStateException("Kudu type not implemented for " + ktype);
    }
}
 

public static Map.Entry<SchemaTableName, KinesisStreamDescription> createSimpleJsonStreamDescription(String streamName, SchemaTableName schemaTableName)
{
    // Format: {"id" : 1324, "name" : "some string"}
    ArrayList<KinesisStreamFieldDescription> fieldList = new ArrayList<KinesisStreamFieldDescription>();
    fieldList.add(new KinesisStreamFieldDescription("id", BigintType.BIGINT, "id", "comment", null, null, false));
    fieldList.add(new KinesisStreamFieldDescription("name", VarcharType.VARCHAR, "name", "comment", null, null, false));
    KinesisStreamFieldGroup grp = new KinesisStreamFieldGroup("json", fieldList);

    KinesisStreamDescription desc = new KinesisStreamDescription(schemaTableName.getTableName(), schemaTableName.getSchemaName(), streamName, grp);
    return new AbstractMap.SimpleImmutableEntry<>(schemaTableName, desc);
}
 

public static org.apache.kudu.Type toKuduClientType(Type type) {
    if (type instanceof VarcharType) {
        return org.apache.kudu.Type.STRING;
    } else if (type == TimestampType.TIMESTAMP) {
        return org.apache.kudu.Type.UNIXTIME_MICROS;
    } else if (type == BigintType.BIGINT) {
        return org.apache.kudu.Type.INT64;
    } else if (type == IntegerType.INTEGER) {
        return org.apache.kudu.Type.INT32;
    } else if (type == SmallintType.SMALLINT) {
        return org.apache.kudu.Type.INT16;
    } else if (type == TinyintType.TINYINT) {
        return org.apache.kudu.Type.INT8;
    } else if (type == RealType.REAL) {
        return org.apache.kudu.Type.FLOAT;
    } else if (type == DoubleType.DOUBLE) {
        return org.apache.kudu.Type.DOUBLE;
    } else if (type == BooleanType.BOOLEAN) {
        return org.apache.kudu.Type.BOOL;
    } else if (type instanceof VarbinaryType) {
        return org.apache.kudu.Type.BINARY;
    } else if (type instanceof DecimalType) {
        return org.apache.kudu.Type.DECIMAL;
    } else if (type == DateType.DATE) {
        return org.apache.kudu.Type.STRING;
    } else if (type instanceof CharType) {
        return org.apache.kudu.Type.STRING;
    } else {
        throw new IllegalStateException("Type mapping implemented for Presto type: " + type);
    }
}
 

private static XContentBuilder buildFieldType(XContentBuilder fieldBuilder, Type type)
        throws IOException
{
    final String dateTimeFormat = "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis";
    //Type mapping
    // see:https://www.elastic.co/guide/en/elasticsearch/reference/6.3/mapping-types.html
    if (type.equals(BooleanType.BOOLEAN)) {
        return fieldBuilder.field("type", "boolean");
    }
    if (type.equals(BigintType.BIGINT)) {
        return fieldBuilder.field("type", "long");
    }
    if (type.equals(IntegerType.INTEGER)) {
        return fieldBuilder.field("type", "integer");
    }
    if (type.equals(SmallintType.SMALLINT)) {
        return fieldBuilder.field("type", "short");
    }
    if (type.equals(TinyintType.TINYINT)) {
        return fieldBuilder.field("type", "byte");
    }
    if (type.equals(DoubleType.DOUBLE)) {
        return fieldBuilder.field("type", "double");
    }
    if (type.equals(DateType.DATE)) {
        return fieldBuilder.field("type", "date")
                .field("format", dateTimeFormat);
    }
    if (type.equals(TimeType.TIME)) {
        return fieldBuilder.field("type", "date")
                .field("format", dateTimeFormat);
    }
    if (type.equals(TimestampType.TIMESTAMP)) {
        return fieldBuilder.field("type", "date")
                .field("format", dateTimeFormat);
    }
    if (type.equals(TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE)) {
        //TODO: TIMESTAMP_WITH_TIME_ZONE
        return fieldBuilder.field("type", "date")
                .field("format", "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis");
    }
    if (type instanceof DecimalType) {
        return fieldBuilder.field("type", "double");
    }
    if (isVarcharType(type)) {
        //es 2.4.x
        // .field("index", "not_analyzed")
        return fieldBuilder.field("type", "string");
    }
    if (type.equals(VarbinaryType.VARBINARY)) {
        return fieldBuilder.field("type", "binary");
    }
    if (isArrayType(type)) {
        Type elementType = type.getTypeParameters().get(0);
        if (isArrayType(elementType) || isMapType(elementType) || isRowType(elementType)) {
            throw new PrestoException(NOT_SUPPORTED, "sorry unsupported type: " + type);
        }
        return buildFieldType(fieldBuilder, elementType);
    }
    if (isMapType(type)) {
        throw new PrestoException(NOT_SUPPORTED, "sorry unsupported type: " + type);
    }
    if (isRowType(type)) {
        throw new PrestoException(NOT_SUPPORTED, "sorry unsupported type: " + type);
    }

    throw new PrestoException(NOT_SUPPORTED, "unsupported type: " + type);
}
 

private static XContentBuilder buildFieldType(XContentBuilder fieldBuilder, Type type)
        throws IOException
{
    final String dateTimeFormat = "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis";
    //Type mapping
    // see:https://www.elastic.co/guide/en/elasticsearch/reference/6.3/mapping-types.html
    if (type.equals(BooleanType.BOOLEAN)) {
        return fieldBuilder.field("type", "boolean");
    }
    if (type.equals(BigintType.BIGINT)) {
        return fieldBuilder.field("type", "long");
    }
    if (type.equals(IntegerType.INTEGER)) {
        return fieldBuilder.field("type", "integer");
    }
    if (type.equals(SmallintType.SMALLINT)) {
        return fieldBuilder.field("type", "short");
    }
    if (type.equals(TinyintType.TINYINT)) {
        return fieldBuilder.field("type", "byte");
    }
    if (type.equals(DoubleType.DOUBLE)) {
        return fieldBuilder.field("type", "double");
    }
    if (type.equals(DateType.DATE)) {
        return fieldBuilder.field("type", "date")
                .field("format", dateTimeFormat);
    }
    if (type.equals(TimeType.TIME)) {
        return fieldBuilder.field("type", "date")
                .field("format", dateTimeFormat);
    }
    if (type.equals(TimestampType.TIMESTAMP)) {
        return fieldBuilder.field("type", "date")
                .field("format", dateTimeFormat);
    }
    if (type.equals(TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE)) {
        //TODO: TIMESTAMP_WITH_TIME_ZONE
        return fieldBuilder.field("type", "date")
                .field("format", "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis");
    }
    if (type instanceof DecimalType) {
        return fieldBuilder.field("type", "double");
    }
    if (isVarcharType(type)) {
        //TODO: text or keyword ?
        return fieldBuilder.field("type", "text");
    }
    if (type.equals(VarbinaryType.VARBINARY)) {
        return fieldBuilder.field("type", "binary");
    }
    if (isArrayType(type)) {
        Type elementType = type.getTypeParameters().get(0);
        if (isArrayType(elementType) || isMapType(elementType) || isRowType(elementType)) {
            throw new PrestoException(NOT_SUPPORTED, "sorry unsupported type: " + type);
        }
        return buildFieldType(fieldBuilder, elementType);
    }
    if (isMapType(type)) {
        throw new PrestoException(NOT_SUPPORTED, "sorry unsupported type: " + type);
    }
    if (isRowType(type)) {
        throw new PrestoException(NOT_SUPPORTED, "sorry unsupported type: " + type);
    }

    throw new PrestoException(NOT_SUPPORTED, "unsupported type: " + type);
}
 

@Test
public void testBooleanStuff()
{
    ByteBuffer buf = ByteBuffer.allocate(100);
    buf.put((byte) 127); // offset 0
    buf.putLong(0); // offset 1
    buf.put((byte) 126); // offset 9
    buf.putLong(1); // offset 10

    buf.put((byte) 125); // offset 18
    buf.putInt(0); // offset 19
    buf.put((byte) 124); // offset 23
    buf.putInt(1); // offset 24

    buf.put((byte) 123); // offset 28
    buf.putShort((short) 0); // offset 29
    buf.put((byte) 122); // offset 31
    buf.putShort((short) 1); // offset 32

    buf.put((byte) 121); // offset 34
    buf.put((byte) 0); // offset 35
    buf.put((byte) 120); // offset 36
    buf.put((byte) 1); // offset 37

    byte[] row = new byte[buf.position()];
    System.arraycopy(buf.array(), 0, row, 0, buf.position());

    RawKinesisRowDecoder rowDecoder = new RawKinesisRowDecoder();
    KinesisColumnHandle row01 = new KinesisColumnHandle("", 0, "row01", BigintType.BIGINT, "0", "BYTE", null, false, false);
    KinesisColumnHandle row02 = new KinesisColumnHandle("", 1, "row02", BooleanType.BOOLEAN, "1", "LONG", null, false, false);
    KinesisColumnHandle row03 = new KinesisColumnHandle("", 2, "row03", BigintType.BIGINT, "9", "BYTE", null, false, false);
    KinesisColumnHandle row04 = new KinesisColumnHandle("", 3, "row04", BooleanType.BOOLEAN, "10", "LONG", null, false, false);

    KinesisColumnHandle row11 = new KinesisColumnHandle("", 4, "row11", BigintType.BIGINT, "18", "BYTE", null, false, false);
    KinesisColumnHandle row12 = new KinesisColumnHandle("", 5, "row12", BooleanType.BOOLEAN, "19", "INT", null, false, false);
    KinesisColumnHandle row13 = new KinesisColumnHandle("", 6, "row13", BigintType.BIGINT, "23", "BYTE", null, false, false);
    KinesisColumnHandle row14 = new KinesisColumnHandle("", 7, "row14", BooleanType.BOOLEAN, "24", "INT", null, false, false);

    KinesisColumnHandle row21 = new KinesisColumnHandle("", 8, "row21", BigintType.BIGINT, "28", "BYTE", null, false, false);
    KinesisColumnHandle row22 = new KinesisColumnHandle("", 9, "row22", BooleanType.BOOLEAN, "29", "SHORT", null, false, false);
    KinesisColumnHandle row23 = new KinesisColumnHandle("", 10, "row23", BigintType.BIGINT, "31", "BYTE", null, false, false);
    KinesisColumnHandle row24 = new KinesisColumnHandle("", 11, "row24", BooleanType.BOOLEAN, "32", "SHORT", null, false, false);

    KinesisColumnHandle row31 = new KinesisColumnHandle("", 12, "row31", BigintType.BIGINT, "34", "BYTE", null, false, false);
    KinesisColumnHandle row32 = new KinesisColumnHandle("", 13, "row32", BooleanType.BOOLEAN, "35", "BYTE", null, false, false);
    KinesisColumnHandle row33 = new KinesisColumnHandle("", 14, "row33", BigintType.BIGINT, "36", "BYTE", null, false, false);
    KinesisColumnHandle row34 = new KinesisColumnHandle("", 15, "row34", BooleanType.BOOLEAN, "37", "BYTE", null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row01,
            row02,
            row03,
            row04,
            row11,
            row12,
            row13,
            row14,
            row21,
            row22,
            row23,
            row24,
            row31,
            row32,
            row33,
            row34);

    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(row, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    DecoderTestUtil.checkValue(providers, row01, 127);
    DecoderTestUtil.checkValue(providers, row02, false);
    DecoderTestUtil.checkValue(providers, row03, 126);
    DecoderTestUtil.checkValue(providers, row04, true);

    DecoderTestUtil.checkValue(providers, row11, 125);
    DecoderTestUtil.checkValue(providers, row12, false);
    DecoderTestUtil.checkValue(providers, row13, 124);
    DecoderTestUtil.checkValue(providers, row14, true);

    DecoderTestUtil.checkValue(providers, row21, 123);
    DecoderTestUtil.checkValue(providers, row22, false);
    DecoderTestUtil.checkValue(providers, row23, 122);
    DecoderTestUtil.checkValue(providers, row24, true);

    DecoderTestUtil.checkValue(providers, row31, 121);
    DecoderTestUtil.checkValue(providers, row32, false);
    DecoderTestUtil.checkValue(providers, row33, 120);
    DecoderTestUtil.checkValue(providers, row34, true);
}
 

private static XContentBuilder buildFieldType(XContentBuilder fieldBuilder, Type type)
        throws IOException
{
    final String dateTimeFormat = "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis";
    //Type mapping
    // see:https://www.elastic.co/guide/en/elasticsearch/reference/6.3/mapping-types.html
    if (type.equals(BooleanType.BOOLEAN)) {
        return fieldBuilder.field("type", "boolean");
    }
    if (type.equals(BigintType.BIGINT)) {
        return fieldBuilder.field("type", "long");
    }
    if (type.equals(IntegerType.INTEGER)) {
        return fieldBuilder.field("type", "integer");
    }
    if (type.equals(SmallintType.SMALLINT)) {
        return fieldBuilder.field("type", "short");
    }
    if (type.equals(TinyintType.TINYINT)) {
        return fieldBuilder.field("type", "byte");
    }
    if (type.equals(DoubleType.DOUBLE)) {
        return fieldBuilder.field("type", "double");
    }
    if (type.equals(DateType.DATE)) {
        return fieldBuilder.field("type", "date")
                .field("format", dateTimeFormat);
    }
    if (type.equals(TimeType.TIME)) {
        return fieldBuilder.field("type", "date")
                .field("format", dateTimeFormat);
    }
    if (type.equals(TimestampType.TIMESTAMP)) {
        return fieldBuilder.field("type", "date")
                .field("format", dateTimeFormat);
    }
    if (type.equals(TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE)) {
        //TODO: TIMESTAMP_WITH_TIME_ZONE
        return fieldBuilder.field("type", "date")
                .field("format", "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis");
    }
    if (type instanceof DecimalType) {
        return fieldBuilder.field("type", "double");
    }
    if (isVarcharType(type)) {
        //TODO: text or keyword ?
        return fieldBuilder.field("type", "text");
    }
    if (type.equals(VarbinaryType.VARBINARY)) {
        return fieldBuilder.field("type", "binary");
    }
    if (isArrayType(type)) {
        Type elementType = type.getTypeParameters().get(0);
        if (isArrayType(elementType) || isMapType(elementType) || isRowType(elementType)) {
            throw new PrestoException(NOT_SUPPORTED, "sorry unsupported type: " + type);
        }
        return buildFieldType(fieldBuilder, elementType);
    }
    if (isMapType(type)) {
        throw new PrestoException(NOT_SUPPORTED, "sorry unsupported type: " + type);
    }
    if (isRowType(type)) {
        throw new PrestoException(NOT_SUPPORTED, "sorry unsupported type: " + type);
    }

    throw new PrestoException(NOT_SUPPORTED, "unsupported type: " + type);
}
 

private Type getType(String typeName)
{
    log.debug("Get type " + typeName);
    typeName = typeName.toLowerCase();
    // check if type is varchar(xx)
    Pattern vcpattern = Pattern.compile("varchar\\(\\s*(\\d+)\\s*\\)");
    Matcher vcmatcher = vcpattern.matcher(typeName);
    if (vcmatcher.find()) {
        String vlen = vcmatcher.group(1);
        if (!vlen.isEmpty()) {
            return VarcharType.createVarcharType(Integer.parseInt(vlen));
        }
        return UnknownType.UNKNOWN;
    }
    // check if type is char(xx)
    Pattern cpattern = Pattern.compile("char\\(\\s*(\\d+)\\s*\\)");
    Matcher cmatcher = cpattern.matcher(typeName);
    if (cmatcher.find()) {
        String clen = cmatcher.group(1);
        if (!clen.isEmpty()) {
            return CharType.createCharType(Integer.parseInt(clen));
        }
        return UnknownType.UNKNOWN;
    }
    // check if type is decimal(precision, scale)
    Pattern dpattern = Pattern.compile("decimal\\((\\d+)\\s*,?\\s*(\\d*)\\)");
    Matcher dmatcher = dpattern.matcher(typeName);
    if (dmatcher.find()) {
        String dprecision = dmatcher.group(1);
        String dscale = dmatcher.group(2);
        if (dprecision.isEmpty()) {
            return UnknownType.UNKNOWN;
        }
        if (dscale.isEmpty()) {
            return DecimalType.createDecimalType(Integer.parseInt(dprecision));
        }
        return DecimalType.createDecimalType(Integer.parseInt(dprecision), Integer.parseInt(dscale));
    }
    switch (typeName) {
        case "boolean":
            return BooleanType.BOOLEAN;
        case "tinyint":
            return TinyintType.TINYINT;
        case "smallint":
            return SmallintType.SMALLINT;
        case "integer":
            return IntegerType.INTEGER;
        case "bigint":
            return BigintType.BIGINT;
        case "real":
            return RealType.REAL;
        case "double":
            return DoubleType.DOUBLE;
        case "date":
            return DateType.DATE;
        case "time":
            return TimeType.TIME;
        case "timestamp":
            return TimestampType.TIMESTAMP;
        default:
            return UnknownType.UNKNOWN;
    }
}
 

@Test
public void testBasicFormatting()
        throws Exception
{
    long now = (System.currentTimeMillis() / 1000) * 1000; // rfc2822 is second granularity
    String nowString = FORMATTER.print(now);

    byte[] json = format("{\"a_number\":%d,\"a_string\":\"%s\"}", now, nowString).getBytes(StandardCharsets.UTF_8);

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);

    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", RFC2822JsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", RFC2822JsonKinesisFieldDecoder.NAME, null, false, false);

    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", RFC2822JsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", RFC2822JsonKinesisFieldDecoder.NAME, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(json, providers, columns, map(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    // sanity checks
    checkValue(providers, row1, now);
    checkValue(providers, row2, nowString);

    // number parsed as number --> as is
    checkValue(providers, row3, now);
    // string parsed as number --> parse text, convert to timestamp
    checkValue(providers, row4, now);

    // number parsed as string --> parse text, convert to timestamp, turn into string
    checkValue(providers, row5, Long.toString(now));

    // string parsed as string --> as is
    checkValue(providers, row6, nowString);
}
 

@SuppressWarnings("ValueOfIncrementOrDecrementUsed")
@Override
public Map<String, ColumnHandle> getColumnHandles(ConnectorSession session, ConnectorTableHandle tableHandle) {
    EthereumTableHandle ethereumTableHandle = convertTableHandle(tableHandle);
    String tableName = ethereumTableHandle.getTableName();

    ImmutableMap.Builder<String, ColumnHandle> columnHandles = ImmutableMap.builder();
    int index = 0;
    if (EthereumTable.BLOCK.getName().equals(tableName)) {
        columnHandles.put("block_number", new EthereumColumnHandle(connectorId, index++, "block_number", BigintType.BIGINT));
        columnHandles.put("block_hash", new EthereumColumnHandle(connectorId, index++, "block_hash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        columnHandles.put("block_parentHash", new EthereumColumnHandle(connectorId, index++, "block_parentHash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        columnHandles.put("block_nonce", new EthereumColumnHandle(connectorId, index++, "block_nonce", VarcharType.createVarcharType(H8_BYTE_HASH_STRING_LENGTH)));
        columnHandles.put("block_sha3Uncles", new EthereumColumnHandle(connectorId, index++, "block_sha3Uncles", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        columnHandles.put("block_logsBloom", new EthereumColumnHandle(connectorId, index++, "block_logsBloom", VarcharType.createVarcharType(H256_BYTE_HASH_STRING_LENGTH)));
        columnHandles.put("block_transactionsRoot", new EthereumColumnHandle(connectorId, index++, "block_transactionsRoot", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        columnHandles.put("block_stateRoot", new EthereumColumnHandle(connectorId, index++, "block_stateRoot", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        columnHandles.put("block_miner", new EthereumColumnHandle(connectorId, index++, "block_miner", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH)));
        columnHandles.put("block_difficulty", new EthereumColumnHandle(connectorId, index++, "block_difficulty", BigintType.BIGINT));
        columnHandles.put("block_totalDifficulty", new EthereumColumnHandle(connectorId, index++, "block_totalDifficulty", BigintType.BIGINT));
        columnHandles.put("block_size", new EthereumColumnHandle(connectorId, index++, "block_size", IntegerType.INTEGER));
        columnHandles.put("block_extraData", new EthereumColumnHandle(connectorId, index++, "block_extraData", VarcharType.VARCHAR));
        columnHandles.put("block_gasLimit", new EthereumColumnHandle(connectorId, index++, "block_gasLimit", DoubleType.DOUBLE));
        columnHandles.put("block_gasUsed", new EthereumColumnHandle(connectorId, index++, "block_gasUsed", DoubleType.DOUBLE));
        columnHandles.put("block_timestamp", new EthereumColumnHandle(connectorId, index++, "block_timestamp", BigintType.BIGINT));
        columnHandles.put("block_transactions", new EthereumColumnHandle(connectorId, index++, "block_transactions", new ArrayType(VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH))));
        columnHandles.put("block_uncles", new EthereumColumnHandle(connectorId, index++, "block_uncles", new ArrayType(VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH))));
    } else if (EthereumTable.TRANSACTION.getName().equals(tableName)) {
        columnHandles.put("tx_hash", new EthereumColumnHandle(connectorId, index++, "tx_hash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        columnHandles.put("tx_nonce", new EthereumColumnHandle(connectorId, index++, "tx_nonce", BigintType.BIGINT));
        columnHandles.put("tx_blockHash", new EthereumColumnHandle(connectorId, index++, "tx_blockHash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        columnHandles.put("tx_blockNumber", new EthereumColumnHandle(connectorId, index++, "tx_blockNumber", BigintType.BIGINT));
        columnHandles.put("tx_transactionIndex", new EthereumColumnHandle(connectorId, index++, "tx_transactionIndex", IntegerType.INTEGER));
        columnHandles.put("tx_from", new EthereumColumnHandle(connectorId, index++, "tx_from", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH)));
        columnHandles.put("tx_to", new EthereumColumnHandle(connectorId, index++, "tx_to", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH)));
        columnHandles.put("tx_value", new EthereumColumnHandle(connectorId, index++, "tx_value", DoubleType.DOUBLE));
        columnHandles.put("tx_gas", new EthereumColumnHandle(connectorId, index++, "tx_gas", DoubleType.DOUBLE));
        columnHandles.put("tx_gasPrice", new EthereumColumnHandle(connectorId, index++, "tx_gasPrice", DoubleType.DOUBLE));
        columnHandles.put("tx_input", new EthereumColumnHandle(connectorId, index++, "tx_input", VarcharType.VARCHAR));
    } else if (EthereumTable.ERC20.getName().equals(tableName)) {
        columnHandles.put("erc20_token", new EthereumColumnHandle(connectorId, index++, "erc20_token", VarcharType.createUnboundedVarcharType()));
        columnHandles.put("erc20_from", new EthereumColumnHandle(connectorId, index++, "erc20_from", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH)));
        columnHandles.put("erc20_to", new EthereumColumnHandle(connectorId, index++, "erc20_to", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH)));
        columnHandles.put("erc20_value", new EthereumColumnHandle(connectorId, index++, "erc20_value", DoubleType.DOUBLE));
        columnHandles.put("erc20_txHash", new EthereumColumnHandle(connectorId, index++, "erc20_txHash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        columnHandles.put("erc20_blockNumber", new EthereumColumnHandle(connectorId, index++, "erc20_blockNumber", BigintType.BIGINT));
    } else {
        throw new IllegalArgumentException("Unknown Table Name " + tableName);
    }

    return columnHandles.build();
}
 

@Test
public void testBasicFormatting()
        throws Exception
{
    long now = System.currentTimeMillis();
    String nowString = PRINTER.print(now);

    byte[] json = format("{\"a_number\":%d,\"a_string\":\"%s\"}", now, nowString).getBytes(StandardCharsets.UTF_8);

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);

    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false);

    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    // sanity checks
    DecoderTestUtil.checkValue(providers, row1, now);
    DecoderTestUtil.checkValue(providers, row2, nowString);

    // number parsed as number --> as is
    DecoderTestUtil.checkValue(providers, row3, now);
    // string parsed as number --> parse text, convert to timestamp
    DecoderTestUtil.checkValue(providers, row4, now);

    // number parsed as string --> parse text, convert to timestamp, turn into string
    DecoderTestUtil.checkValue(providers, row5, Long.toString(now));

    // string parsed as string --> as is
    DecoderTestUtil.checkValue(providers, row6, nowString);
}
 

@SuppressWarnings("ValueOfIncrementOrDecrementUsed")
private ConnectorTableMetadata getTableMetadata(SchemaTableName schemaTableName) {
    ImmutableList.Builder<ColumnMetadata> builder = ImmutableList.builder();

    if (EthereumTable.BLOCK.getName().equals(schemaTableName.getTableName())) {
        builder.add(new ColumnMetadata("block_number", BigintType.BIGINT));
        builder.add(new ColumnMetadata("block_hash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("block_parentHash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("block_nonce", VarcharType.createVarcharType(H8_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("block_sha3Uncles", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("block_logsBloom", VarcharType.createVarcharType(H256_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("block_transactionsRoot", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("block_stateRoot", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("block_miner", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("block_difficulty", BigintType.BIGINT));
        builder.add(new ColumnMetadata("block_totalDifficulty", BigintType.BIGINT));
        builder.add(new ColumnMetadata("block_size", IntegerType.INTEGER));
        builder.add(new ColumnMetadata("block_extraData", VarcharType.VARCHAR));
        builder.add(new ColumnMetadata("block_gasLimit", DoubleType.DOUBLE));
        builder.add(new ColumnMetadata("block_gasUsed", DoubleType.DOUBLE));
        builder.add(new ColumnMetadata("block_timestamp", BigintType.BIGINT));
        builder.add(new ColumnMetadata("block_transactions", new ArrayType(VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH))));
        builder.add(new ColumnMetadata("block_uncles", new ArrayType(VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH))));
    } else if (EthereumTable.TRANSACTION.getName().equals(schemaTableName.getTableName())) {
        builder.add(new ColumnMetadata("tx_hash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("tx_nonce", BigintType.BIGINT));
        builder.add(new ColumnMetadata("tx_blockHash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("tx_blockNumber", BigintType.BIGINT));
        builder.add(new ColumnMetadata("tx_transactionIndex", IntegerType.INTEGER));
        builder.add(new ColumnMetadata("tx_from", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("tx_to", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("tx_value", DoubleType.DOUBLE));
        builder.add(new ColumnMetadata("tx_gas", DoubleType.DOUBLE));
        builder.add(new ColumnMetadata("tx_gasPrice", DoubleType.DOUBLE));
        builder.add(new ColumnMetadata("tx_input", VarcharType.VARCHAR));
    } else if (EthereumTable.ERC20.getName().equals(schemaTableName.getTableName())) {
        builder.add(new ColumnMetadata("erc20_token", VarcharType.createUnboundedVarcharType()));
        builder.add(new ColumnMetadata("erc20_from", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("erc20_to", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("erc20_value", DoubleType.DOUBLE));
        builder.add(new ColumnMetadata("erc20_txHash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("erc20_blockNumber", BigintType.BIGINT));
    } else {
        throw new IllegalArgumentException("Unknown Table Name " + schemaTableName.getTableName());
    }

    return new ConnectorTableMetadata(schemaTableName, builder.build());
}
 

@Test
public void testBasicFormatting()
        throws Exception
{
    long now = System.currentTimeMillis();
    String nowString = MillisecondsSinceEpochJsonKinesisFieldDecoder.FORMATTER.print(now);

    byte[] json = format("{\"a_number\":%d,\"a_string\":\"%d\"}", now, now).getBytes(StandardCharsets.UTF_8);

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);

    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);

    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    // sanity checks
    DecoderTestUtil.checkValue(providers, row1, now);
    DecoderTestUtil.checkValue(providers, row2, Long.toString(now));

    // number parsed as number --> return as time stamp (millis)
    DecoderTestUtil.checkValue(providers, row3, now);
    // string parsed as number --> parse text, convert to timestamp
    DecoderTestUtil.checkValue(providers, row4, now);

    // number parsed as string --> parse text, convert to timestamp, turn into string
    DecoderTestUtil.checkValue(providers, row5, nowString);

    // string parsed as string --> parse text, convert to timestamp, turn into string
    DecoderTestUtil.checkValue(providers, row6, nowString);
}