Java Code Examples for com.facebook.presto.spi.type.VarcharType#VARCHAR

The following examples show how to use com.facebook.presto.spi.type.VarcharType#VARCHAR . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestRawDecoder.java    From presto-kinesis with Apache License 2.0 6 votes vote down vote up 
@Test
public void testFixedWithString()
{
    String str = "Ich bin zwei Oeltanks";
    byte[] row = str.getBytes(StandardCharsets.UTF_8);

    RawKinesisRowDecoder rowDecoder = new RawKinesisRowDecoder();
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", VarcharType.VARCHAR, null, null, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "0", null, null, false, false);
    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", VarcharType.VARCHAR, "0:4", null, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", VarcharType.VARCHAR, "5:8", null, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(row, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    DecoderTestUtil.checkValue(providers, row1, str);
    DecoderTestUtil.checkValue(providers, row2, str);
    // these only work for single byte encodings...
    DecoderTestUtil.checkValue(providers, row3, str.substring(0, 4));
    DecoderTestUtil.checkValue(providers, row4, str.substring(5, 8));
}
Example 2
Source File: TestJsonDecoder.java    From presto-kinesis with Apache License 2.0 6 votes vote down vote up 
@Test
public void testNonExistent()
        throws Exception
{
    byte[] json = "{}".getBytes(StandardCharsets.UTF_8);

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", VarcharType.VARCHAR, "very/deep/varchar", null, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", BigintType.BIGINT, "no_bigint", null, null, false, false);
    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", DoubleType.DOUBLE, "double/is_missing", null, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BooleanType.BOOLEAN, "hello", null, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    DecoderTestUtil.checkIsNull(providers, row1);
    DecoderTestUtil.checkIsNull(providers, row2);
    DecoderTestUtil.checkIsNull(providers, row3);
    DecoderTestUtil.checkIsNull(providers, row4);
}
Example 3
Source File: TestJsonDecoder.java    From presto-kinesis with Apache License 2.0 6 votes vote down vote up 
@Test
public void testStringNumber()
        throws Exception
{
    byte[] json = "{\"a_number\":481516,\"a_string\":\"2342\"}".getBytes(StandardCharsets.UTF_8);

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", VarcharType.VARCHAR, "a_number", null, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", BigintType.BIGINT, "a_number", null, null, false, false);
    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", VarcharType.VARCHAR, "a_string", null, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", null, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    DecoderTestUtil.checkValue(providers, row1, "481516");
    DecoderTestUtil.checkValue(providers, row2, 481516);
    DecoderTestUtil.checkValue(providers, row3, "2342");
    DecoderTestUtil.checkValue(providers, row4, 2342);
}
Example 4
Source File: TestMillisecondsSinceEpochJsonKinesisFieldDecoder.java    From presto-kinesis with Apache License 2.0 5 votes vote down vote up 
@Test
public void testNullValues()
        throws Exception
{
    byte[] json = "{}".getBytes(StandardCharsets.UTF_8);

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);

    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);

    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    // sanity checks
    DecoderTestUtil.checkIsNull(providers, row1);
    DecoderTestUtil.checkIsNull(providers, row2);
    DecoderTestUtil.checkIsNull(providers, row3);
    DecoderTestUtil.checkIsNull(providers, row4);
    DecoderTestUtil.checkIsNull(providers, row5);
    DecoderTestUtil.checkIsNull(providers, row6);
}
Example 5
Source File: TestCsvDecoder.java    From presto-kinesis with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSimple()
{
    String csv = "\"row 1\",row2,\"row3\",100,\"200\",300,4.5";

    CsvKinesisRowDecoder rowDecoder = new CsvKinesisRowDecoder();
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", VarcharType.VARCHAR, "0", null, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "1", null, null, false, false);
    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", VarcharType.VARCHAR, "2", null, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "3", null, null, false, false);
    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", BigintType.BIGINT, "4", null, null, false, false);
    KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", BigintType.BIGINT, "5", null, null, false, false);
    KinesisColumnHandle row7 = new KinesisColumnHandle("", 6, "row7", DoubleType.DOUBLE, "6", null, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6, row7);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(csv.getBytes(StandardCharsets.UTF_8), providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    DecoderTestUtil.checkValue(providers, row1, "row 1");
    DecoderTestUtil.checkValue(providers, row2, "row2");
    DecoderTestUtil.checkValue(providers, row3, "row3");
    DecoderTestUtil.checkValue(providers, row4, 100);
    DecoderTestUtil.checkValue(providers, row5, 200);
    DecoderTestUtil.checkValue(providers, row6, 300);
    DecoderTestUtil.checkValue(providers, row7, 4.5d);
}
Example 6
Source File: TestRawDecoder.java    From presto-kinesis with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("NumericCastThatLosesPrecision")
@Test
public void testFloatStuff()
{
    ByteBuffer buf = ByteBuffer.allocate(100);
    buf.putDouble(Math.PI);
    buf.putFloat((float) Math.E);
    buf.putDouble(Math.E);

    byte[] row = new byte[buf.position()];
    System.arraycopy(buf.array(), 0, row, 0, buf.position());

    RawKinesisRowDecoder rowDecoder = new RawKinesisRowDecoder();
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", VarcharType.VARCHAR, null, "DOUBLE", null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "8", "FLOAT", null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(row, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    DecoderTestUtil.checkValue(providers, row1, Math.PI);
    DecoderTestUtil.checkValue(providers, row2, Math.E);
}
Example 7
Source File: TestRawDecoder.java    From presto-kinesis with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSimple()
{
    ByteBuffer buf = ByteBuffer.allocate(100);
    buf.putLong(4815162342L); // 0 - 7
    buf.putInt(12345678); // 8 - 11
    buf.putShort((short) 4567); // 12 - 13
    buf.put((byte) 123); // 14
    buf.put("Ich bin zwei Oeltanks".getBytes(StandardCharsets.UTF_8)); // 15+

    byte[] row = new byte[buf.position()];
    System.arraycopy(buf.array(), 0, row, 0, buf.position());

    RawKinesisRowDecoder rowDecoder = new RawKinesisRowDecoder();
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "0", "LONG", null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", BigintType.BIGINT, "8", "INT", null, false, false);
    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "12", "SHORT", null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "14", "BYTE", null, false, false);
    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "15", null, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(row, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    DecoderTestUtil.checkValue(providers, row1, 4815162342L);
    DecoderTestUtil.checkValue(providers, row2, 12345678);
    DecoderTestUtil.checkValue(providers, row3, 4567);
    DecoderTestUtil.checkValue(providers, row4, 123);
    DecoderTestUtil.checkValue(providers, row5, "Ich bin zwei Oeltanks");
}
Example 8
Source File: TestRFC2822JsonKinesisFieldDecoder.java    From presto-kinesis with Apache License 2.0 5 votes vote down vote up 
@Test
public void testNullValues()
        throws Exception
{
    byte[] json = "{}".getBytes(StandardCharsets.UTF_8);

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);

    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", RFC2822JsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", RFC2822JsonKinesisFieldDecoder.NAME, null, false, false);

    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", RFC2822JsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", RFC2822JsonKinesisFieldDecoder.NAME, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(json, providers, columns, map(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    // sanity checks
    checkIsNull(providers, row1);
    checkIsNull(providers, row2);
    checkIsNull(providers, row3);
    checkIsNull(providers, row4);
    checkIsNull(providers, row5);
    checkIsNull(providers, row6);
}
Example 9
Source File: TestISO8601JsonKinesisFieldDecoder.java    From presto-kinesis with Apache License 2.0 5 votes vote down vote up 
@Test
public void testNullValues()
        throws Exception
{
    byte[] json = "{}".getBytes(StandardCharsets.UTF_8);

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);

    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false);

    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    // sanity checks
    DecoderTestUtil.checkIsNull(providers, row1);
    DecoderTestUtil.checkIsNull(providers, row2);
    DecoderTestUtil.checkIsNull(providers, row3);
    DecoderTestUtil.checkIsNull(providers, row4);
    DecoderTestUtil.checkIsNull(providers, row5);
    DecoderTestUtil.checkIsNull(providers, row6);
}
Example 10
Source File: TypeHelper.java    From presto-kudu with Apache License 2.0 5 votes vote down vote up 
private static Type fromKuduClientType(org.apache.kudu.Type ktype, ColumnTypeAttributes attributes) {
    switch (ktype) {
        case STRING:
            return VarcharType.VARCHAR;
        case UNIXTIME_MICROS:
            return TimestampType.TIMESTAMP;
        case INT64:
            return BigintType.BIGINT;
        case INT32:
            return IntegerType.INTEGER;
        case INT16:
            return SmallintType.SMALLINT;
        case INT8:
            return TinyintType.TINYINT;
        case FLOAT:
            return RealType.REAL;
        case DOUBLE:
            return DoubleType.DOUBLE;
        case BOOL:
            return BooleanType.BOOLEAN;
        case BINARY:
            return VarbinaryType.VARBINARY;
        case DECIMAL:
            return DecimalType.createDecimalType(attributes.getPrecision(), attributes.getScale());
        default:
            throw new IllegalStateException("Kudu type not implemented for " + ktype);
    }
}
Example 11
Source File: TestSecondsSinceEpochJsonKinesisFieldDecoder.java    From presto-kinesis with Apache License 2.0 5 votes vote down vote up 
@Test
public void testNullValues()
        throws Exception
{
    byte[] json = "{}".getBytes(StandardCharsets.UTF_8);

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);

    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", SecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", SecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);

    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", SecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", SecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    // sanity checks
    DecoderTestUtil.checkIsNull(providers, row1);
    DecoderTestUtil.checkIsNull(providers, row2);
    DecoderTestUtil.checkIsNull(providers, row3);
    DecoderTestUtil.checkIsNull(providers, row4);
    DecoderTestUtil.checkIsNull(providers, row5);
    DecoderTestUtil.checkIsNull(providers, row6);
}
Example 12
Source File: TestJsonDecoder.java    From presto-kinesis with Apache License 2.0 5 votes vote down vote up 
@Test
public void testOtherExtracts()
        throws Exception
{
    // Test other scenarios: deeper dive into object, get JSON constructs as strings, etc.
    byte[] json = ByteStreams.toByteArray(TestJsonDecoder.class.getResourceAsStream("/decoder/json/event.json"));

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "event_source", VarcharType.VARCHAR, "source", null, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "user", VarcharType.VARCHAR, "user/handle", null, null, false, false);
    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "user_string", VarcharType.VARCHAR, "user", null, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "timestamp", BigintType.BIGINT, "timestamp", null, null, false, false);
    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "browser_name", VarcharType.VARCHAR, "environment/browser/name", null, null, false, false);
    KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "tags_array", VarcharType.VARCHAR, "tags", null, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    log.info("Decoding row from event JSON file");
    boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    DecoderTestUtil.checkValue(providers, row1, "otherworld");
    DecoderTestUtil.checkValue(providers, row2, "joeblow");
    DecoderTestUtil.checkValue(providers, row3, "{\"email\":\"[email protected]\",\"handle\":\"joeblow\"}");
    KinesisFieldValueProvider provider = DecoderTestUtil.findValueProvider(providers, row6);
    assertNotNull(provider);
    log.info(new String(provider.getSlice().getBytes(), StandardCharsets.UTF_8));

    DecoderTestUtil.checkValue(providers, row4, 1450214872847L);
    DecoderTestUtil.checkValue(providers, row5, "Chrome");
    DecoderTestUtil.checkValue(providers, row6, "[\"tag1\",\"tag2\",\"tag3\"]");
    log.info("DONE");
}
Example 13
Source File: TestJsonDecoder.java    From presto-kinesis with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSimple()
        throws Exception
{
    byte[] json = ByteStreams.toByteArray(TestJsonDecoder.class.getResourceAsStream("/decoder/json/message.json"));

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", VarcharType.VARCHAR, "source", null, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "user/screen_name", null, null, false, false);
    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "id", null, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "user/statuses_count", null, null, false, false);
    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", BooleanType.BOOLEAN, "user/geo_enabled", null, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    DecoderTestUtil.checkValue(providers, row1, "<a href=\"http://twitterfeed.com\" rel=\"nofollow\">twitterfeed</a>");
    DecoderTestUtil.checkValue(providers, row2, "EKentuckyNews");
    DecoderTestUtil.checkValue(providers, row3, 493857959588286460L);
    DecoderTestUtil.checkValue(providers, row4, 7630);
    DecoderTestUtil.checkValue(providers, row5, true);
}
Example 14
Source File: TypeHelper.java    From presto-kudu with Apache License 2.0 5 votes vote down vote up 
public static Type mappedType(Type sourceType) {
    if (sourceType == DateType.DATE) {
        return VarcharType.VARCHAR;
    } else {
        return sourceType;
    }
}
Example 15
Source File: TestISO8601JsonKinesisFieldDecoder.java    From presto-kinesis with Apache License 2.0 4 votes vote down vote up 
@Test
public void testBasicFormatting()
        throws Exception
{
    long now = System.currentTimeMillis();
    String nowString = PRINTER.print(now);

    byte[] json = format("{\"a_number\":%d,\"a_string\":\"%s\"}", now, nowString).getBytes(StandardCharsets.UTF_8);

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);

    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false);

    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    // sanity checks
    DecoderTestUtil.checkValue(providers, row1, now);
    DecoderTestUtil.checkValue(providers, row2, nowString);

    // number parsed as number --> as is
    DecoderTestUtil.checkValue(providers, row3, now);
    // string parsed as number --> parse text, convert to timestamp
    DecoderTestUtil.checkValue(providers, row4, now);

    // number parsed as string --> parse text, convert to timestamp, turn into string
    DecoderTestUtil.checkValue(providers, row5, Long.toString(now));

    // string parsed as string --> as is
    DecoderTestUtil.checkValue(providers, row6, nowString);
}
Example 16
Source File: TestMillisecondsSinceEpochJsonKinesisFieldDecoder.java    From presto-kinesis with Apache License 2.0 4 votes vote down vote up 
@Test
public void testBasicFormatting()
        throws Exception
{
    long now = System.currentTimeMillis();
    String nowString = MillisecondsSinceEpochJsonKinesisFieldDecoder.FORMATTER.print(now);

    byte[] json = format("{\"a_number\":%d,\"a_string\":\"%d\"}", now, now).getBytes(StandardCharsets.UTF_8);

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);

    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);

    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    // sanity checks
    DecoderTestUtil.checkValue(providers, row1, now);
    DecoderTestUtil.checkValue(providers, row2, Long.toString(now));

    // number parsed as number --> return as time stamp (millis)
    DecoderTestUtil.checkValue(providers, row3, now);
    // string parsed as number --> parse text, convert to timestamp
    DecoderTestUtil.checkValue(providers, row4, now);

    // number parsed as string --> parse text, convert to timestamp, turn into string
    DecoderTestUtil.checkValue(providers, row5, nowString);

    // string parsed as string --> parse text, convert to timestamp, turn into string
    DecoderTestUtil.checkValue(providers, row6, nowString);
}
Example 17
Source File: TestRFC2822JsonKinesisFieldDecoder.java    From presto-kinesis with Apache License 2.0 4 votes vote down vote up 
@Test
public void testBasicFormatting()
        throws Exception
{
    long now = (System.currentTimeMillis() / 1000) * 1000; // rfc2822 is second granularity
    String nowString = FORMATTER.print(now);

    byte[] json = format("{\"a_number\":%d,\"a_string\":\"%s\"}", now, nowString).getBytes(StandardCharsets.UTF_8);

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);

    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", RFC2822JsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", RFC2822JsonKinesisFieldDecoder.NAME, null, false, false);

    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", RFC2822JsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", RFC2822JsonKinesisFieldDecoder.NAME, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(json, providers, columns, map(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    // sanity checks
    checkValue(providers, row1, now);
    checkValue(providers, row2, nowString);

    // number parsed as number --> as is
    checkValue(providers, row3, now);
    // string parsed as number --> parse text, convert to timestamp
    checkValue(providers, row4, now);

    // number parsed as string --> parse text, convert to timestamp, turn into string
    checkValue(providers, row5, Long.toString(now));

    // string parsed as string --> as is
    checkValue(providers, row6, nowString);
}
Example 18
Source File: TestSecondsSinceEpochJsonKinesisFieldDecoder.java    From presto-kinesis with Apache License 2.0 4 votes vote down vote up 
@Test
public void testBasicFormatting()
        throws Exception
{
    long now = System.currentTimeMillis() / 1000; // SecondsSinceEpoch is second granularity
    String nowString = SecondsSinceEpochJsonKinesisFieldDecoder.FORMATTER.print(now * 1000);

    byte[] json = format("{\"a_number\":%d,\"a_string\":\"%d\"}", now, now).getBytes(StandardCharsets.UTF_8);

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);

    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", SecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", SecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);

    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", SecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", SecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    // sanity checks
    DecoderTestUtil.checkValue(providers, row1, now);
    DecoderTestUtil.checkValue(providers, row2, Long.toString(now));

    // number parsed as number --> return as time stamp (millis)
    DecoderTestUtil.checkValue(providers, row3, now * 1000);
    // string parsed as number --> parse text, convert to timestamp
    DecoderTestUtil.checkValue(providers, row4, now * 1000);

    // number parsed as string --> parse text, convert to timestamp, turn into string
    DecoderTestUtil.checkValue(providers, row5, nowString);

    // string parsed as string --> parse text, convert to timestamp, turn into string
    DecoderTestUtil.checkValue(providers, row6, nowString);
}