org.apache.hadoop.hive.serde2.io.TimestampWritable Java Examples

protected TimestampWritable narrowToTimestamp(JsonParser parser)
        throws IOException {
    switch (parser.getCurrentToken()) {
        case VALUE_STRING:
            try {
                return new TimestampWritable(new Timestamp(JsonHelper.parseTimestamp(parser.getValueAsString()).getTime()));
            } catch (Exception e) {
                // String wasn't an ISO8601 timestamp
                return null;
            }
        case VALUE_NUMBER_INT:
            return new TimestampWritable(new Timestamp(parser.getLongValue()));
        case VALUE_NUMBER_FLOAT:
            return new TimestampWritable(new Timestamp((long) parser.getFloatValue()));
        default:
            return null;

    }
}
 

public IntWritable evaluate(TimestampWritable t) {
    if (t == null) {
        return null;
    }

    try {
        Calendar calendar = Calendar.getInstance();
        calendar.setTime(t.getTimestamp());
        LocalDate date = LocalDate.fromCalendarFields(calendar);
        String dateString = date.toString(DEFAULT_DATE_FORMATTER);

        return evaluate(new Text(dateString));
    } catch (Exception e) {
        return null;
    }
}
 

@Override
public Result write(Writable writable, Generator generator) {
    if (writable instanceof ByteWritable) {
        generator.writeNumber(((ByteWritable) writable).get());
    }
    else if (writable instanceof DoubleWritable) {
        generator.writeNumber(((DoubleWritable) writable).get());
    }
    else if (writable instanceof ShortWritable) {
        generator.writeNumber(((ShortWritable) writable).get());
    }
    // HiveDecimal - Hive 0.11+
    else if (writable != null && HiveConstants.DECIMAL_WRITABLE.equals(writable.getClass().getName())) {
        generator.writeString(writable.toString());
    }
    // pass the UNIX epoch
    else if (writable instanceof TimestampWritable) {
        long ts = ((TimestampWritable) writable).getTimestamp().getTime();
        Calendar cal = Calendar.getInstance();
        cal.setTimeInMillis(ts);
        generator.writeString(DatatypeConverter.printDateTime(cal));
    }
    // HiveDate - Hive 0.12+
    else if (writable != null && HiveConstants.DATE_WRITABLE.equals(writable.getClass().getName())) {
        generator.writeString(DateWritableWriter.toES(writable));
    }
    // HiveVarcharWritable - Hive 0.12+
    else if (writable != null && HiveConstants.VARCHAR_WRITABLE.equals(writable.getClass().getName())) {
        generator.writeString(writable.toString());
    }
    // HiveChar - Hive 0.13+
    else if (writable != null && HiveConstants.CHAR_WRITABLE.equals(writable.getClass().getName())) {
        generator.writeString(StringUtils.trim(writable.toString()));
    }
    else {
        return super.write(writable, generator);
    }

    return Result.SUCCESFUL();
}
 

@Test
public void testTimestampTypeConverting() throws UnsupportedDataTypeException {
  Datum testDatum;
  long currentMills = System.currentTimeMillis();

  testDatum = new TimestampDatum(DateTimeUtil.javaTimeToJulianTime(currentMills));

  Writable resultWritable = WritableTypeConverter.convertDatum2Writable(testDatum);
  assertEquals(currentMills / 1000, ((TimestampWritable)resultWritable).getSeconds());

  Datum resultDatum = WritableTypeConverter.convertWritable2Datum(resultWritable);
  assertEquals(testDatum, resultDatum);
}
 

public static Datum convertWritable2Datum(Writable value) throws UnsupportedDataTypeException {
  if (value == null) {
    return NullDatum.get();
  }

  DataType type = convertWritableToTajoType(value.getClass());

  switch(type.getType()) {
    case INT1: return new Int2Datum(((ByteWritable)value).get());
    case INT2: return new Int2Datum(((ShortWritable)value).get());
    case INT4: return new Int4Datum(((IntWritable)value).get());
    case INT8: return new Int8Datum(((LongWritable)value).get());

    case FLOAT4: return new Float4Datum(((FloatWritable)value).get());
    case FLOAT8: return new Float8Datum(((DoubleWritable)value).get());

    case DATE: return new DateDatum(((DateWritable)value).getDays() + DateTimeConstants.UNIX_EPOCH_JDATE);
    case TIMESTAMP: return new TimestampDatum(DateTimeUtil.javaTimeToJulianTime(
        ((TimestampWritable)value).getTimestamp().getTime()));

    case CHAR: return new CharDatum(value.toString());
    case TEXT: return new TextDatum(value.toString());
    case VARBINARY: return new BlobDatum(((BytesWritable)value).getBytes());
  }

  throw new TajoRuntimeException(new UnsupportedDataTypeException(value.getClass().getTypeName()));
}
 

public static Writable convertDatum2Writable(Datum value) {
  switch(value.kind()) {
    case INT1: return new ByteWritable(value.asByte());
    case INT2: return new ShortWritable(value.asInt2());
    case INT4: return new IntWritable(value.asInt4());
    case INT8: return new LongWritable(value.asInt8());

    case FLOAT4: return new FloatWritable(value.asFloat4());
    case FLOAT8: return new DoubleWritable(value.asFloat8());

    // NOTE: value should be DateDatum
    case DATE: return new DateWritable(value.asInt4() - DateTimeConstants.UNIX_EPOCH_JDATE);

    // NOTE: value should be TimestampDatum
    case TIMESTAMP:
      TimestampWritable result = new TimestampWritable();
      result.setTime(DateTimeUtil.julianTimeToJavaTime(value.asInt8()));
      return result;

    case CHAR: {
      String str = value.asChars();
      return new HiveCharWritable(new HiveChar(str, str.length()));
    }
    case TEXT: return new Text(value.asChars());
    case VARBINARY: return new BytesWritable(value.asByteArray());

    case NULL_TYPE: return null;
  }

  throw new TajoRuntimeException(new NotImplementedException(TypeStringEncoder.encode(value.type())));
}
 

@Test
public void TestTimeParse() throws Exception {
	Configuration config = new Configuration();
	Text value = new Text();

	AbstractSerDe jserde = new EsriJsonSerDe();
	Properties proptab = new Properties();
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "when");
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "timestamp");
	jserde.initialize(config, proptab);
       StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();

       value.set("{\"attributes\":{\"when\":\"2020-02-20\"}}");
	Object row = jserde.deserialize(value);
	StructField f0 = rowOI.getStructFieldRef("when");
	Object fieldData = rowOI.getStructFieldData(row, f0);
	Assert.assertEquals(
		new java.text.SimpleDateFormat("yyyy-MM-dd").parse("2020-02-20").getTime(),
		((TimestampWritable)fieldData).getTimestamp().getTime());
       value.set("{\"attributes\":{\"when\":\"2017-05-05 05:05\"}}");
       row = jserde.deserialize(value);
	fieldData = rowOI.getStructFieldData(row, f0);
	Assert.assertEquals(
		new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm").parse("2017-05-05 05:05").getTime(),
		((TimestampWritable)fieldData).getTimestamp().getTime());
       value.set("{\"attributes\":{\"when\":\"2017-08-09 10:11:12\"}}");
       row = jserde.deserialize(value);
	fieldData = rowOI.getStructFieldData(row, f0);
	Assert.assertEquals(
	  new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parse("2017-08-09 10:11:12").getTime(),
	  ((TimestampWritable)fieldData).getTimestamp().getTime());
       value.set("{\"attributes\":{\"when\":\"2017-06-05 04:03:02.123456789\"}}");
       row = jserde.deserialize(value);
	fieldData = rowOI.getStructFieldData(row, f0);
	Assert.assertEquals(
	  new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS").parse("2017-06-05 04:03:02.123").getTime(),
	  ((TimestampWritable)fieldData).getTimestamp().getTime());  // ns parsed but not checked
}
 

public IntWritable evaluate(TimestampWritable t) {
    if (t == null) {
        return null;
    }

    Calendar calendar = Calendar.getInstance();
    calendar.setTime(t.getTimestamp());
    LocalDate date = LocalDate.fromCalendarFields(calendar);
    result.set(date.getDayOfYear());
    return result;
}
 

public IntWritable evaluate(TimestampWritable t) {
    if (t == null) {
        return null;
    }

    Calendar calendar = Calendar.getInstance();
    calendar.setTime(t.getTimestamp());
    LocalDate date = LocalDate.fromCalendarFields(calendar);
    result.set(date.getDayOfWeek());
    return result;
}
 

@Override
public TimestampWritable getPrimitiveWritableObject(Object o) {
<#if mode == "Optional">
  if (o == null) {
    return null;
  }
  final NullableTimeStampMilliHolder h = (NullableTimeStampMilliHolder) o;
<#else>
  final TimeStampMilliHolder h = (TimeStampMilliHolder) o;
</#if>
  return new TimestampWritable(new java.sql.Timestamp(h.value));
}
 

@Override
public void setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
  final TimestampWritable value = ((TimestampObjectInspector)oi).getPrimitiveWritableObject(hiveFieldValue);
  long seconds = value.getSeconds();
  long nanos = value.getNanos();
  long millis = seconds * 1000 + nanos/1000/1000;
  ((TimeStampMilliVector) outputVV).setSafe(outputIndex, millis);
}
 

/**
 * Tests that after copying a lazy timestamp object, calling materialize on the original and the
 * copy doesn't advance the tree reader twice
 * @throws Exception
 */
@Test
public void TestCopyTimestamp() throws Exception {
  ReaderWriterProfiler.setProfilerOptions(null);
  OrcLazyTimestamp lazyTimestamp = new OrcLazyTimestamp(new LazyTimestampTreeReader(0, 0) {
    int nextCalls = 0;

    @Override
    public Object next(Object previous) throws IOException {
      if (nextCalls == 0) {
        return new TimestampWritable(new Timestamp(1));
      }

      throw new IOException("next should only be called once");
    }

    @Override
    protected boolean seekToRow(long currentRow) throws IOException {
      return true;
    }
  });

  TimestampObjectInspector timestampOI = (TimestampObjectInspector)
      OrcLazyObjectInspectorUtils.createLazyObjectInspector(TypeInfoFactory.timestampTypeInfo);

  OrcLazyTimestamp lazyTimestamp2 = (OrcLazyTimestamp) timestampOI.copyObject(lazyTimestamp);

  Assert.assertEquals(new Timestamp(1), ((TimestampWritable) lazyTimestamp.materialize()).getTimestamp());
  Assert.assertEquals(new Timestamp(1), ((TimestampWritable) lazyTimestamp2.materialize()).getTimestamp());
}
 

private static Timestamp getTimestamp(Object object, TimestampObjectInspector inspector)
{
    // handle broken ObjectInspectors
    if (object instanceof TimestampWritable) {
        return ((TimestampWritable) object).getTimestamp();
    }
    return inspector.getPrimitiveJavaObject(object);
}
 

@Override
public Timestamp getPrimitiveJavaObject(Object o) {
  TimestampWritable writable = getPrimitiveWritableObject(o);
  return writable == null ? null : writable.getTimestamp();
}
 

@Test
public void testTimestamp() {
    assertTrue(hiveTypeToJson(
            new MyHiveType(new TimestampWritable(new Timestamp(1407239910771l)), timestampTypeInfo)).startsWith(
                    "\"2014-08-0"));
}
 

@Override
public void write(Writable w) throws IOException {
    ArrayWritable datas = (ArrayWritable) w;
    for (int colId = 0; colId < sqlTypes.length; colId++) {
        SQLType type = sqlTypes[colId];
        Writable currentValue = datas.get()[colId];
        switch (type) {
            case INT:
                if (currentValue == null) {
                    rowBuilder.appendInt(0);
                } else {
                    rowBuilder.appendInt(((IntWritable) currentValue).get());
                }
                break;
            case BIGINT:
                if (currentValue == null) {
                    rowBuilder.appendLong(0L);
                } else {
                    rowBuilder.appendLong(((LongWritable) currentValue).get());
                }
                break;
            case FLOAT:
                if (currentValue == null) {
                    rowBuilder.appendFloat(0f);
                } else {
                    rowBuilder.appendFloat(((FloatWritable) currentValue).get());
                }
                break;
            case DOUBLE:
                if (currentValue == null) {
                    rowBuilder.appendDouble(0d);
                } else {
                    rowBuilder.appendDouble(((DoubleWritable) currentValue).get());
                }
                break;
            case VARCHAR:
                if (currentValue == null) {
                    rowBuilder.appendString("");
                } else {
                    Text v = (Text) currentValue;
                    rowBuilder.appendUTF8String(v.getBytes(), 0, v.getLength());
                }
                break;
            case DATE:
                if (currentValue == null) {
                    rowBuilder.appendLong(0);
                } else {
                    rowBuilder.appendLong(DateTimeUtil.getEpochMillisecond(((DateWritable) currentValue).get()));
                }
                break;
            case DATETIME:
                if (currentValue == null) {
                    rowBuilder.appendLong(0);
                } else {
                    rowBuilder.appendLong(DateTimeUtil.getEpochMillisecond(((TimestampWritable) currentValue).getTimestamp()));
                }
                break;
            default:
                throw new IOException("can't recognize this type [" + type + "]");
        }
    }
    segmentGen.add(rowBuilder.buildAndReset());
}
 

@Override
protected Object parseDate(String value, boolean richDate) {
    return (richDate ? new TimestampWritable(new Timestamp(DatatypeConverter.parseDateTime(value).getTimeInMillis())) : parseString(value));
}
 

@Override
protected Object parseDate(Long value, boolean richDate) {
    return (richDate ? new TimestampWritable(new Timestamp(value)) : processLong(value));
}
 

@Override
protected Class<? extends Writable> dateType() {
    return TimestampWritable.class;
}
 

protected void addWritable(ArrayList<Object> stuff, java.sql.Timestamp item) {
	stuff.add(new TimestampWritable(item));
}
 

/**
 * Copies the Writable at fieldIndex from rowBase to row, then sets the value of the Writable
 * to the value in parser
 * 
 * @param fieldIndex column index of field in row
 * @param parser JsonParser pointing to the attribute
 * @throws JsonParseException
 * @throws IOException
 */
private void setRowFieldFromParser(int fieldIndex, JsonParser parser) throws JsonParseException, IOException{

	PrimitiveObjectInspector poi = (PrimitiveObjectInspector)this.columnOIs.get(fieldIndex);
	if (JsonToken.VALUE_NULL == parser.getCurrentToken())
		return;  // leave the row-cell as null

	// set the field in the row to the writable from rowBase
	row.set(fieldIndex, rowBase.get(fieldIndex));

	switch (poi.getPrimitiveCategory()){
	case BYTE:
		((ByteWritable)row.get(fieldIndex)).set(parser.getByteValue());
		break;
	case SHORT:
		((ShortWritable)row.get(fieldIndex)).set(parser.getShortValue());
		break;
	case INT:
		((IntWritable)row.get(fieldIndex)).set(parser.getIntValue());
		break;
	case LONG:
		((LongWritable)row.get(fieldIndex)).set(parser.getLongValue());
		break;
	case DOUBLE:
		((DoubleWritable)row.get(fieldIndex)).set(parser.getDoubleValue());
		break;
	case FLOAT:
		((FloatWritable)row.get(fieldIndex)).set(parser.getFloatValue());
		break;
	case BOOLEAN:
		((BooleanWritable)row.get(fieldIndex)).set(parser.getBooleanValue());
		break;
	case DATE:    // DateWritable stores days not milliseconds.
		((DateWritable)row.get(fieldIndex)).set(parseDate(parser));
		break;
	case TIMESTAMP:
		((TimestampWritable)row.get(fieldIndex)).set(parseTime(parser));
		break;
	default:    // STRING/unrecognized
		((Text)row.get(fieldIndex)).set(parser.getText());
		break;	
	}
}
 

@SuppressWarnings("rawtypes")
private void compareData(Object expected, Object actual) {
    if (expected instanceof Text) {
        assertEquals(String.class, actual.getClass());
        assertEquals(expected.toString(), actual);
    } else if (expected instanceof ShortWritable) {
        assertEquals(Integer.class, actual.getClass());
        assertEquals((int)((ShortWritable) expected).get(), actual);
    } else if (expected instanceof IntWritable) {
        assertEquals(Integer.class, actual.getClass());
        assertEquals(((IntWritable) expected).get(), actual);
    } else if (expected instanceof LongWritable) {
        assertEquals(Long.class, actual.getClass());
        assertEquals(((LongWritable) expected).get(), actual);
    } else if (expected instanceof FloatWritable) {
        assertEquals(Float.class, actual.getClass());
        assertEquals(((FloatWritable) expected).get(), actual);
    } else if (expected instanceof HiveDecimalWritable) {
        assertEquals(BigDecimal.class, actual.getClass());
        assertEquals(((HiveDecimalWritable) expected).toString(), actual.toString());
    } else if (expected instanceof DoubleWritable) {
        assertEquals(Double.class, actual.getClass());
        assertEquals(((DoubleWritable) expected).get(), actual);
    } else if (expected instanceof BooleanWritable) {
        assertEquals(Boolean.class, actual.getClass());
        assertEquals(((BooleanWritable) expected).get(), actual);
    } else if (expected instanceof TimestampWritable) {
        assertEquals(DateTime.class, actual.getClass());
        assertEquals(((TimestampWritable) expected).getTimestamp().getTime(),
                ((DateTime) actual).getMillis());
    } else if (expected instanceof BytesWritable) {
        assertEquals(DataByteArray.class, actual.getClass());
        BytesWritable bw = (BytesWritable) expected;
        assertEquals(new DataByteArray(bw.getBytes(), 0, bw.getLength()), actual);
    } else if (expected instanceof ByteWritable) {
        assertEquals(Integer.class, actual.getClass());
        assertEquals((int) ((ByteWritable) expected).get(), actual);
    } else if (expected instanceof OrcStruct) {
        assertEquals(BinSedesTuple.class, actual.getClass());
        // TODO: compare actual values. No getters in OrcStruct
    } else if (expected instanceof ArrayList) {
        assertEquals(DefaultDataBag.class, actual.getClass());
        // TODO: compare actual values. No getters in OrcStruct
    } else if (expected instanceof HashMap) {
        assertEquals(HashMap.class, actual.getClass());
        assertEquals(((HashMap) expected).size(), ((HashMap) actual).size());
        // TODO: compare actual values. No getters in OrcStruct
    } else if (expected == null) {
        assertEquals(expected, actual);
    } else {
        Assert.fail("Unknown object type: " + expected.getClass().getName());
    }
}
 

@Override
public TimestampWritable getPrimitiveWritableObject(Object value) {
	return new TimestampWritable((Timestamp)value);
}
 

@Override
public TimestampWritable getPrimitiveWritableObject(Object o) {
    return o == null ? null : new TimestampWritable(new Timestamp(((DateTime)o).getMillis()));
}
 

public static DataType convertWritableToTajoType(Class<? extends Writable> writableClass) throws UnsupportedDataTypeException {
  if (writableClass == null)
    return null;

  Set<Class<?>> parents = ReflectionUtils.getAllSuperTypes(writableClass);

  if (writableClass == ByteWritable.class || parents.contains(ByteWritable.class)) {
    return builder.setType(Type.INT1).build();
  }
  if (writableClass == ShortWritable.class || parents.contains(ShortWritable.class)) {
    return builder.setType(Type.INT2).build();
  }
  if (writableClass == IntWritable.class || parents.contains(IntWritable.class)) {
    return builder.setType(Type.INT4).build();
  }
  if (writableClass == LongWritable.class || parents.contains(LongWritable.class)) {
    return builder.setType(Type.INT8).build();
  }
  if (writableClass == HiveCharWritable.class || parents.contains(HiveCharWritable.class)) {
    return builder.setType(Type.CHAR).build();
  }
  if (writableClass == Text.class || parents.contains(Text.class)) {
    return builder.setType(Type.TEXT).build();
  }
  if (writableClass == FloatWritable.class || parents.contains(FloatWritable.class)) {
    return builder.setType(Type.FLOAT4).build();
  }
  if (writableClass == DoubleWritable.class || parents.contains(DoubleWritable.class)) {
    return builder.setType(Type.FLOAT8).build();
  }
  if (writableClass == DateWritable.class || parents.contains(DateWritable.class)) {
    return builder.setType(Type.DATE).build();
  }
  if (writableClass == TimestampWritable.class || parents.contains(TimestampWritable.class)) {
    return builder.setType(Type.TIMESTAMP).build();
  }
  if (writableClass == BytesWritable.class || parents.contains(BytesWritable.class)) {
    return builder.setType(Type.VARBINARY).build();
  }

  throw new UnsupportedDataTypeException(writableClass.getSimpleName());
}
 

private static Object decodeRecordReaderValue(Type type, Object actualValue)
{
    if (actualValue instanceof BooleanWritable) {
        actualValue = ((BooleanWritable) actualValue).get();
    }
    else if (actualValue instanceof ByteWritable) {
        actualValue = ((ByteWritable) actualValue).get();
    }
    else if (actualValue instanceof BytesWritable) {
        actualValue = new SqlVarbinary(((BytesWritable) actualValue).copyBytes());
    }
    else if (actualValue instanceof DateWritable) {
        actualValue = new SqlDate(((DateWritable) actualValue).getDays());
    }
    else if (actualValue instanceof DoubleWritable) {
        actualValue = ((DoubleWritable) actualValue).get();
    }
    else if (actualValue instanceof FloatWritable) {
        actualValue = ((FloatWritable) actualValue).get();
    }
    else if (actualValue instanceof IntWritable) {
        actualValue = ((IntWritable) actualValue).get();
    }
    else if (actualValue instanceof HiveCharWritable) {
        actualValue = ((HiveCharWritable) actualValue).getPaddedValue().toString();
    }
    else if (actualValue instanceof LongWritable) {
        actualValue = ((LongWritable) actualValue).get();
    }
    else if (actualValue instanceof ShortWritable) {
        actualValue = ((ShortWritable) actualValue).get();
    }
    else if (actualValue instanceof HiveDecimalWritable) {
        DecimalType decimalType = (DecimalType) type;
        HiveDecimalWritable writable = (HiveDecimalWritable) actualValue;
        // writable messes with the scale so rescale the values to the Presto type
        BigInteger rescaledValue = rescale(writable.getHiveDecimal().unscaledValue(), writable.getScale(), decimalType.getScale());
        actualValue = new SqlDecimal(rescaledValue, decimalType.getPrecision(), decimalType.getScale());
    }
    else if (actualValue instanceof Text) {
        actualValue = actualValue.toString();
    }
    else if (actualValue instanceof TimestampWritable) {
        TimestampWritable timestamp = (TimestampWritable) actualValue;
        actualValue = sqlTimestampOf((timestamp.getSeconds() * 1000) + (timestamp.getNanos() / 1000000L), SESSION);
    }
    else if (actualValue instanceof OrcStruct) {
        List<Object> fields = new ArrayList<>();
        OrcStruct structObject = (OrcStruct) actualValue;
        for (int fieldId = 0; fieldId < structObject.getNumFields(); fieldId++) {
            fields.add(OrcUtil.getFieldValue(structObject, fieldId));
        }
        actualValue = decodeRecordReaderStruct(type, fields);
    }
    else if (actualValue instanceof List) {
        actualValue = decodeRecordReaderList(type, ((List<?>) actualValue));
    }
    else if (actualValue instanceof Map) {
        actualValue = decodeRecordReaderMap(type, (Map<?, ?>) actualValue);
    }
    return actualValue;
}
 

@Override
protected TimestampWritable narrow(JsonParser parser)
        throws IOException {
    return narrowToTimestamp(parser);
}
 

public TimestampWritable evaluate(final Text json) {
    return evaluateAndNarrow(json, Intrinsic.LAST_MUTATE_AT);
}
 

@Override
protected TimestampWritable narrow(JsonParser parser)
        throws IOException {
    return narrowToTimestamp(parser);
}
 

public TimestampWritable evaluate(final Text json) {
    return evaluateAndNarrow(json, Intrinsic.FIRST_UPDATE_AT);
}