Java Code Examples for org.apache.pig.data.DataType#DATETIME
The following examples show how to use
org.apache.pig.data.DataType#DATETIME .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: DBStorage.java From spork with Apache License 2.0 | 6 votes |
|
protected int sqlDataTypeFromPigDataType(byte pigDataType) {
switch(pigDataType) {
case DataType.INTEGER:
return java.sql.Types.INTEGER;
case DataType.LONG:
return java.sql.Types.BIGINT;
case DataType.FLOAT:
return java.sql.Types.FLOAT;
case DataType.DOUBLE:
return java.sql.Types.DOUBLE;
case DataType.BOOLEAN:
return java.sql.Types.BOOLEAN;
case DataType.DATETIME:
return java.sql.Types.DATE;
case DataType.BYTEARRAY:
case DataType.CHARARRAY:
case DataType.BYTE:
return java.sql.Types.VARCHAR;
default:
log.warn("Can not find SQL data type for " + pigDataType + " returning VARCHAR");
return java.sql.Types.VARCHAR;
}
}
Example 2
| Source File: SchemaTupleClassGenerator.java From spork with Apache License 2.0 | 6 votes |
|
@Override
public void process(int fieldPos, Schema.FieldSchema fs) {
add("public "+typeName()+" getDummy_"+fieldPos+"() {");
switch (fs.type) {
case (DataType.INTEGER): add(" return 0;"); break;
case (DataType.LONG): add(" return 0L;"); break;
case (DataType.FLOAT): add(" return 0.0f;"); break;
case (DataType.DOUBLE): add(" return 0.0;"); break;
case (DataType.BOOLEAN): add(" return true;"); break;
case (DataType.DATETIME): add(" return new DateTime();"); break;
case (DataType.BIGDECIMAL): add(" return (BigDecimal)null;"); break;
case (DataType.BIGINTEGER): add(" return (BigInteger)null;"); break;
case (DataType.BYTEARRAY): add(" return (byte[])null;"); break;
case (DataType.CHARARRAY): add(" return (String)null;"); break;
case (DataType.TUPLE): add(" return (Tuple)null;"); break;
case (DataType.BAG): add(" return (DataBag)null;"); break;
case (DataType.MAP): add(" return (Map<String,Object>)null;"); break;
default: throw new RuntimeException("Unsupported type");
}
add("}");
addBreak();
}
Example 3
| Source File: CastUtils.java From spork with Apache License 2.0 | 6 votes |
|
/**
*
* @param caster LoadCaster to be used to convert the bytes into a field.
* @param bytes
* @param fieldSchema schema of Bag or Tuple; pass in null if a simple type.
* @param dataType type from DataType
* @return converted object.
* @throws IOException
*/
public static Object convertToType(LoadCaster caster, byte[] bytes,
ResourceFieldSchema fieldSchema, byte dataType) throws IOException {
switch (dataType) {
case (DataType.BAG): return caster.bytesToBag(bytes, fieldSchema);
case (DataType.BYTEARRAY): return new DataByteArray(bytes);
case (DataType.CHARARRAY): return caster.bytesToCharArray(bytes);
case (DataType.DOUBLE): return caster.bytesToDouble(bytes);
case (DataType.FLOAT): return caster.bytesToFloat(bytes);
case (DataType.INTEGER): return caster.bytesToInteger(bytes);
case (DataType.BIGINTEGER): return caster.bytesToBigInteger(bytes);
case (DataType.BIGDECIMAL): return caster.bytesToBigDecimal(bytes);
case (DataType.LONG): return caster.bytesToLong(bytes);
case (DataType.BOOLEAN): return caster.bytesToBoolean(bytes);
case (DataType.DATETIME): return caster.bytesToDateTime(bytes);
case (DataType.MAP): return caster.bytesToMap(bytes, fieldSchema);
case (DataType.TUPLE): return caster.bytesToTuple(bytes, fieldSchema);
default: throw new IOException("Unknown type " + dataType);
}
}
Example 4
| Source File: OrcStorage.java From spork with Apache License 2.0 | 6 votes |
|
@Override
public List<String> getPredicateFields(String location, Job job) throws IOException {
ResourceSchema schema = getSchema(location, job);
List<String> predicateFields = new ArrayList<String>();
for (ResourceFieldSchema field : schema.getFields()) {
switch(field.getType()) {
case DataType.BOOLEAN:
case DataType.INTEGER:
case DataType.LONG:
case DataType.FLOAT:
case DataType.DOUBLE:
case DataType.DATETIME:
case DataType.CHARARRAY:
case DataType.BIGINTEGER:
case DataType.BIGDECIMAL:
predicateFields.add(field.getName());
break;
default:
// Skip DataType.BYTEARRAY, DataType.TUPLE, DataType.MAP and DataType.BAG
break;
}
}
return predicateFields;
}
Example 5
| Source File: Sty.java From validatar with Apache License 2.0 | 5 votes |
|
private TypedObject getTypedObject(Object data, FieldDetail detail) throws ExecException {
if (data == null) {
return null;
}
byte type = detail.type;
switch (type) {
case DataType.BOOLEAN:
return TypeSystem.asTypedObject(DataType.toBoolean(data, type));
case DataType.INTEGER:
case DataType.LONG:
return TypeSystem.asTypedObject(DataType.toLong(data, type));
case DataType.FLOAT:
case DataType.DOUBLE:
return TypeSystem.asTypedObject(DataType.toDouble(data, type));
case DataType.DATETIME:
return TypeSystem.asTypedObject(new Timestamp(DataType.toDateTime(data, type).getMillis()));
case DataType.BYTE:
case DataType.BYTEARRAY:
case DataType.CHARARRAY:
return TypeSystem.asTypedObject(DataType.toString(data, type));
case DataType.BIGINTEGER:
case DataType.BIGDECIMAL:
return TypeSystem.asTypedObject(DataType.toBigDecimal(data, type));
default:
//TUPLE, BAG, MAP, INTERNALMAP, GENERIC_WRITABLECOMPARABLE, ERROR, UNKNOWN, NULL and anything else
return null;
}
}
Example 6
| Source File: NotEqualToExpr.java From spork with Apache License 2.0 | 5 votes |
|
@Override
public Result getNextBoolean() throws ExecException {
Result left, right;
switch (operandType) {
case DataType.BYTEARRAY:
case DataType.DOUBLE:
case DataType.FLOAT:
case DataType.BOOLEAN:
case DataType.INTEGER:
case DataType.BIGINTEGER:
case DataType.BIGDECIMAL:
case DataType.LONG:
case DataType.DATETIME:
case DataType.CHARARRAY:
case DataType.TUPLE:
case DataType.MAP: {
Result r = accumChild(null, operandType);
if (r != null) {
return r;
}
left = lhs.getNext(operandType);
right = rhs.getNext(operandType);
return doComparison(left, right);
}
default: {
int errCode = 2067;
String msg = this.getClass().getSimpleName() + " does not know how to " +
"handle type: " + DataType.findTypeName(operandType);
throw new ExecException(msg, errCode, PigException.BUG);
}
}
}
Example 7
| Source File: GTOrEqualToExpr.java From spork with Apache License 2.0 | 5 votes |
|
@Override
public Result getNextBoolean() throws ExecException {
Result left, right;
switch (operandType) {
case DataType.BYTEARRAY:
case DataType.DOUBLE:
case DataType.FLOAT:
case DataType.INTEGER:
case DataType.LONG:
case DataType.BIGINTEGER:
case DataType.BIGDECIMAL:
case DataType.DATETIME:
case DataType.CHARARRAY: {
Result r = accumChild(null, operandType);
if (r != null) {
return r;
}
left = lhs.getNext(operandType);
right = rhs.getNext(operandType);
return doComparison(left, right);
}
default: {
int errCode = 2067;
String msg = this.getClass().getSimpleName() + " does not know how to " +
"handle type: " + DataType.findTypeName(operandType);
throw new ExecException(msg, errCode, PigException.BUG);
}
}
}
Example 8
| Source File: HDataType.java From spork with Apache License 2.0 | 5 votes |
|
public static byte findTypeFromNullableWritable(PigNullableWritable o) throws ExecException {
if (o instanceof NullableBooleanWritable)
return DataType.BOOLEAN;
else if (o instanceof NullableBytesWritable)
return DataType.BYTEARRAY;
else if (o instanceof NullableText)
return DataType.CHARARRAY;
else if (o instanceof NullableFloatWritable)
return DataType.FLOAT;
else if (o instanceof NullableDoubleWritable)
return DataType.DOUBLE;
else if (o instanceof NullableIntWritable)
return DataType.INTEGER;
else if (o instanceof NullableLongWritable)
return DataType.LONG;
else if (o instanceof NullableBigIntegerWritable)
return DataType.BIGINTEGER;
else if (o instanceof NullableBigDecimalWritable)
return DataType.BIGDECIMAL;
else if (o instanceof NullableDateTimeWritable)
return DataType.DATETIME;
else if (o instanceof NullableBag)
return DataType.BAG;
else if (o instanceof NullableTuple)
return DataType.TUPLE;
else {
int errCode = 2044;
String msg = "Cannot find Pig type for " + o.getClass().getName();
throw new ExecException(msg, errCode, PigException.BUG);
}
}
Example 9
| Source File: PigStreamingUDF.java From spork with Apache License 2.0 | 4 votes |
|
private Object deserialize(FieldSchema fs, byte[] bytes, int startIndex, int endIndex) throws IOException {
//If null, return null;
if (WritableComparator.compareBytes(
bytes, startIndex, DELIMS.getNull().length,
DELIMS.getNull(), 0, DELIMS.getNull().length) == 0) {
return null;
}
if (fs.type == DataType.BAG) {
return deserializeBag(fs, bytes, startIndex + 3, endIndex - 2);
} else if (fs.type == DataType.TUPLE) {
return deserializeTuple(fs, bytes, startIndex + 3, endIndex - 2);
} else if (fs.type == DataType.MAP) {
return deserializeMap(bytes, startIndex + 3, endIndex - 2);
}
if (fs.type == DataType.CHARARRAY) {
return extractString(bytes, startIndex, endIndex, true);
} else if (fs.type == DataType.BYTEARRAY) {
return new DataByteArray(bytes, startIndex, endIndex+1);
}
//Can we do this faster?
String val = extractString(bytes, startIndex, endIndex, false);
if (fs.type == DataType.LONG) {
return Long.valueOf(val);
} else if (fs.type == DataType.INTEGER) {
return Integer.valueOf(val);
} else if (fs.type == DataType.FLOAT) {
return Float.valueOf(val);
} else if (fs.type == DataType.DOUBLE) {
return Double.valueOf(val);
} else if (fs.type == DataType.BOOLEAN) {
return Boolean.valueOf(val);
} else if (fs.type == DataType.DATETIME) {
return ToDate.extractDateTime(val);
} else if (fs.type == DataType.BIGINTEGER) {
return new BigInteger(val);
} else if (fs.type == DataType.BIGDECIMAL) {
return new BigDecimal(val);
} else {
throw new ExecException("Can't deserialize type: " + DataType.findTypeName(fs.type));
}
}
Example 10
| Source File: VespaQuerySchema.java From vespa with Apache License 2.0 | 4 votes |
|
public Tuple buildTuple(int rank, JsonNode hit) {
Tuple tuple = TupleFactory.getInstance().newTuple();
for (VespaQuerySchema.AliasTypePair tupleElement : tupleSchema) {
String alias = tupleElement.getAlias();
Byte type = DataType.findTypeByName(tupleElement.getType());
// reserved word
if ("rank".equals(alias)) {
tuple.append(rank);
} else {
JsonNode field = hit;
String[] path = alias.split("/"); // move outside
for (String p : path) {
field = field.get(p);
if (field == null) {
type = DataType.NULL; // effectively skip field as it is not found
break;
}
}
switch (type) {
case DataType.BOOLEAN:
tuple.append(field.asBoolean());
break;
case DataType.INTEGER:
tuple.append(field.asInt());
break;
case DataType.LONG:
tuple.append(field.asLong());
break;
case DataType.FLOAT:
case DataType.DOUBLE:
tuple.append(field.asDouble());
break;
case DataType.DATETIME:
tuple.append(field.asText());
break;
case DataType.CHARARRAY:
tuple.append(field.asText());
break;
default:
// the rest of the data types are currently not supported
}
}
}
return tuple;
}
Example 11
| Source File: AugmentBaseDataVisitor.java From spork with Apache License 2.0 | 4 votes |
|
Object GetLargerValue(Object v) {
byte type = DataType.findType(v);
if (type == DataType.BAG || type == DataType.TUPLE
|| type == DataType.MAP)
return null;
switch (type) {
case DataType.CHARARRAY:
return (String) v + "0";
case DataType.BYTEARRAY:
String str = ((DataByteArray) v).toString();
str = str + "0";
return new DataByteArray(str);
case DataType.INTEGER:
return Integer.valueOf((Integer) v + 1);
case DataType.LONG:
return Long.valueOf((Long) v + 1);
case DataType.FLOAT:
return Float.valueOf((Float) v + 1);
case DataType.DOUBLE:
return Double.valueOf((Double) v + 1);
case DataType.BIGINTEGER:
return ((BigInteger)v).add(BigInteger.ONE);
case DataType.BIGDECIMAL:
return ((BigDecimal)v).add(BigDecimal.ONE);
case DataType.DATETIME:
DateTime dt = (DateTime) v;
if (dt.getMillisOfSecond() != 0) {
return dt.plusMillis(1);
} else if (dt.getSecondOfMinute() != 0) {
return dt.plusSeconds(1);
} else if (dt.getMinuteOfHour() != 0) {
return dt.plusMinutes(1);
} else if (dt.getHourOfDay() != 0) {
return dt.plusHours(1);
} else {
return dt.plusDays(1);
}
default:
return null;
}
}
Example 12
| Source File: SubtractDuration.java From spork with Apache License 2.0 | 4 votes |
|
@Override
public Schema outputSchema(Schema input) {
return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), input), DataType.DATETIME));
}
Example 13
| Source File: DateTimeMax.java From spork with Apache License 2.0 | 4 votes |
|
@Override
public Schema outputSchema(Schema input) {
return new Schema(new Schema.FieldSchema(null, DataType.DATETIME));
}
Example 14
| Source File: TezDagBuilder.java From spork with Apache License 2.0 | 4 votes |
|
private static Class<? extends WritableComparator> getGroupingComparatorForKeyType(byte keyType)
throws JobCreationException {
switch (keyType) {
case DataType.BOOLEAN:
return PigGroupingBooleanWritableComparator.class;
case DataType.INTEGER:
return PigGroupingIntWritableComparator.class;
case DataType.BIGINTEGER:
return PigGroupingBigIntegerWritableComparator.class;
case DataType.BIGDECIMAL:
return PigGroupingBigDecimalWritableComparator.class;
case DataType.LONG:
return PigGroupingLongWritableComparator.class;
case DataType.FLOAT:
return PigGroupingFloatWritableComparator.class;
case DataType.DOUBLE:
return PigGroupingDoubleWritableComparator.class;
case DataType.DATETIME:
return PigGroupingDateTimeWritableComparator.class;
case DataType.CHARARRAY:
return PigGroupingCharArrayWritableComparator.class;
case DataType.BYTEARRAY:
return PigGroupingDBAWritableComparator.class;
case DataType.MAP:
int errCode = 1068;
String msg = "Using Map as key not supported.";
throw new JobCreationException(msg, errCode, PigException.INPUT);
case DataType.TUPLE:
return PigGroupingTupleWritableComparator.class;
case DataType.BAG:
errCode = 1068;
msg = "Using Bag as key not supported.";
throw new JobCreationException(msg, errCode, PigException.INPUT);
default:
errCode = 2036;
msg = "Unhandled key type " + DataType.findTypeName(keyType);
throw new JobCreationException(msg, errCode, PigException.BUG);
}
}
Example 15
| Source File: ToDate.java From spork with Apache License 2.0 | 4 votes |
|
@Override
public Schema outputSchema(Schema input) {
return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass()
.getName().toLowerCase(), input), DataType.DATETIME));
}
Example 16
| Source File: TezDagBuilder.java From spork with Apache License 2.0 | 4 votes |
|
private static Class<? extends WritableComparator> comparatorForKeyType(byte keyType, boolean hasOrderBy)
throws JobCreationException {
switch (keyType) {
case DataType.BOOLEAN:
return PigBooleanRawComparator.class;
case DataType.INTEGER:
return PigIntRawComparator.class;
case DataType.BIGINTEGER:
return PigBigIntegerRawComparator.class;
case DataType.BIGDECIMAL:
return PigBigDecimalRawComparator.class;
case DataType.LONG:
return PigLongRawComparator.class;
case DataType.FLOAT:
return PigFloatRawComparator.class;
case DataType.DOUBLE:
return PigDoubleRawComparator.class;
case DataType.DATETIME:
return PigDateTimeRawComparator.class;
case DataType.CHARARRAY:
return PigTextRawComparator.class;
case DataType.BYTEARRAY:
//if (hasOrderBy) {
return PigBytesRawComparator.class;
//} else {
// return PigDBAWritableComparator.class;
//}
case DataType.MAP:
int errCode = 1068;
String msg = "Using Map as key not supported.";
throw new JobCreationException(msg, errCode, PigException.INPUT);
case DataType.TUPLE:
//TODO: PigTupleWritableComparator gives wrong results with cogroup in
//Checkin_2 and few other e2e tests. But MR has PigTupleWritableComparator
//Investigate the difference later
//if (hasOrderBy) {
return PigTupleSortComparator.class;
//} else {
// return PigTupleWritableComparator.class;
//}
case DataType.BAG:
errCode = 1068;
msg = "Using Bag as key not supported.";
throw new JobCreationException(msg, errCode, PigException.INPUT);
default:
errCode = 2036;
msg = "Unhandled key type " + DataType.findTypeName(keyType);
throw new JobCreationException(msg, errCode, PigException.BUG);
}
}
Example 17
| Source File: SchemaTupleClassGenerator.java From spork with Apache License 2.0 | 4 votes |
|
public boolean isDateTime() {
return type == DataType.DATETIME;
}
Example 18
| Source File: TypeCheckingExpVisitor.java From spork with Apache License 2.0 | 4 votes |
|
/***
* Helper for collecting warning when casting is inserted
* to the plan (implicit casting)
*
* @param node
* @param originalType
* @param toType
*/
static void collectCastWarning(Operator node,
byte originalType,
byte toType,
CompilationMessageCollector msgCollector
) {
String originalTypeName = DataType.findTypeName(originalType) ;
String toTypeName = DataType.findTypeName(toType) ;
String opName= node.getClass().getSimpleName() ;
PigWarning kind = null;
switch(toType) {
case DataType.BAG:
kind = PigWarning.IMPLICIT_CAST_TO_BAG;
break;
case DataType.CHARARRAY:
kind = PigWarning.IMPLICIT_CAST_TO_CHARARRAY;
break;
case DataType.DOUBLE:
kind = PigWarning.IMPLICIT_CAST_TO_DOUBLE;
break;
case DataType.FLOAT:
kind = PigWarning.IMPLICIT_CAST_TO_FLOAT;
break;
case DataType.INTEGER:
kind = PigWarning.IMPLICIT_CAST_TO_INT;
break;
case DataType.LONG:
kind = PigWarning.IMPLICIT_CAST_TO_LONG;
break;
case DataType.BOOLEAN:
kind = PigWarning.IMPLICIT_CAST_TO_BOOLEAN;
break;
case DataType.DATETIME:
kind = PigWarning.IMPLICIT_CAST_TO_DATETIME;
break;
case DataType.MAP:
kind = PigWarning.IMPLICIT_CAST_TO_MAP;
break;
case DataType.TUPLE:
kind = PigWarning.IMPLICIT_CAST_TO_TUPLE;
break;
case DataType.BIGINTEGER:
kind = PigWarning.IMPLICIT_CAST_TO_BIGINTEGER;
break;
case DataType.BIGDECIMAL:
kind = PigWarning.IMPLICIT_CAST_TO_BIGDECIMAL;
break;
}
msgCollector.collect(originalTypeName + " is implicitly cast to "
+ toTypeName +" under " + opName + " Operator",
MessageType.Warning, kind) ;
}
Example 19
| Source File: AugmentBaseDataVisitor.java From spork with Apache License 2.0 | 4 votes |
|
Object GetSmallerValue(Object v) {
byte type = DataType.findType(v);
if (type == DataType.BAG || type == DataType.TUPLE
|| type == DataType.MAP)
return null;
switch (type) {
case DataType.CHARARRAY:
String str = (String) v;
if (str.length() > 0)
return str.substring(0, str.length() - 1);
else
return null;
case DataType.BYTEARRAY:
DataByteArray data = (DataByteArray) v;
if (data.size() > 0)
return new DataByteArray(data.get(), 0, data.size() - 1);
else
return null;
case DataType.INTEGER:
return Integer.valueOf((Integer) v - 1);
case DataType.LONG:
return Long.valueOf((Long) v - 1);
case DataType.FLOAT:
return Float.valueOf((Float) v - 1);
case DataType.DOUBLE:
return Double.valueOf((Double) v - 1);
case DataType.BIGINTEGER:
return ((BigInteger)v).subtract(BigInteger.ONE);
case DataType.BIGDECIMAL:
return ((BigDecimal)v).subtract(BigDecimal.ONE);
case DataType.DATETIME:
DateTime dt = (DateTime) v;
if (dt.getMillisOfSecond() != 0) {
return dt.minusMillis(1);
} else if (dt.getSecondOfMinute() != 0) {
return dt.minusSeconds(1);
} else if (dt.getMinuteOfHour() != 0) {
return dt.minusMinutes(1);
} else if (dt.getHourOfDay() != 0) {
return dt.minusHours(1);
} else {
return dt.minusDays(1);
}
default:
return null;
}
}
Example 20
| Source File: AbstractAccumuloStorage.java From spork with Apache License 2.0 | 4 votes |
|
@SuppressWarnings("unchecked")
protected byte[] objToBytes(Object o, byte type) throws IOException {
if (o == null)
return null;
switch (type) {
case DataType.BYTEARRAY:
return ((DataByteArray) o).get();
case DataType.BAG:
return caster.toBytes((DataBag) o);
case DataType.CHARARRAY:
return caster.toBytes((String) o);
case DataType.DOUBLE:
return caster.toBytes((Double) o);
case DataType.FLOAT:
return caster.toBytes((Float) o);
case DataType.INTEGER:
return caster.toBytes((Integer) o);
case DataType.LONG:
return caster.toBytes((Long) o);
case DataType.BIGINTEGER:
return caster.toBytes((BigInteger) o);
case DataType.BIGDECIMAL:
return caster.toBytes((BigDecimal) o);
case DataType.BOOLEAN:
return caster.toBytes((Boolean) o);
case DataType.DATETIME:
return caster.toBytes((DateTime) o);
// The type conversion here is unchecked.
// Relying on DataType.findType to do the right thing.
case DataType.MAP:
return caster.toBytes((Map<String, Object>) o);
case DataType.NULL:
return null;
case DataType.TUPLE:
return caster.toBytes((Tuple) o);
case DataType.ERROR:
throw new IOException("Unable to determine type of " + o.getClass());
default:
throw new IOException("Unable to find a converter for tuple field "
+ o);
}
}
