Java Code Examples for org.apache.pig.data.DataType#BIGCHARARRAY
The following examples show how to use
org.apache.pig.data.DataType#BIGCHARARRAY .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: PigSchema2Avro.java From Cubert with Apache License 2.0 | 6 votes |
|
/**
* Convert Pig primitive type to Avro type
*
*/
protected static Schema convertPrimitiveType(byte pigType) throws IOException {
if (pigType == DataType.BOOLEAN) {
return AvroStorageUtils.BooleanSchema;
} else if (pigType == DataType.BYTEARRAY) {
return AvroStorageUtils.BytesSchema;
} else if (pigType == DataType.CHARARRAY
|| pigType == DataType.BIGCHARARRAY) {
return AvroStorageUtils.StringSchema;
} else if (pigType == DataType.DOUBLE) {
return AvroStorageUtils.DoubleSchema;
} else if (pigType == DataType.FLOAT) {
return AvroStorageUtils.FloatSchema;
} else if (pigType == DataType.INTEGER) {
return AvroStorageUtils.IntSchema;
} else if (pigType == DataType.LONG) {
return AvroStorageUtils.LongSchema;
} else
throw new IOException("unsupported pig type:"
+ DataType.findTypeName(pigType));
}
Example 2
| Source File: PigSchema2Avro.java From spork with Apache License 2.0 | 6 votes |
|
/**
* Convert Pig primitive type to Avro type
*
*/
protected static Schema convertPrimitiveType(byte pigType) throws IOException {
if (pigType == DataType.BOOLEAN) {
return AvroStorageUtils.BooleanSchema;
} else if (pigType == DataType.BYTEARRAY) {
return AvroStorageUtils.BytesSchema;
} else if (pigType == DataType.CHARARRAY
|| pigType == DataType.BIGCHARARRAY) {
return AvroStorageUtils.StringSchema;
} else if (pigType == DataType.DOUBLE) {
return AvroStorageUtils.DoubleSchema;
} else if (pigType == DataType.FLOAT) {
return AvroStorageUtils.FloatSchema;
} else if (pigType == DataType.INTEGER) {
return AvroStorageUtils.IntSchema;
} else if (pigType == DataType.LONG) {
return AvroStorageUtils.LongSchema;
} else
throw new IOException("unsupported pig type:"
+ DataType.findTypeName(pigType));
}
Example 3
| Source File: PigSchema2Avro.java From Cubert with Apache License 2.0 | 4 votes |
|
/**
* Check whether Avro type is compatible with Pig type
*
*/
protected static boolean isCompatible(Schema avroSchema, ResourceFieldSchema pigSchema) {
Schema.Type avroType = avroSchema.getType();
byte pigType = pigSchema.getType();
if (avroType.equals(Schema.Type.UNION)) {
return true;
} else if (pigType == DataType.TUPLE) {
/* Tuple is compatible with any type; for users may want to
get rid of the tuple wrapper */
return true;
}
return (avroType.equals(Schema.Type.ARRAY) && pigType == DataType.BAG)
|| (avroType.equals(Schema.Type.MAP) && pigType == DataType.MAP)
|| (avroType.equals(Schema.Type.STRING)
&& pigType == DataType.CHARARRAY
|| pigType == DataType.BIGCHARARRAY)
|| (avroType.equals(Schema.Type.ENUM)
&& pigType == DataType.CHARARRAY)
|| (avroType.equals(Schema.Type.BOOLEAN)
&& pigType == DataType.BOOLEAN
|| pigType == DataType.INTEGER)
|| (avroType.equals(Schema.Type.BYTES)
&& pigType == DataType.BYTEARRAY)
|| (avroType.equals(Schema.Type.DOUBLE)
&& pigType == DataType.DOUBLE
|| pigType == DataType.FLOAT
|| pigType == DataType.INTEGER
|| pigType == DataType.LONG)
|| (avroType.equals(Schema.Type.FLOAT)
&& pigType == DataType.FLOAT
|| pigType == DataType.INTEGER
|| pigType == DataType.LONG)
|| (avroType.equals(Schema.Type.FIXED)
&& pigType == DataType.BYTEARRAY)
|| (avroType.equals(Schema.Type.INT)
&& pigType == DataType.INTEGER)
|| (avroType.equals(Schema.Type.LONG)
&& pigType == DataType.LONG
|| pigType == DataType.INTEGER);
}
Example 4
| Source File: TestPackage.java From spork with Apache License 2.0 | 4 votes |
|
/**
* To show that it does not have any type specific
* code
*/
private void pickTest(byte t, boolean[] inner) throws ExecException, IOException {
Random r = new Random();
switch (t) {
case DataType.BAG:
runTest(GenRandomData.genRandSmallTupDataBag(r, 10, 100), inner, DataType.BAG);
break;
case DataType.BOOLEAN:
runTest(r.nextBoolean(), inner, DataType.BOOLEAN);
break;
case DataType.BYTEARRAY:
runTest(GenRandomData.genRandDBA(r), inner, DataType.BYTEARRAY);
break;
case DataType.BIGCHARARRAY: {
String s = GenRandomData.genRandString(r);
for (; s.length() < 65535;) {
s += GenRandomData.genRandString(r);
}
runTest(s, inner, DataType.CHARARRAY);
break;
}
case DataType.CHARARRAY:
runTest(GenRandomData.genRandString(r), inner, DataType.CHARARRAY);
break;
case DataType.DOUBLE:
runTest(r.nextDouble(), inner, DataType.DOUBLE);
break;
case DataType.FLOAT:
runTest(r.nextFloat(), inner, DataType.FLOAT);
break;
case DataType.INTEGER:
runTest(r.nextInt(), inner, DataType.INTEGER);
break;
case DataType.LONG:
runTest(r.nextLong(), inner, DataType.LONG);
break;
case DataType.DATETIME:
runTest(new DateTime(r.nextLong()), inner, DataType.DATETIME);
break;
case DataType.MAP:
case DataType.INTERNALMAP:
case DataType.BYTE:
return; // map not key type
case DataType.TUPLE:
runTest(GenRandomData.genRandSmallBagTuple(r, 10, 100), inner, DataType.TUPLE);
break;
case DataType.BIGINTEGER:
runTest(new BigInteger(256, r), inner, DataType.BIGINTEGER);
break;
case DataType.BIGDECIMAL:
runTest(new BigDecimal(r.nextDouble()), inner, DataType.BIGDECIMAL);
break;
default:
fail("No test case for type " + DataType.findTypeName(t));
}
}
Example 5
| Source File: PigSchema2Avro.java From spork with Apache License 2.0 | 4 votes |
|
/**
* Check whether Avro type is compatible with Pig type
*
*/
protected static boolean isCompatible(Schema avroSchema, ResourceFieldSchema pigSchema) {
Schema.Type avroType = avroSchema.getType();
byte pigType = pigSchema.getType();
if (avroType.equals(Schema.Type.UNION)) {
return true;
} else if (pigType == DataType.TUPLE) {
/* Tuple is compatible with any type; for users may want to
get rid of the tuple wrapper */
return true;
}
return (avroType.equals(Schema.Type.ARRAY) && pigType == DataType.BAG)
|| (avroType.equals(Schema.Type.MAP) && pigType == DataType.MAP)
|| (avroType.equals(Schema.Type.STRING)
&& pigType == DataType.CHARARRAY
|| pigType == DataType.BIGCHARARRAY)
|| (avroType.equals(Schema.Type.ENUM)
&& pigType == DataType.CHARARRAY)
|| (avroType.equals(Schema.Type.BOOLEAN)
&& pigType == DataType.BOOLEAN
|| pigType == DataType.INTEGER)
|| (avroType.equals(Schema.Type.BYTES)
&& pigType == DataType.BYTEARRAY)
|| (avroType.equals(Schema.Type.DOUBLE)
&& pigType == DataType.DOUBLE
|| pigType == DataType.FLOAT
|| pigType == DataType.INTEGER
|| pigType == DataType.LONG)
|| (avroType.equals(Schema.Type.FLOAT)
&& pigType == DataType.FLOAT
|| pigType == DataType.INTEGER
|| pigType == DataType.LONG)
|| (avroType.equals(Schema.Type.FIXED)
&& pigType == DataType.BYTEARRAY)
|| (avroType.equals(Schema.Type.INT)
&& pigType == DataType.INTEGER)
|| (avroType.equals(Schema.Type.LONG)
&& pigType == DataType.LONG
|| pigType == DataType.INTEGER);
}
Example 6
| Source File: PigBytesConverter.java From elasticsearch-hadoop with Apache License 2.0 | 4 votes |
|
@Override
public void convert(Object from, BytesArray to) {
// expect PigTuple holding a Tuple with only one field - chararray or bytearray
Assert.isTrue(from instanceof PigTuple,
String.format("Unexpected object type, expecting [%s], given [%s]", PigTuple.class, from.getClass()));
PigTuple pt = (PigTuple) from;
ResourceFieldSchema schema = pt.getSchema();
// unwrap the tuple
ResourceSchema tupleSchema = schema.getSchema();
// empty tuple shortcut
if (tupleSchema == null) {
// write empty doc
to.bytes("{}");
return;
}
ResourceFieldSchema[] fields = tupleSchema.getFields();
Assert.isTrue(fields.length == 1, "When using JSON input, only one field is expected");
Object object;
byte type;
try {
object = pt.getTuple().get(0);
type = pt.getTuple().getType(0);
} catch (Exception ex) {
throw new EsHadoopIllegalStateException("Encountered exception while processing tuple", ex);
}
if (type == DataType.BIGCHARARRAY || type == DataType.CHARARRAY) {
to.bytes(object.toString());
return;
}
if (type == DataType.BYTEARRAY) {
DataByteArray dba = (DataByteArray) object;
to.bytes(dba.get(), dba.size());
return;
}
throw new EsHadoopIllegalArgumentException(String.format("Cannot handle Pig type [%s]; expecting [%s,%s]", object.getClass(), String.class, DataByteArray.class));
}
