org.apache.hadoop.hive.serde2.AbstractSerDe Java Examples
The following examples show how to use
org.apache.hadoop.hive.serde2.AbstractSerDe.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: TestGeoJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestNullAttr() throws Exception {
ArrayList<Object> stuff = new ArrayList<Object>();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "num");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "int");
AbstractSerDe jserde = mkSerDe(proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
//value.set("{\"properties\":{\"num\":7}}");
addWritable(stuff, 7);
Object row = runSerDe(stuff, jserde, rowOI);
Object fieldData = getField("num", row, rowOI);
Assert.assertEquals(7, ((IntWritable)fieldData).get());
//value.set("{\"properties\":{}}");
stuff.set(0, null);
row = runSerDe(stuff, jserde, rowOI);
fieldData = getField("num", row, rowOI);
Assert.assertNull(fieldData);
}
Example #2
| Source File: TestGeoJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestPointWrite() throws Exception {
ArrayList<Object> stuff = new ArrayList<Object>();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "shape");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "binary");
AbstractSerDe jserde = mkSerDe(proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
// {"properties":{},"geometry":{"type":"Point","coordinates":[15.0,5.0]}}
addWritable(stuff, new Point(15.0, 5.0));
Writable jsw = jserde.serialize(stuff, rowOI);
String rslt = ((Text)jsw).toString();
JsonNode jn = new ObjectMapper().readTree(rslt);
jn = jn.findValue("geometry");
Assert.assertNotNull(jn.findValue("type"));
Assert.assertNotNull(jn.findValue("coordinates"));
}
Example #3
| Source File: TestGeoJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestIntParse() throws Exception {
Configuration config = new Configuration();
Text value = new Text();
AbstractSerDe jserde = new GeoJsonSerDe();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "num");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "int");
jserde.initialize(config, proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
value.set("{\"properties\":{\"num\":7}}");
Object row = jserde.deserialize(value);
StructField f0 = rowOI.getStructFieldRef("num");
Object fieldData = rowOI.getStructFieldData(row, f0);
Assert.assertEquals(7, ((IntWritable)fieldData).get());
value.set("{\"properties\":{\"num\":9}}");
row = jserde.deserialize(value);
f0 = rowOI.getStructFieldRef("num");
fieldData = rowOI.getStructFieldData(row, f0);
Assert.assertEquals(9, ((IntWritable)fieldData).get());
}
Example #4
| Source File: TestGeoJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestDateParse() throws Exception {
Configuration config = new Configuration();
Text value = new Text();
AbstractSerDe jserde = new GeoJsonSerDe();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "when");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "date");
jserde.initialize(config, proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
value.set("{\"properties\":{\"when\":\"2020-02-20\"}}");
Object row = jserde.deserialize(value);
StructField f0 = rowOI.getStructFieldRef("when");
Object fieldData = rowOI.getStructFieldData(row, f0);
Assert.assertEquals("2020-02-20",
((DateWritable)fieldData).get().toString());
value.set("{\"properties\":{\"when\":\"2017-05-05\"}}");
row = jserde.deserialize(value);
fieldData = rowOI.getStructFieldData(row, f0);
Assert.assertEquals("2017-05-05",
((DateWritable)fieldData).get().toString());
}
Example #5
| Source File: TestGeoJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestEpochParse() throws Exception {
Configuration config = new Configuration();
Text value = new Text();
AbstractSerDe jserde = new GeoJsonSerDe();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "when");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "date");
jserde.initialize(config, proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
value.set("{\"properties\":{\"when\":147147147147}}");
Object row = jserde.deserialize(value);
StructField f0 = rowOI.getStructFieldRef("when");
Object fieldData = rowOI.getStructFieldData(row, f0);
//Assert.assertEquals(147147147147L, ((DateWritable)fieldData).get().getTime());
Assert.assertEquals(new java.sql.Date(147147147147L).toString(),
((DateWritable)fieldData).get().toString());
value.set("{\"properties\":{\"when\":142857142857}}");
row = jserde.deserialize(value);
fieldData = rowOI.getStructFieldData(row, f0);
//Assert.assertEquals(142857142857L, ((DateWritable)fieldData).get());
Assert.assertEquals(new java.sql.Date(142857142857L).toString(),
((DateWritable)fieldData).get().toString());
}
Example #6
| Source File: TestGeoJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestPointParse() throws Exception {
Configuration config = new Configuration();
Text value = new Text();
AbstractSerDe jserde = new GeoJsonSerDe();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "shape");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "binary");
jserde.initialize(config, proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
value.set("{\"properties\":{},\"geometry\":{\"type\":\"Point\",\"coordinates\":[15.0,5.0]}}");
Object row = jserde.deserialize(value);
StructField f0 = rowOI.getStructFieldRef("shape");
Object fieldData = rowOI.getStructFieldData(row, f0);
ckPoint(new Point(15.0, 5.0), (BytesWritable)fieldData);
value.set("{\"properties\":{},\"geometry\":{\"type\":\"Point\",\"type\":\"Point\",\"coordinates\":[7.0,4.0]}}");
row = jserde.deserialize(value);
f0 = rowOI.getStructFieldRef("shape");
fieldData = rowOI.getStructFieldData(row, f0);
ckPoint(new Point(7.0, 4.0), (BytesWritable)fieldData);
}
Example #7
| Source File: TestGeoJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestIntOnly() throws Exception { // Is this valid for GeoJSON?
ArrayList<Object> stuff = new ArrayList<Object>();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "num");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "int");
AbstractSerDe jserde = mkSerDe(proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
//value.set("{\"properties\":{\"num\":7}}");
addWritable(stuff, 7);
Object row = runSerDe(stuff, jserde, rowOI);
Object fieldData = getField("num", row, rowOI);
Assert.assertEquals(7, ((IntWritable)fieldData).get());
stuff.clear();
addWritable(stuff, 9);
row = runSerDe(stuff, jserde, rowOI);
fieldData = getField("num", row, rowOI);
Assert.assertEquals(9, ((IntWritable)fieldData).get());
}
Example #8
| Source File: TestGeoJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestPointOnly() throws Exception {
ArrayList<Object> stuff = new ArrayList<Object>();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "shape");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "binary");
AbstractSerDe jserde = mkSerDe(proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
//value.set("{\"properties\":{},\"geometry\":{\"type\":\"Point\",\"coordinates\":[15.0,5.0]}}");
addWritable(stuff, new Point(15.0, 5.0));
Object row = runSerDe(stuff, jserde, rowOI);
Object fieldData = getField("shape", row, rowOI);
ckPoint(new Point(15.0, 5.0), (BytesWritable)fieldData);
//value.set("{\"properties\":{},\"geometry\":{\"type\":\"Point\",\"type\":\"Point\",\"coordinates\":[7.0,4.0]}}");
stuff.clear();
addWritable(stuff, new Point(7.0, 4.0));
row = runSerDe(stuff, jserde, rowOI);
fieldData = getField("shape", row, rowOI);
ckPoint(new Point(7.0, 4.0), (BytesWritable)fieldData);
}
Example #9
| Source File: TestGeoJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestIntPoint() throws Exception {
ArrayList<Object> stuff = new ArrayList<Object>();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "num,shape");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "bigint,binary");
AbstractSerDe jserde = mkSerDe(proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
// value.set("{\"properties\":{\"num\":7},\"geometry\":{\"type\":\"Point\",\"coordinates\":[15.0,5.0]}}");
addWritable(stuff, 7L);
addWritable(stuff, new Point(15.0, 5.0));
Object row = runSerDe(stuff, jserde, rowOI);
Object fieldData = getField("num", row, rowOI);
Assert.assertEquals(7, ((LongWritable)fieldData).get());
//value.set("{\"properties\":{\"num\":4},\"geometry\":{\"type\":\"Point\",\"coordinates\":[7.0,2.0]}}");
stuff.clear();
addWritable(stuff, 4L);
addWritable(stuff, new Point(7.0, 2.0));
row = runSerDe(stuff, jserde, rowOI);
fieldData = getField("num", row, rowOI);
Assert.assertEquals(4, ((LongWritable)fieldData).get());
fieldData = getField("shape", row, rowOI);
ckPoint(new Point(7.0, 2.0), (BytesWritable)fieldData);
}
Example #10
| Source File: TestGeoJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestEpochWrite() throws Exception {
ArrayList<Object> stuff = new ArrayList<Object>();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "when");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "date");
AbstractSerDe jserde = mkSerDe(proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
// {"properties":{"when":147147147147}}
long epoch = 0L; // 147147147147L;
java.sql.Date expected = new java.sql.Date(epoch - TimeZone.getDefault().getOffset(epoch));
addWritable(stuff, expected);
Writable jsw = jserde.serialize(stuff, rowOI);
JsonNode jn = new ObjectMapper().readTree(((Text)jsw).toString());
jn = jn.findValue("properties");
jn = jn.findValue("when");
java.sql.Date actual = new java.sql.Date(jn.getLongValue());
long day = 24*3600*1000; // DateWritable stores days not milliseconds.
Assert.assertEquals(epoch/day, jn.getLongValue()/day);
}
Example #11
| Source File: TestGeoJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestNullGeom() throws Exception {
ArrayList<Object> stuff = new ArrayList<Object>();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "shape");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "binary");
AbstractSerDe jserde = mkSerDe(proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
//value.set("{\"properties\":{},\"geometry\":{\"type\":\"Point\",\"coordinates\":[15.0,5.0]}}");
addWritable(stuff, new Point(15.0, 5.0));
Object row = runSerDe(stuff, jserde, rowOI);
Object fieldData = getField("shape", row, rowOI);
ckPoint(new Point(15.0, 5.0), (BytesWritable)fieldData);
//value.set("{\"properties\":{},\"coordinates\":null}");
stuff.set(0, null);
row = runSerDe(stuff, jserde, rowOI);
fieldData = getField("shape", row, rowOI);
Assert.assertNull(fieldData);
}
Example #12
| Source File: TestEsriJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestEpochParse() throws Exception {
Configuration config = new Configuration();
Text value = new Text();
AbstractSerDe jserde = new EsriJsonSerDe();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "when");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "date");
jserde.initialize(config, proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
value.set("{\"attributes\":{\"when\":147147147147}}");
Object row = jserde.deserialize(value);
StructField f0 = rowOI.getStructFieldRef("when");
Object fieldData = rowOI.getStructFieldData(row, f0);
//Assert.assertEquals(147147147147L, ((DateWritable)fieldData).get().getTime());
Assert.assertEquals(new java.sql.Date(147147147147L).toString(),
((DateWritable)fieldData).get().toString());
value.set("{\"attributes\":{\"when\":142857142857}}");
row = jserde.deserialize(value);
fieldData = rowOI.getStructFieldData(row, f0);
//Assert.assertEquals(142857142857L, ((DateWritable)fieldData).get());
Assert.assertEquals(new java.sql.Date(142857142857L).toString(),
((DateWritable)fieldData).get().toString());
}
Example #13
| Source File: TestEsriJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestIntWrite() throws Exception {
ArrayList<Object> stuff = new ArrayList<Object>();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "num");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "int");
AbstractSerDe jserde = mkSerDe(proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
// {"attributes":{"num":7}}
addWritable(stuff, 7);
Writable jsw = jserde.serialize(stuff, rowOI);
JsonNode jn = new ObjectMapper().readTree(((Text)jsw).toString());
jn = jn.findValue("attributes");
jn = jn.findValue("num");
Assert.assertEquals(7, jn.getIntValue());
}
Example #14
| Source File: TestEsriJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestEpochWrite() throws Exception {
ArrayList<Object> stuff = new ArrayList<Object>();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "when");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "date");
AbstractSerDe jserde = mkSerDe(proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
// {"attributes":{"when":147147147147}}
long epoch = 147147147147L;
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MMM-dd");
sdf.setTimeZone(TimeZone.getTimeZone("America/New_York"));
java.sql.Date expected = new java.sql.Date(epoch);
String expString = sdf.format(expected);
//System.err.println(expected.getTime());
addWritable(stuff, expected);
Writable jsw = jserde.serialize(stuff, rowOI);
JsonNode jn = new ObjectMapper().readTree(((Text)jsw).toString());
jn = jn.findValue("attributes");
jn = jn.findValue("when");
java.sql.Date actual = new java.sql.Date(jn.getLongValue());
String actualDateString = sdf.format(actual);
Assert.assertEquals(expString, actualDateString); // workaround DateWritable,j.s.Date
}
Example #15
| Source File: TestEsriJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestTimeWrite() throws Exception {
ArrayList<Object> stuff = new ArrayList<Object>();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "when");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "timestamp");
AbstractSerDe jserde = mkSerDe(proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
// {"attributes":{"when":147147147147}}
long epoch = 147147147147L;
java.sql.Timestamp expected = new java.sql.Timestamp(epoch);
addWritable(stuff, expected);
Writable jsw = jserde.serialize(stuff, rowOI);
JsonNode jn = new ObjectMapper().readTree(((Text)jsw).toString());
jn = jn.findValue("attributes");
jn = jn.findValue("when");
java.sql.Timestamp actual = new java.sql.Timestamp(jn.getLongValue());
Assert.assertEquals(expected, actual);
}
Example #16
| Source File: TestEsriJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestPointWrite() throws Exception {
ArrayList<Object> stuff = new ArrayList<Object>();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "shape");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "binary");
AbstractSerDe jserde = mkSerDe(proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
// {"attributes":{},"geometry":{"x":15.0,"y":5.0}}
addWritable(stuff, new Point(15.0, 5.0));
Writable jsw = jserde.serialize(stuff, rowOI);
String rslt = ((Text)jsw).toString();
JsonNode jn = new ObjectMapper().readTree(rslt);
jn = jn.findValue("geometry");
Assert.assertNotNull(jn.findValue("x"));
Assert.assertNotNull(jn.findValue("y"));
}
Example #17
| Source File: TestEsriJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestIntParse() throws Exception {
Configuration config = new Configuration();
Text value = new Text();
AbstractSerDe jserde = new EsriJsonSerDe();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "num");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "int");
jserde.initialize(config, proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
//value.set("{\"attributes\":{\"num\":7},\"geometry\":null}");
value.set("{\"attributes\":{\"num\":7}}");
Object row = jserde.deserialize(value);
StructField f0 = rowOI.getStructFieldRef("num");
Object fieldData = rowOI.getStructFieldData(row, f0);
Assert.assertEquals(7, ((IntWritable)fieldData).get());
value.set("{\"attributes\":{\"num\":9}}");
row = jserde.deserialize(value);
f0 = rowOI.getStructFieldRef("num");
fieldData = rowOI.getStructFieldData(row, f0);
Assert.assertEquals(9, ((IntWritable)fieldData).get());
}
Example #18
| Source File: TestEsriJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestDateParse() throws Exception {
Configuration config = new Configuration();
Text value = new Text();
AbstractSerDe jserde = new EsriJsonSerDe();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "when");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "date");
jserde.initialize(config, proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
value.set("{\"attributes\":{\"when\":\"2020-02-20\"}}");
Object row = jserde.deserialize(value);
StructField f0 = rowOI.getStructFieldRef("when");
Object fieldData = rowOI.getStructFieldData(row, f0);
Assert.assertEquals("2020-02-20",
((DateWritable)fieldData).get().toString());
value.set("{\"attributes\":{\"when\":\"2017-05-05\"}}");
row = jserde.deserialize(value);
fieldData = rowOI.getStructFieldData(row, f0);
Assert.assertEquals("2017-05-05",
((DateWritable)fieldData).get().toString());
}
Example #19
| Source File: TestGeoJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestIntWrite() throws Exception {
ArrayList<Object> stuff = new ArrayList<Object>();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "num");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "int");
AbstractSerDe jserde = mkSerDe(proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
// {"properties":{"num":7}}
addWritable(stuff, 7);
Writable jsw = jserde.serialize(stuff, rowOI);
JsonNode jn = new ObjectMapper().readTree(((Text)jsw).toString());
jn = jn.findValue("properties");
jn = jn.findValue("num");
Assert.assertEquals(7, jn.getIntValue());
}
Example #20
| Source File: HiveAbstractReader.java From dremio-oss with Apache License 2.0 | 6 votes |
|
public HiveAbstractReader(final HiveTableXattr tableAttr, final SplitAndPartitionInfo split,
final List<SchemaPath> projectedColumns, final OperatorContext context, final JobConf jobConf,
final AbstractSerDe tableSerDe, final StructObjectInspector tableOI, final AbstractSerDe partitionSerDe,
final StructObjectInspector partitionOI, final ScanFilter filter,
final Collection<List<String>> referencedTables, final UserGroupInformation readerUgi) {
super(context, projectedColumns);
this.tableAttr = tableAttr;
this.split = split;
this.jobConf = jobConf;
this.tableSerDe = tableSerDe;
this.tableOI = tableOI;
this.partitionSerDe = partitionSerDe == null ? tableSerDe : partitionSerDe;
this.partitionOI = partitionOI == null ? tableOI : partitionOI;
this.filter = filter;
this.referencedTables = referencedTables;
this.readerUgi = readerUgi;
}
Example #21
| Source File: TestEsriJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestNullGeom() throws Exception {
ArrayList<Object> stuff = new ArrayList<Object>();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "shape");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "binary");
AbstractSerDe jserde = mkSerDe(proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
//value.set("{\"attributes\":{},\"geometry\":{\"x\":15.0,\"y\":5.0}}");
addWritable(stuff, new Point(15.0, 5.0));
Object row = runSerDe(stuff, jserde, rowOI);
Object fieldData = getField("shape", row, rowOI);
ckPoint(new Point(15.0, 5.0), (BytesWritable)fieldData);
//value.set("{\"attributes\":{},\"geometry\":null}");
stuff.set(0, null);
row = runSerDe(stuff, jserde, rowOI);
fieldData = getField("shape", row, rowOI);
Assert.assertNull(fieldData);
}
Example #22
| Source File: TestEsriJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestNullAttr() throws Exception {
ArrayList<Object> stuff = new ArrayList<Object>();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "num");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "int");
AbstractSerDe jserde = mkSerDe(proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
//value.set("{\"attributes\":{\"num\":7}}");
addWritable(stuff, 7);
Object row = runSerDe(stuff, jserde, rowOI);
Object fieldData = getField("num", row, rowOI);
Assert.assertEquals(7, ((IntWritable)fieldData).get());
//value.set("{\"attributes\":{}}");
stuff.set(0, null);
row = runSerDe(stuff, jserde, rowOI);
fieldData = getField("num", row, rowOI);
Assert.assertNull(fieldData);
}
Example #23
| Source File: TestEsriJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestIntPoint() throws Exception {
ArrayList<Object> stuff = new ArrayList<Object>();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "num,shape");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "bigint,binary");
AbstractSerDe jserde = mkSerDe(proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
//value.set("{\"attributes\":{\"num\":7},\"geometry\":{\"x\":15.0,\"y\":5.0}}");
addWritable(stuff, 7L);
addWritable(stuff, new Point(15.0, 5.0));
Object row = runSerDe(stuff, jserde, rowOI);
Object fieldData = getField("num", row, rowOI);
Assert.assertEquals(7, ((LongWritable)fieldData).get());
//value.set("{\"attributes\":{\"num\":4},\"geometry\":{\"x\":7.0,\"y\":2.0}}");
stuff.clear();
addWritable(stuff, 4L);
addWritable(stuff, new Point(7.0, 2.0));
row = runSerDe(stuff, jserde, rowOI);
fieldData = getField("num", row, rowOI);
Assert.assertEquals(4, ((LongWritable)fieldData).get());
fieldData = getField("shape", row, rowOI);
ckPoint(new Point(7.0, 2.0), (BytesWritable)fieldData);
}
Example #24
| Source File: HdfsSerDeImportServiceTest.java From hadoop-etl-udfs with MIT License | 6 votes |
|
private void runImportRCFile(ExaIterator ctx, List<HCatTableColumn> columns, List<HCatTableColumn> partitionColumns, List<OutputColumnSpec> outputColumns, String file) throws Exception {
List<HCatSerDeParameter> serDeParameters = new ArrayList<>();
serDeParameters.add(new HCatSerDeParameter("serialization.format", "1"));
String inputFormatClassName = "org.apache.hadoop.hive.ql.io.RCFileInputFormat";
String serDeClassName = "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe";
String hdfsUser = "hdfs";
boolean useKerberos = false;
List<String> hdfsServers = new ArrayList<>();
hdfsServers.add("file:///");
final Configuration conf = new Configuration();
FileSystem fs = HdfsService.getFileSystem(hdfsServers,conf);
InputFormat<?, ?> inputFormat = (InputFormat<?, ?>) UdfUtils.getInstanceByName(inputFormatClassName);
AbstractSerDe serDe = (AbstractSerDe) UdfUtils.getInstanceByName(serDeClassName);
HdfsSerDeImportService.importFile(fs, file, partitionColumns, inputFormat, serDe, serDeParameters, hdfsServers, hdfsUser, columns, outputColumns, useKerberos, false, ctx);
}
Example #25
| Source File: TestEsriJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestPointOnly() throws Exception {
ArrayList<Object> stuff = new ArrayList<Object>();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "shape");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "binary");
AbstractSerDe jserde = mkSerDe(proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
//value.set("{\"attributes\":{},\"geometry\":{\"x\":15.0,\"y\":5.0}}");
addWritable(stuff, new Point(15.0, 5.0));
Object row = runSerDe(stuff, jserde, rowOI);
Object fieldData = getField("shape", row, rowOI);
ckPoint(new Point(15.0, 5.0), (BytesWritable)fieldData);
//value.set("{\"attributes\":{},\"geometry\":{\"x\":7.0,\"y\":4.0}}");
stuff.clear();
addWritable(stuff, new Point(7.0, 4.0));
row = runSerDe(stuff, jserde, rowOI);
fieldData = getField("shape", row, rowOI);
ckPoint(new Point(7.0, 4.0), (BytesWritable)fieldData);
}
Example #26
| Source File: TestEsriJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestIntOnly() throws Exception {
ArrayList<Object> stuff = new ArrayList<Object>();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "num");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "int");
AbstractSerDe jserde = mkSerDe(proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
addWritable(stuff, 7);
Object row = runSerDe(stuff, jserde, rowOI);
Object fieldData = getField("num", row, rowOI);
Assert.assertEquals(7, ((IntWritable)fieldData).get());
// value.set("{\"attributes\":{\"num\":9}}");
stuff.clear();
addWritable(stuff, 9);
row = runSerDe(stuff, jserde, rowOI);
fieldData = getField("num", row, rowOI);
Assert.assertEquals(9, ((IntWritable)fieldData).get());
}
Example #27
| Source File: TestEsriJsonSerDe.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Test
public void TestPointParse() throws Exception {
Configuration config = new Configuration();
Text value = new Text();
AbstractSerDe jserde = new EsriJsonSerDe();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "shape");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "binary");
jserde.initialize(config, proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
value.set("{\"attributes\":{},\"geometry\":{\"x\":15.0,\"y\":5.0}}");
Object row = jserde.deserialize(value);
StructField f0 = rowOI.getStructFieldRef("shape");
Object fieldData = rowOI.getStructFieldData(row, f0);
ckPoint(new Point(15.0, 5.0), (BytesWritable)fieldData);
value.set("{\"attributes\":{},\"geometry\":{\"x\":7.0,\"y\":4.0}}");
row = jserde.deserialize(value);
f0 = rowOI.getStructFieldRef("shape");
fieldData = rowOI.getStructFieldData(row, f0);
ckPoint(new Point(7.0, 4.0), (BytesWritable)fieldData);
}
Example #28
| Source File: OrcStorage.java From spork with Apache License 2.0 | 6 votes |
|
@Override
public List<String> getShipFiles() {
List<String> cacheFiles = new ArrayList<String>();
String hadoopVersion = "20S";
if (Utils.isHadoop23() || Utils.isHadoop2()) {
hadoopVersion = "23";
}
Class hadoopVersionShimsClass;
try {
hadoopVersionShimsClass = Class.forName("org.apache.hadoop.hive.shims.Hadoop" +
hadoopVersion + "Shims");
} catch (ClassNotFoundException e) {
throw new RuntimeException("Cannot find Hadoop" + hadoopVersion + "ShimsClass in classpath");
}
Class[] classList = new Class[] {OrcFile.class, HiveConf.class, AbstractSerDe.class,
org.apache.hadoop.hive.shims.HadoopShims.class, HadoopShimsSecure.class, hadoopVersionShimsClass,
Input.class};
return FuncUtils.getShipFiles(classList);
}
Example #29
| Source File: HdfsSerDeExportServiceTest.java From hadoop-etl-udfs with MIT License | 5 votes |
|
private void importFile(ExaIterator ctx, List<HCatTableColumn> columns, List<HCatTableColumn> partitionColumns, String file, String inputFormatName, String serdeName) throws Exception {
List<HCatSerDeParameter> serDeParameters = new ArrayList<>();
serDeParameters.add(new HCatSerDeParameter("serialization.format", "1"));
String hdfsUser = "hdfs";
boolean useKerberos = false;
List<String> hdfsServers = new ArrayList<>();
hdfsServers.add("file:///");
final Configuration conf = new Configuration();
FileSystem fs = HdfsService.getFileSystem(hdfsServers, conf);
InputFormat<?, ?> inputFormat = (InputFormat<?, ?>) UdfUtils.getInstanceByName(inputFormatName);
AbstractSerDe serDe = (AbstractSerDe) UdfUtils.getInstanceByName(serdeName);
List<OutputColumnSpec> outputColumns = OutputColumnSpecUtil.generateDefaultOutputSpecification(columns, new ArrayList<HCatTableColumn>());
HdfsSerDeImportService.importFile(fs, file, partitionColumns, inputFormat, serDe, serDeParameters, hdfsServers, hdfsUser, columns, outputColumns, useKerberos, false, ctx);
}
Example #30
| Source File: HiveRecordReaders.java From dremio-oss with Apache License 2.0 | 5 votes |
|
Reader(final HiveTableXattr tableAttr, final SplitAndPartitionInfo split,
final List<SchemaPath> projectedColumns, final OperatorContext context, final JobConf jobConf,
final AbstractSerDe tableSerDe, final StructObjectInspector tableOI, final AbstractSerDe partitionSerDe,
final StructObjectInspector partitionOI, final ScanFilter filter, final Collection<List<String>> referencedTables,
final UserGroupInformation readerUgi) {
super(tableAttr, split, projectedColumns, context, jobConf, tableSerDe, tableOI, partitionSerDe, partitionOI, filter,
referencedTables, readerUgi);
}
