org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector Java Examples

@Test
public void TestPointOnly() throws Exception {
       ArrayList<Object> stuff = new ArrayList<Object>();
	Properties proptab = new Properties();
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "shape");
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "binary");
	AbstractSerDe jserde = mkSerDe(proptab);
       StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();

       //value.set("{\"attributes\":{},\"geometry\":{\"x\":15.0,\"y\":5.0}}");
       addWritable(stuff, new Point(15.0, 5.0));
	Object row = runSerDe(stuff, jserde, rowOI);
	Object fieldData = getField("shape", row, rowOI);
	ckPoint(new Point(15.0, 5.0), (BytesWritable)fieldData);

       //value.set("{\"attributes\":{},\"geometry\":{\"x\":7.0,\"y\":4.0}}");
	stuff.clear();
       addWritable(stuff, new Point(7.0, 4.0));
	row = runSerDe(stuff, jserde, rowOI);
	fieldData = getField("shape", row, rowOI);
	ckPoint(new Point(7.0, 4.0), (BytesWritable)fieldData);
}
 

@Test
public void TestIntWrite() throws Exception {
       ArrayList<Object> stuff = new ArrayList<Object>();
	Properties proptab = new Properties();
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "num");
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "int");
	AbstractSerDe jserde = mkSerDe(proptab);
       StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();

       // {"properties":{"num":7}}
       addWritable(stuff, 7);
	Writable jsw = jserde.serialize(stuff, rowOI);
	JsonNode jn = new ObjectMapper().readTree(((Text)jsw).toString());
	jn = jn.findValue("properties");
	jn = jn.findValue("num");
	Assert.assertEquals(7, jn.getIntValue());
}
 

@Test
public void TestPointWrite() throws Exception {
       ArrayList<Object> stuff = new ArrayList<Object>();
	Properties proptab = new Properties();
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "shape");
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "binary");
	AbstractSerDe jserde = mkSerDe(proptab);
       StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();

       // {"properties":{},"geometry":{"type":"Point","coordinates":[15.0,5.0]}}
       addWritable(stuff, new Point(15.0, 5.0));
	Writable jsw = jserde.serialize(stuff, rowOI);
       String rslt = ((Text)jsw).toString();
	JsonNode jn = new ObjectMapper().readTree(rslt);
	jn = jn.findValue("geometry");
	Assert.assertNotNull(jn.findValue("type"));
	Assert.assertNotNull(jn.findValue("coordinates"));
}
 

@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
	if (arguments==null) {
     		throw new UDFArgumentLengthException("ethereumGetSendAddress only takes two arguments: Struct<EthereumTransction>, INT chainId ");
	}
	if (arguments.length != 2) {
     		throw new UDFArgumentLengthException("ethereumGetSendAddress only takes two arguments: Struct<EthereumTransction>, INT chainId ");
	}
	if (!(arguments[0] instanceof StructObjectInspector)) { 
	throw new UDFArgumentException("first argument must be a Struct containing a EthereumTransction");
	}
	if (!(arguments[1] instanceof IntObjectInspector)) {
		throw new UDFArgumentException("second argument must be an int with the chainId");
	}
	this.ethereumUDFUtil=new EthereumUDFUtil((StructObjectInspector) arguments[0]);
	return PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;
}
 

@Test
public void TestIntPoint() throws Exception {
       ArrayList<Object> stuff = new ArrayList<Object>();
	Properties proptab = new Properties();
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "num,shape");
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "bigint,binary");
	AbstractSerDe jserde = mkSerDe(proptab);
       StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();

       //value.set("{\"attributes\":{\"num\":7},\"geometry\":{\"x\":15.0,\"y\":5.0}}");
       addWritable(stuff, 7L);
       addWritable(stuff, new Point(15.0, 5.0));
	Object row = runSerDe(stuff, jserde, rowOI);
	Object fieldData = getField("num", row, rowOI);
	Assert.assertEquals(7, ((LongWritable)fieldData).get());

       //value.set("{\"attributes\":{\"num\":4},\"geometry\":{\"x\":7.0,\"y\":2.0}}");
	stuff.clear();
       addWritable(stuff, 4L);
       addWritable(stuff, new Point(7.0, 2.0));
	row = runSerDe(stuff, jserde, rowOI);
	fieldData = getField("num", row, rowOI);
	Assert.assertEquals(4, ((LongWritable)fieldData).get());
	fieldData = getField("shape", row, rowOI);
	ckPoint(new Point(7.0, 2.0), (BytesWritable)fieldData);
}
 

@Nonnull
protected StructObjectInspector getReturnOI(@Nonnull ObjectInspector featureOutputOI) {
    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();

    fieldNames.add("feature");
    fieldOIs.add(featureOutputOI);
    fieldNames.add("weight");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);
    if (useCovariance()) {
        fieldNames.add("covar");
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);
    }

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 

/**
* Read list of Bitcoin transaction outputs from a table in Hive in any format (e.g. ORC, Parquet)
*
* @param loi ObjectInspector for processing the Object containing a list
* @param listOfOutputsObject object containing the list of outputs to a Bitcoin Transaction
*
* @return a list of BitcoinTransactionOutputs 
*
*/

private List<BitcoinTransactionOutput> readListOfOutputsFromTable(ListObjectInspector loi, Object listOfOutputsObject) {
int listLength=loi.getListLength(listOfOutputsObject);
List<BitcoinTransactionOutput> result=new ArrayList<>(listLength);
StructObjectInspector listOfOutputsElementObjectInspector = (StructObjectInspector)loi.getListElementObjectInspector();
	for (int i=0;i<listLength;i++) {
		Object currentListOfOutputsObject = loi.getListElement(listOfOutputsObject,i);
		StructField valueSF = listOfOutputsElementObjectInspector.getStructFieldRef("value");
		StructField txoutscriptlengthSF = listOfOutputsElementObjectInspector.getStructFieldRef("txoutscriptlength");
		StructField txoutscriptSF = listOfOutputsElementObjectInspector.getStructFieldRef("txoutscript");
		if ((valueSF==null) || (txoutscriptlengthSF==null) || (txoutscriptSF==null)) {
			LOG.warn("Invalid BitcoinTransactionOutput detected at position "+i);
			return new ArrayList<>();
		}
		HiveDecimal currentValue=hdoi.getPrimitiveJavaObject(listOfOutputsElementObjectInspector.getStructFieldData(currentListOfOutputsObject,valueSF));	
		byte[] currentTxOutScriptLength=wboi.getPrimitiveJavaObject(listOfOutputsElementObjectInspector.getStructFieldData(currentListOfOutputsObject,txoutscriptlengthSF));
		byte[] currentTxOutScript=wboi.getPrimitiveJavaObject(listOfOutputsElementObjectInspector.getStructFieldData(currentListOfOutputsObject,txoutscriptSF));
		BitcoinTransactionOutput currentBitcoinTransactionOutput = new BitcoinTransactionOutput(currentValue.bigDecimalValue().toBigIntegerExact(),currentTxOutScriptLength,currentTxOutScript);
		result.add(currentBitcoinTransactionOutput);
	}
return result;
}
 

/**
*
* Initialize HiveUDF and create object inspectors. It requires that the argument length is = 1 and that the ObjectInspector of arguments[0] is of type StructObjectInspector
*
* @param arguments array of length 1 containing one StructObjectInspector
*
* @return ObjectInspector that is able to parse the result of the evaluate method of the UDF (BinaryWritable)
*
* @throws org.apache.hadoop.hive.ql.exec.UDFArgumentException in case the first argument is not of StructObjectInspector
* @throws org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException in case the number of arguments is != 1
*
*/

@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
	if (arguments==null) {
      		throw new UDFArgumentLengthException("bitcoinTransactionHash only takes one argument: Struct<BitcoinTransaction> ");
	}
  	if (arguments.length != 1) {
      		throw new UDFArgumentLengthException("bitcoinTransactionHash only takes one argument: Struct<BitcoinTransaction> ");
	}
	if (!(arguments[0] instanceof StructObjectInspector)) { 
		throw new UDFArgumentException("first argument must be a Struct containing a BitcoinTransaction");
	}
	this.soi = (StructObjectInspector)arguments[0];
	// these are only used for bitcointransaction structs exported to other formats, such as ORC
	this.wboi = PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;
	this.wioi = PrimitiveObjectInspectorFactory.writableIntObjectInspector;
	this.wloi = PrimitiveObjectInspectorFactory.writableLongObjectInspector;
	this.wbyoi = PrimitiveObjectInspectorFactory.writableByteObjectInspector;
	this.hdoi = PrimitiveObjectInspectorFactory.javaHiveDecimalObjectInspector;
	// the UDF returns the hash value of a BitcoinTransaction as byte array
	return PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;
}
 

static void checkIntermediateResultInspector(ObjectInspector resultInspector) {
    Assert.assertNotNull(resultInspector);
    Assert.assertEquals(resultInspector.getCategory(), ObjectInspector.Category.STRUCT);
    StructObjectInspector structResultInspector = (StructObjectInspector) resultInspector;
    List<?> fields = structResultInspector.getAllStructFieldRefs();
    Assert.assertEquals(fields.size(), 3);

    ObjectInspector inspector1 = ((StructField) fields.get(0)).getFieldObjectInspector();
    Assert.assertEquals(inspector1.getCategory(), ObjectInspector.Category.PRIMITIVE);
    PrimitiveObjectInspector primitiveInspector1 = (PrimitiveObjectInspector) inspector1;
    Assert.assertEquals(primitiveInspector1.getPrimitiveCategory(), PrimitiveCategory.INT);

    ObjectInspector inspector2 = ((StructField) fields.get(1)).getFieldObjectInspector();
    Assert.assertEquals(inspector2.getCategory(), ObjectInspector.Category.PRIMITIVE);
    PrimitiveObjectInspector primitiveInspector2 = (PrimitiveObjectInspector) inspector2;
    Assert.assertEquals(primitiveInspector2.getPrimitiveCategory(), PrimitiveCategory.LONG);

    ObjectInspector inspector3 = ((StructField) fields.get(2)).getFieldObjectInspector();
    Assert.assertEquals(inspector3.getCategory(), ObjectInspector.Category.PRIMITIVE);
    PrimitiveObjectInspector primitiveInspector3 = (PrimitiveObjectInspector) inspector3;
    Assert.assertEquals(primitiveInspector3.getPrimitiveCategory(), PrimitiveCategory.BINARY);
}
 

@Nonnull
protected StructObjectInspector getOutputOI(@Nonnull FMHyperParameters params) {
    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
    fieldNames.add("feature");
    if (params.parseFeatureAsInt) {
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
    } else {
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableStringObjectInspector);
    }
    fieldNames.add("W_i");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);
    fieldNames.add("V_if");
    fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.writableFloatObjectInspector));

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 

public HiveAbstractReader(final HiveTableXattr tableAttr, final SplitAndPartitionInfo split,
                          final List<SchemaPath> projectedColumns, final OperatorContext context, final JobConf jobConf,
                          final AbstractSerDe tableSerDe, final StructObjectInspector tableOI, final AbstractSerDe partitionSerDe,
                          final StructObjectInspector partitionOI, final ScanFilter filter,
                          final Collection<List<String>> referencedTables, final UserGroupInformation readerUgi) {
  super(context, projectedColumns);
  this.tableAttr = tableAttr;
  this.split = split;
  this.jobConf = jobConf;
  this.tableSerDe = tableSerDe;
  this.tableOI = tableOI;
  this.partitionSerDe = partitionSerDe == null ? tableSerDe : partitionSerDe;
  this.partitionOI = partitionOI == null ? tableOI : partitionOI;
  this.filter = filter;
  this.referencedTables = referencedTables;
  this.readerUgi = readerUgi;
}
 

@Override
protected StructObjectInspector getReturnOI(ObjectInspector featureRawOI) {
    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();

    fieldNames.add("h");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
    fieldNames.add("w0");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);
    fieldNames.add("w1");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);
    fieldNames.add("w2");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);
    fieldNames.add("hk");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
    fieldNames.add("w3");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 

@Test
public void TestDateParse() throws Exception {
	Configuration config = new Configuration();
	Text value = new Text();

	AbstractSerDe jserde = new GeoJsonSerDe();
	Properties proptab = new Properties();
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "when");
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "date");
	jserde.initialize(config, proptab);
       StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();

       value.set("{\"properties\":{\"when\":\"2020-02-20\"}}");
	Object row = jserde.deserialize(value);
	StructField f0 = rowOI.getStructFieldRef("when");
	Object fieldData = rowOI.getStructFieldData(row, f0);
	Assert.assertEquals("2020-02-20",
						((DateWritable)fieldData).get().toString());
       value.set("{\"properties\":{\"when\":\"2017-05-05\"}}");
       row = jserde.deserialize(value);
	fieldData = rowOI.getStructFieldData(row, f0);
	Assert.assertEquals("2017-05-05",
						((DateWritable)fieldData).get().toString());
}
 

@Override
public Writable serialize(Object obj, ObjectInspector objectInspector) throws SerDeException {
    if (!objectInspector.getCategory().equals(ObjectInspector.Category.STRUCT)) {
        throw new SerDeException("Cannot serialize " + objectInspector.getCategory() + ". Can only serialize a struct");
    }

    StructObjectInspector inspector = (StructObjectInspector) objectInspector;
    List<? extends StructField> fields = inspector.getAllStructFieldRefs();
    Writable[] arr = new Writable[fields.size()];
    for (int i = 0; i < fields.size(); i++) {
        StructField field = fields.get(i);
        Object subObj = inspector.getStructFieldData(obj, field);
        ObjectInspector subInspector = field.getFieldObjectInspector();
        arr[i] = createPrimitive(subObj, (PrimitiveObjectInspector) subInspector);
    }
    serdeSize = arr.length;
    return new ArrayWritable(Writable.class, arr);
}
 

@Override
public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (!(argOIs.length >= 2)) {
        throw new UDFArgumentException("_FUNC_(int xtimes, *) takes at least two arguments");
    }
    this.xtimes = HiveUtils.getAsConstInt(argOIs[0]);
    if (!(xtimes >= 1)) {
        throw new UDFArgumentException("Illegal xtimes value: " + xtimes);
    }
    this.forwardObjs = new Object[argOIs.length - 1];

    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();

    for (int i = 1; i < argOIs.length; i++) {
        fieldNames.add("c" + i);
        fieldOIs.add(argOIs[i]);
    }

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 

@Nonnull
private static StructObjectInspector internalMergeOutputOI(
        @CheckForNull PrimitiveObjectInspector[] inputOIs) throws UDFArgumentException {
    Preconditions.checkNotNull(inputOIs);

    final int numOIs = inputOIs.length;
    final List<String> fieldNames = new ArrayList<String>(numOIs);
    final List<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(numOIs);
    for (int i = 0; i < numOIs; i++) {
        fieldNames.add("f" + String.valueOf(i));
        ObjectInspector elemOI = ObjectInspectorUtils.getStandardObjectInspector(
            inputOIs[i], ObjectInspectorCopyOption.WRITABLE);
        ListObjectInspector listOI =
                ObjectInspectorFactory.getStandardListObjectInspector(elemOI);
        fieldOIs.add(listOI);
    }
    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 

static void checkIntermediateResultInspector(ObjectInspector resultInspector) {
  Assert.assertNotNull(resultInspector);
  Assert.assertEquals(resultInspector.getCategory(), ObjectInspector.Category.STRUCT);
  StructObjectInspector structResultInspector = (StructObjectInspector) resultInspector;
  List<?> fields = structResultInspector.getAllStructFieldRefs();
  Assert.assertEquals(fields.size(), 3);

  ObjectInspector inspector1 = ((StructField) fields.get(0)).getFieldObjectInspector();
  Assert.assertEquals(inspector1.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector1 = (PrimitiveObjectInspector) inspector1;
  Assert.assertEquals(primitiveInspector1.getPrimitiveCategory(), PrimitiveCategory.INT);

  ObjectInspector inspector2 = ((StructField) fields.get(1)).getFieldObjectInspector();
  Assert.assertEquals(inspector2.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector2 = (PrimitiveObjectInspector) inspector2;
  Assert.assertEquals(primitiveInspector2.getPrimitiveCategory(), PrimitiveCategory.STRING);

  ObjectInspector inspector3 = ((StructField) fields.get(2)).getFieldObjectInspector();
  Assert.assertEquals(inspector3.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector3 = (PrimitiveObjectInspector) inspector3;
  Assert.assertEquals(primitiveInspector3.getPrimitiveCategory(), PrimitiveCategory.BINARY);
}
 

/**
 * Deparse a Hive object into a Jackson-serializable object. This uses
 * the ObjectInspector to extract the column data.
 *
 * @param obj - Hive object to deparse
 * @param oi  - ObjectInspector for the object
 * @return - A deparsed object
 */
private Object deparseObject(Object obj, ObjectInspector oi) {
	switch (oi.getCategory()) {
		case LIST:
			return deparseList(obj, (ListObjectInspector) oi);
		case MAP:
			return deparseMap(obj, (MapObjectInspector) oi);
		case PRIMITIVE:
			return deparsePrimitive(obj, (PrimitiveObjectInspector) oi);
		case STRUCT:
			return deparseStruct(obj, (StructObjectInspector) oi, false);
		case UNION:
			// Unsupported by JSON
		default:
			return null;
	}
}
 

@Override
public void openReader(Readable readable) {
    Path path = new Path(this.source().path());
    try {
        this.reader = OrcFile.createReader(path, OrcFile.readerOptions(
                                                         this.conf));
        this.recordReader = this.reader.rows();
        this.inspector = (StructObjectInspector) this.reader
                                                     .getObjectInspector();
        this.row = null;
    } catch (IOException e) {
        throw new LoadException("Failed to open orc reader for '%s'",
                                e, readable);
    }
    this.resetOffset();
}
 

/**
 * Deparse a Hive object into a Jackson-serializable object. This uses the
 * ObjectInspector to extract the column data.
 *
 * @param obj
 *            - Hive object to deparse
 * @param oi
 *            - ObjectInspector for the object
 * @return - A deparsed object
 */
private Object deparseObject(final Object obj, final ObjectInspector oi) {
	switch (oi.getCategory()) {
	case PRIMITIVE:
		return deparsePrimitive(obj, (PrimitiveObjectInspector) oi);
	case LIST:
		return deparseList(obj, (ListObjectInspector) oi);
	case MAP:
		return deparseMap(obj, (MapObjectInspector) oi);
	case STRUCT:
		return deparseStruct(obj, (StructObjectInspector) oi, false);
	case UNION:
		// Unsupported by JSON
	default:
		return null;
	}
}
 

/**
 * This method takes an object representing a row of data from Hive, and uses
 * the ObjectInspector to get the data for each column and serialize.
 */
@Override
public DbRecordWritable serialize(Object row, ObjectInspector inspector) throws SerDeException {
    final StructObjectInspector structInspector = (StructObjectInspector) inspector;
    final List<? extends StructField> fields = structInspector.getAllStructFieldRefs();
    if(fields.size() != fieldCount) {
        throw new SerDeException(String.format("Required %d columns, received %d.", fieldCount, fields.size()));
    }

    cachedWritable.clear();

    for(int i = 0; i < fieldCount; i++) {
        StructField structField = fields.get(i);
        if(structField != null) {
            Object field = structInspector.getStructFieldData(row, structField);
            ObjectInspector fieldOI = structField.getFieldObjectInspector();
            Object javaObject = HiveJdbcBridgeUtils.deparseObject(field, fieldOI);
            cachedWritable.set(i, javaObject);
        }
    }

    return cachedWritable;
}
 

@Override
public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (argOIs.length != 2 && argOIs.length != 3) {
        showHelp(String.format(
            "%s takes 2 or 3 arguments: array<string> x, double y [, CONSTANT string options]: %s",
            getClass().getSimpleName(), Arrays.toString(argOIs)));
    }
    this._xOI = HiveUtils.asListOI(argOIs, 0);
    HiveUtils.validateFeatureOI(_xOI.getListElementObjectInspector());
    this._yOI = HiveUtils.asDoubleCompatibleOI(argOIs, 1);

    this._params = newHyperParameters();
    processOptions(argOIs);

    this._model = null;
    this._t = 0L;
    this._numValidations = 0L;

    if (LOG.isInfoEnabled()) {
        LOG.info(_params);
    }

    return getOutputOI(_params);
}
 

private static StructObjectInspector internalMergeOI() {
    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();

    fieldNames.add("wcList");
    fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector));

    fieldNames.add("lambdaMap");
    fieldOIs.add(ObjectInspectorFactory.getStandardMapObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
        ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.javaFloatObjectInspector)));

    fieldNames.add("topics");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);

    fieldNames.add("alpha");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);

    fieldNames.add("delta");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 

@Test
public void TestNullGeom() throws Exception {
       ArrayList<Object> stuff = new ArrayList<Object>();
	Properties proptab = new Properties();
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "shape");
	proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "binary");
	AbstractSerDe jserde = mkSerDe(proptab);
       StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();

       //value.set("{\"attributes\":{},\"geometry\":{\"x\":15.0,\"y\":5.0}}");
       addWritable(stuff, new Point(15.0, 5.0));
	Object row = runSerDe(stuff, jserde, rowOI);
	Object fieldData = getField("shape", row, rowOI);
	ckPoint(new Point(15.0, 5.0), (BytesWritable)fieldData);

       //value.set("{\"attributes\":{},\"geometry\":null}");
	stuff.set(0, null);
	row = runSerDe(stuff, jserde, rowOI);
	fieldData = getField("shape", row, rowOI);
	Assert.assertNull(fieldData);
}
 

public static StructObjectInspector getTableObjectInspector(Deserializer deserializer)
{
    try {
        ObjectInspector inspector = deserializer.getObjectInspector();
        checkArgument(inspector.getCategory() == Category.STRUCT, "expected STRUCT: %s", inspector.getCategory());
        return (StructObjectInspector) inspector;
    }
    catch (SerDeException e) {
        throw new RuntimeException(e);
    }
}
 

public HiveTextReader(final HiveTableXattr tableAttr, final SplitAndPartitionInfo split,
    final List<SchemaPath> projectedColumns, final OperatorContext context, final JobConf jobConf,
    final SerDe tableSerDe, final StructObjectInspector tableOI, final SerDe partitionSerDe,
    final StructObjectInspector partitionOI, final ScanFilter filter, final Collection<List<String>> referencedTables,
    final UserGroupInformation readerUgi) {
  super(tableAttr, split, projectedColumns, context, jobConf, tableSerDe, tableOI, partitionSerDe, partitionOI, filter,
    referencedTables, readerUgi);
}
 

@Override
public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
	checkArgument(argOIs.length == 2);

	// TEST for constant arguments
	checkArgument(argOIs[1] instanceof ConstantObjectInspector);
	Object constant = ((ConstantObjectInspector) argOIs[1]).getWritableConstantValue();
	checkArgument(constant instanceof IntWritable);
	checkArgument(((IntWritable) constant).get() == 1);

	return ObjectInspectorFactory.getStandardStructObjectInspector(
		Collections.singletonList("col1"),
		Collections.singletonList(PrimitiveObjectInspectorFactory.javaStringObjectInspector));
}
 

@Override
public StructObjectInspector initialize(@Nonnull ObjectInspector[] argOIs)
        throws UDFArgumentException {
    if (argOIs.length != 2 && argOIs.length != 3) {
        showHelp("Invalid argment length=" + argOIs.length);
    }
    processOptions(argOIs);

    ListObjectInspector listOI = HiveUtils.asListOI(argOIs, 0);
    ObjectInspector elemOI = listOI.getListElementObjectInspector();
    this.featureListOI = listOI;
    if (HiveUtils.isNumberOI(elemOI)) {
        this.featureElemOI = HiveUtils.asDoubleCompatibleOI(elemOI);
        this.denseInput = true;
        this.matrixBuilder = new DenseDMatrixBuilder(8192);
    } else if (HiveUtils.isStringOI(elemOI)) {
        this.featureElemOI = HiveUtils.asStringOI(elemOI);
        this.denseInput = false;
        this.matrixBuilder = new SparseDMatrixBuilder(8192);
    } else {
        throw new UDFArgumentException(
            "train_xgboost takes array<double> or array<string> for the first argument: "
                    + listOI.getTypeName());
    }
    this.targetOI = HiveUtils.asDoubleCompatibleOI(argOIs, 1);
    this.labels = new FloatArrayList(1024);

    final List<String> fieldNames = new ArrayList<>(2);
    final List<ObjectInspector> fieldOIs = new ArrayList<>(2);
    fieldNames.add("model_id");
    fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
    fieldNames.add("model");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableStringObjectInspector);
    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 

@Override
public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (argOIs.length < 1) {
        throw new UDFArgumentException(
            "_FUNC_ takes 1 arguments: array<string> words [, const string options]");
    }

    this.wordCountsOI = HiveUtils.asListOI(argOIs[0]);
    HiveUtils.validateFeatureOI(wordCountsOI.getListElementObjectInspector());

    processOptions(argOIs);

    this.model = null;
    this.miniBatch = new String[miniBatchSize][];
    this.miniBatchCount = 0;
    this.cumPerplexity = 0.f;

    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
    fieldNames.add("topic");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
    fieldNames.add("word");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableStringObjectInspector);
    fieldNames.add("score");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 

@Override
public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    final int numArgs = argOIs.length;
    if (numArgs != 2 && numArgs != 3) {
        throw new UDFArgumentException(
            "_FUNC_ takes arguments: List<Int|BigInt|Text> features, float target [, constant string options]");
    }

    return super.initialize(argOIs);
}