org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils Java Examples

@Nonnull
private static Int2ObjectMap<Int2FloatMap> kNNentries(@Nonnull final Object kNNiObj,
        @Nonnull final MapObjectInspector knnItemsOI,
        @Nonnull final PrimitiveObjectInspector knnItemsKeyOI,
        @Nonnull final MapObjectInspector knnItemsValueOI,
        @Nonnull final PrimitiveObjectInspector knnItemsValueKeyOI,
        @Nonnull final PrimitiveObjectInspector knnItemsValueValueOI,
        @Nullable Int2ObjectMap<Int2FloatMap> knnItems, @Nonnull final MutableInt nnzKNNi) {
    if (knnItems == null) {
        knnItems = new Int2ObjectOpenHashMap<>(1024);
    } else {
        knnItems.clear();
    }

    int numElementOfKNNItems = 0;
    for (Map.Entry<?, ?> entry : knnItemsOI.getMap(kNNiObj).entrySet()) {
        int user = PrimitiveObjectInspectorUtils.getInt(entry.getKey(), knnItemsKeyOI);
        Int2FloatMap ru = int2floatMap(knnItemsValueOI.getMap(entry.getValue()),
            knnItemsValueKeyOI, knnItemsValueValueOI);
        knnItems.put(user, ru);
        numElementOfKNNItems += ru.size();
    }

    nnzKNNi.setValue(numElementOfKNNItems);
    return knnItems;
}
 

@Override
public DoubleWritable evaluate(DeferredObject[] arguments) throws HiveException {
    Object arg0 = arguments[0].get();
    Object arg1 = arguments[1].get();

    if (arg0 == null) {
        return null;
    }
    if (arg1 == null) {
        throw new UDFArgumentException("zoom level should not be null");
    }

    int x = PrimitiveObjectInspectorUtils.getInt(arg0, xOI);
    int zoom = PrimitiveObjectInspectorUtils.getInt(arg1, zoomOI);
    Preconditions.checkArgument(zoom >= 0, "Invalid zoom level", UDFArgumentException.class);

    final double lon;
    try {
        lon = GeoSpatialUtils.tilex2lon(x, zoom);
    } catch (IllegalArgumentException ex) {
        throw new UDFArgumentException(ex);
    }

    result.set(lon);
    return result;
}
 

@Override
public IntWritable evaluate(DeferredObject[] arguments) throws HiveException {
    Object arg0 = arguments[0].get();
    Object arg1 = arguments[1].get();

    if (arg0 == null) {
        return null;
    }
    if (arg1 == null) {
        throw new UDFArgumentException("zoom level should not be null");
    }

    double lat = PrimitiveObjectInspectorUtils.getDouble(arg0, latOI);
    int zoom = PrimitiveObjectInspectorUtils.getInt(arg1, zoomOI);
    Preconditions.checkArgument(zoom >= 0, "Invalid zoom level", UDFArgumentException.class);

    final int y;
    try {
        y = GeoSpatialUtils.lat2tiley(lat, zoom);
    } catch (IllegalArgumentException ex) {
        throw new UDFArgumentException(ex);
    }

    result.set(y);
    return result;
}
 

/**
 * @return the number of true bits
 */
@Nonnull
public static int setBits(@Nullable final Object argObj,
        @Nonnull final ListObjectInspector listOI,
        @Nonnull final PrimitiveObjectInspector elemOI, @Nonnull final BitSet bitset)
        throws UDFArgumentException {
    if (argObj == null) {
        return 0;
    }
    int count = 0;
    final int length = listOI.getListLength(argObj);
    for (int i = 0; i < length; i++) {
        final Object o = listOI.getListElement(argObj, i);
        if (o == null) {
            continue;
        }
        final int index = PrimitiveObjectInspectorUtils.getInt(o, elemOI);
        if (index < 0) {
            throw new UDFArgumentException("Negative index is not allowed: " + index);
        }
        bitset.set(index);
        count++;
    }
    return count;
}
 

@Nonnull
public static void toDoubleArray(@Nullable final Object argObj,
        @Nonnull final ListObjectInspector listOI,
        @Nonnull final PrimitiveObjectInspector elemOI, @Nonnull final double[] out,
        final double nullValue) throws UDFArgumentException {
    if (argObj == null) {
        return;
    }
    final int length = listOI.getListLength(argObj);
    if (out.length != length) {
        throw new UDFArgumentException(
            "Dimension mismatched. Expected: " + out.length + ", Actual: " + length);
    }
    for (int i = 0; i < length; i++) {
        Object o = listOI.getListElement(argObj, i);
        if (o == null) {
            out[i] = nullValue;
            continue;
        }
        out[i] = PrimitiveObjectInspectorUtils.getDouble(o, elemOI);
    }
    return;
}
 

@Override
public IntWritable evaluate(DeferredObject[] arguments) throws HiveException {
    Object arg0 = arguments[0].get();
    Object arg1 = arguments[1].get();

    if (arg0 == null) {
        return null;
    }
    if (arg1 == null) {
        throw new UDFArgumentException("zoom level should not be null");
    }

    double lon = PrimitiveObjectInspectorUtils.getDouble(arg0, lonOI);
    int zoom = PrimitiveObjectInspectorUtils.getInt(arg1, zoomOI);
    Preconditions.checkArgument(zoom >= 0, "Invalid zoom level", UDFArgumentException.class);

    final int x;
    try {
        x = GeoSpatialUtils.lon2tilex(lon, zoom);
    } catch (IllegalArgumentException ex) {
        throw new UDFArgumentException(ex);
    }

    result.set(x);
    return result;
}
 

@Override
public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveException {
    RfAggregationBufferV2 buf = (RfAggregationBufferV2) agg;

    Preconditions.checkNotNull(parameters[0]);
    int yhat = PrimitiveObjectInspectorUtils.getInt(parameters[0], yhatOI);
    Preconditions.checkNotNull(parameters[1]);
    double[] posteriori =
            HiveUtils.asDoubleArray(parameters[1], posterioriOI, posterioriElemOI);

    double weight = 1.0d;
    if (parameters.length == 3) {
        Preconditions.checkNotNull(parameters[2]);
        weight = PrimitiveObjectInspectorUtils.getDouble(parameters[2], weightOI);
    }

    buf.iterate(yhat, weight, posteriori);
}
 

@Override
public void iterate(@SuppressWarnings("deprecation") AggregationBuffer agg,
        Object[] parameters) throws HiveException {
    if (parameters[0] == null) {
        return;
    }
    FMPredictAggregationBuffer buf = (FMPredictAggregationBuffer) agg;

    double w = PrimitiveObjectInspectorUtils.getDouble(parameters[0], wOI);
    if (parameters[1] == null || /* for TD */vOI.getListLength(parameters[1]) == 0) {// Vif was null
        buf.iterate(w);
    } else {
        if (parameters[2] == null) {
            throw new UDFArgumentException("The third argument Xj must not be null");
        }
        double x = PrimitiveObjectInspectorUtils.getDouble(parameters[2], xOI);
        buf.iterate(w, x, parameters[1], vOI, vElemOI);
    }
}
 

@Nullable
public static double[] asDoubleArray(@Nullable final Object argObj,
        @Nonnull final ListObjectInspector listOI,
        @Nonnull final PrimitiveObjectInspector elemOI, final boolean avoidNull)
        throws UDFArgumentException {
    if (argObj == null) {
        return null;
    }
    final int length = listOI.getListLength(argObj);
    final double[] ary = new double[length];
    for (int i = 0; i < length; i++) {
        Object o = listOI.getListElement(argObj, i);
        if (o == null) {
            if (avoidNull) {
                continue;
            }
            throw new UDFArgumentException("Found null at index " + i);
        }
        ary[i] = PrimitiveObjectInspectorUtils.getDouble(o, elemOI);
    }
    return ary;
}
 

@Nullable
public static float[] asFloatArray(@Nullable final Object argObj,
        @Nonnull final ListObjectInspector listOI,
        @Nonnull final PrimitiveObjectInspector elemOI, final boolean avoidNull)
        throws UDFArgumentException {
    if (argObj == null) {
        return null;
    }
    final int length = listOI.getListLength(argObj);
    final float[] ary = new float[length];
    for (int i = 0; i < length; i++) {
        Object o = listOI.getListElement(argObj, i);
        if (o == null) {
            if (avoidNull) {
                continue;
            }
            throw new UDFArgumentException("Found null at index " + i);
        }
        ary[i] = PrimitiveObjectInspectorUtils.getFloat(o, elemOI);
    }
    return ary;
}
 

@Nonnull
private FVec parseDenseFeatures(@Nonnull Object argObj) throws UDFArgumentException {
    final int length = featureListOI.getListLength(argObj);
    final double[] values = new double[length];
    for (int i = 0; i < length; i++) {
        final Object o = featureListOI.getListElement(argObj, i);
        final double v;
        if (o == null) {
            v = Double.NaN;
        } else {
            v = PrimitiveObjectInspectorUtils.getDouble(o, featureElemOI);
        }
        values[i] = v;

    }
    return FVec.Transformer.fromArray(values, false);
}
 

@Override
public void process(Object[] args) throws HiveException {
    if (mapToModel == null) {
        this.mapToModel = new HashMap<String, Predictor>();
    }
    if (args[1] == null) {// features is null
        return;
    }

    String modelId =
            PrimitiveObjectInspectorUtils.getString(nonNullArgument(args, 2), modelIdOI);
    Predictor model = mapToModel.get(modelId);
    if (model == null) {
        Text arg3 = modelOI.getPrimitiveWritableObject(nonNullArgument(args, 3));
        model = XGBoostUtils.loadPredictor(arg3);
        mapToModel.put(modelId, model);
    }

    Writable rowId = HiveUtils.copyToWritable(nonNullArgument(args, 0), rowIdOI);
    FVec features = denseFeatures ? parseDenseFeatures(args[1])
            : parseSparseFeatures(featureListOI.getList(args[1]));

    predictAndForward(model, rowId, features);
}
 

@Nullable
private void addFeedback(final int userId, @Nonnull final Object arg)
        throws UDFArgumentException {
    final int size = itemListOI.getListLength(arg);
    if (size == 0) {
        return;
    }

    int maxItemId = feedback.getMaxItemId();
    final IntArrayList posItems = new IntArrayList(size);
    for (int i = 0; i < size; i++) {
        Object elem = itemListOI.getListElement(arg, i);
        if (elem == null) {
            continue;
        }
        int index = PrimitiveObjectInspectorUtils.getInt(elem, itemElemOI);
        validateIndex(index);
        maxItemId = Math.max(index, maxItemId);
        posItems.add(index);
    }

    feedback.addFeedback(userId, posItems);
    feedback.setMaxItemId(maxItemId);
}
 

private static int hotIndex(@Nonnull List<?> labels, PrimitiveObjectInspector labelOI)
        throws UDFArgumentException {
    final int nClasses = labels.size();

    int clazz = -1;
    for (int i = 0; i < nClasses; i++) {
        final int label = PrimitiveObjectInspectorUtils.getInt(labels.get(i), labelOI);
        if (label == 1) {// assumes one hot encoding 
            if (clazz != -1) {
                throw new UDFArgumentException(
                    "Specify one-hot vectorized array. Multiple hot elements found.");
            }
            clazz = i;
        } else {
            if (label != 0) {
                throw new UDFArgumentException(
                    "Assumed one-hot encoding (0/1) but found an invalid label: " + label);
            }
        }
    }
    if (clazz == -1) {
        throw new UDFArgumentException(
            "Specify one-hot vectorized array for label. Hot element not found.");
    }
    return clazz;
}
 

@Override
public void iterate(@SuppressWarnings("deprecation") AggregationBuffer agg,
        Object[] parameters) throws HiveException {
    OnlineLDAPredictAggregationBuffer myAggr = (OnlineLDAPredictAggregationBuffer) agg;

    if (parameters[0] == null || parameters[1] == null || parameters[2] == null
            || parameters[3] == null) {
        return;
    }

    String word = PrimitiveObjectInspectorUtils.getString(parameters[0], wordOI);
    float value = HiveUtils.getFloat(parameters[1], valueOI);
    int label = PrimitiveObjectInspectorUtils.getInt(parameters[2], labelOI);
    float lambda = HiveUtils.getFloat(parameters[3], lambdaOI);

    myAggr.iterate(word, value, label, lambda);
}
 

@Override
public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveException {
    ClassificationAUCAggregationBuffer myAggr = (ClassificationAUCAggregationBuffer) agg;

    if (parameters[0] == null) {
        return;
    }
    if (parameters[1] == null) {
        return;
    }

    double score = HiveUtils.getDouble(parameters[0], scoreOI);
    if (score < 0.0d || score > 1.0d) {
        throw new UDFArgumentException("score value MUST be in range [0,1]: " + score);
    }

    int label = PrimitiveObjectInspectorUtils.getInt(parameters[1], labelOI);
    if (label == -1) {
        label = 0;
    } else if (label != 0 && label != 1) {
        throw new UDFArgumentException("label MUST be 0/1 or -1/1: " + label);
    }

    myAggr.iterate(score, label);
}
 

@Override
public Object convert(Object input) {
  if (input == null) {
    return null;
  }
  // unfortunately we seem to get instances of varchar object inspectors without params
  // when an old-style UDF has an evaluate() method with varchar arguments.
  // If we disallow varchar in old-style UDFs and only allow GenericUDFs to be defined
  // with varchar arguments, then we might be able to enforce this properly.
  //if (typeParams == null) {
  //  throw new RuntimeException("varchar type used without type params");
  //}
  HiveVarcharWritable hc = new HiveVarcharWritable();
  switch (inputOI.getPrimitiveCategory()) {
    case BOOLEAN:
      return outputOI.set(hc,
          ((BooleanObjectInspector) inputOI).get(input)
              ? new HiveVarchar("TRUE", -1) : new HiveVarchar("FALSE", -1));
    default:
      return outputOI.set(hc, PrimitiveObjectInspectorUtils.getHiveVarchar(input, inputOI));
  }
}
 

@Override
public void iterate(@SuppressWarnings("deprecation") AggregationBuffer agg,
        Object[] parameters) throws HiveException {
    PLSAPredictAggregationBuffer myAggr = (PLSAPredictAggregationBuffer) agg;

    if (parameters[0] == null || parameters[1] == null || parameters[2] == null
            || parameters[3] == null) {
        return;
    }

    String word = PrimitiveObjectInspectorUtils.getString(parameters[0], wordOI);
    float value = PrimitiveObjectInspectorUtils.getFloat(parameters[1], valueOI);
    int label = PrimitiveObjectInspectorUtils.getInt(parameters[2], labelOI);
    float prob = PrimitiveObjectInspectorUtils.getFloat(parameters[3], probOI);

    myAggr.iterate(word, value, label, prob);
}
 

@Override
public void process(Object[] args) throws HiveException {
    final Object[] positiveObjs = this.positiveObjs;
    for (int i = 0, last = positiveObjs.length - 1; i < last; i++) {
        positiveObjs[i] = args[i + 2];
    }
    // Forward positive label
    final int positive = PrimitiveObjectInspectorUtils.getInt(args[0], positiveOI);
    for (int i = 0; i < positive; i++) {
        forward(positiveObjs);
    }

    final Object[] negativeObjs = this.negativeObjs;
    for (int i = 0, last = negativeObjs.length - 1; i < last; i++) {
        negativeObjs[i] = args[i + 2];
    }
    // Forward negative label
    final int negative = PrimitiveObjectInspectorUtils.getInt(args[1], negativeOI);
    for (int i = 0; i < negative; i++) {
        forward(negativeObjs);
    }
}
 

@Nonnull
public static long[] asLongArray(@Nullable final Object argObj,
        @Nonnull final ListObjectInspector listOI, @Nonnull PrimitiveObjectInspector elemOI) {
    if (argObj == null) {
        return null;
    }
    final int length = listOI.getListLength(argObj);
    final long[] ary = new long[length];
    for (int i = 0; i < length; i++) {
        Object o = listOI.getListElement(argObj, i);
        if (o == null) {
            continue;
        }
        ary[i] = PrimitiveObjectInspectorUtils.getLong(o, elemOI);
    }
    return ary;
}
 

@Override
public void update(@Nonnull final Object arg, @Nonnull final double[] outScores)
        throws HiveException {
    double x = PrimitiveObjectInspectorUtils.getDouble(arg, oi);

    // [Stage#1] Outlier Detection
    xRing.add(x).toArray(xSeries, false /* LIFO */);
    int k1 = xRing.size() - 1;
    double x_hat = sdar1.update(xSeries, k1);

    double scoreX = (k1 == 0.d) ? 0.d : loss(sdar1, x, x_hat, lossFunc1);
    // smoothing
    double y = ChangeFinderUDF.smoothing(outlierScores.add(scoreX));

    // [Stage#2] Change-point Detection
    yRing.add(y).toArray(ySeries, false /* LIFO */);
    int k2 = yRing.size() - 1;
    double y_hat = sdar2.update(ySeries, k2);

    // <LogLoss>
    double lossY = (k2 == 0.d) ? 0.d : loss(sdar2, y, y_hat, lossFunc2);
    double scoreY = ChangeFinderUDF.smoothing(changepointScores.add(lossY));

    outScores[0] = scoreX;
    outScores[1] = scoreY;
}
 

@Nonnull
private static Int2FloatMap int2floatMap(final int item, @Nonnull final Map<?, ?> map,
        @Nonnull final PrimitiveObjectInspector keyOI,
        @Nonnull final PrimitiveObjectInspector valueOI, @Nullable final FloatMatrix dataMatrix,
        @Nullable Int2FloatMap dst) {
    if (dst == null) {
        dst = new Int2FloatOpenHashMap(map.size());
        dst.defaultReturnValue(0.f);
    } else {
        dst.clear();
    }

    for (Map.Entry<?, ?> entry : map.entrySet()) {
        float rating = PrimitiveObjectInspectorUtils.getFloat(entry.getValue(), valueOI);
        if (rating == 0.f) {
            continue;
        }
        int user = PrimitiveObjectInspectorUtils.getInt(entry.getKey(), keyOI);
        dst.put(user, rating);
        if (dataMatrix != null) {
            dataMatrix.set(item, user, rating);
        }
    }

    return dst;
}
 

@Nonnull
private List<Double> evaluateDouble(@Nonnull final Object vecX, @Nonnull final Object vecY,
        @Nonnegative final int size) {
    final Double[] arr = new Double[size];
    for (int i = 0; i < size; i++) {
        Object x = xOI.getListElement(vecX, i);
        Object y = yOI.getListElement(vecY, i);
        if (x == null || y == null) {
            continue;
        }
        double xd = PrimitiveObjectInspectorUtils.getDouble(x, xElemOI);
        double yd = PrimitiveObjectInspectorUtils.getDouble(y, yElemOI);
        double v = xd + yd;
        arr[i] = Double.valueOf(v);
    }
    return Arrays.asList(arr);
}
 

@Nonnull
private List<Long> evaluateLong(@Nonnull final Object vecX, @Nonnull final Object vecY,
        @Nonnegative final int size) {
    final Long[] arr = new Long[size];
    for (int i = 0; i < size; i++) {
        Object x = xOI.getListElement(vecX, i);
        Object y = yOI.getListElement(vecY, i);
        if (x == null || y == null) {
            continue;
        }
        long xd = PrimitiveObjectInspectorUtils.getLong(x, xElemOI);
        long yd = PrimitiveObjectInspectorUtils.getLong(y, yElemOI);
        long v = xd + yd;
        arr[i] = Long.valueOf(v);
    }
    return Arrays.asList(arr);
}
 

@Override
public List<Double> dot(@Nonnull Object x, @Nonnull Object y) throws HiveException {
    final double yd = PrimitiveObjectInspectorUtils.getDouble(y, yOI);

    final int xLen = xListOI.getListLength(x);
    final Double[] arr = new Double[xLen];
    for (int i = 0; i < xLen; i++) {
        Object xi = xListOI.getListElement(x, i);
        if (xi == null) {
            continue;
        }
        double xd = PrimitiveObjectInspectorUtils.getDouble(xi, xElemOI);
        double v = xd * yd;
        arr[i] = Double.valueOf(v);
    }

    return Arrays.asList(arr);
}
 

@Override
public Object evaluate(DeferredObject[] args) throws HiveException {
	double binSize = PrimitiveObjectInspectorUtils.getDouble(args[0].get(), oiBinSize);

	if (!binSizeIsConstant || bins == null) {
		bins = new BinUtils(binSize);
	} 

	OGCPoint point = geomHelper.getPoint(args);

	if (point == null) {
		return null;
	}

	return bins.getId(point.X(), point.Y());
}
 

@Nullable
public static double[] getConstDoubleArray(@Nonnull final ObjectInspector oi)
        throws UDFArgumentException {
    if (!ObjectInspectorUtils.isConstantObjectInspector(oi)) {
        throw new UDFArgumentException("argument must be a constant value: "
                + TypeInfoUtils.getTypeInfoFromObjectInspector(oi));
    }
    ConstantObjectInspector constOI = (ConstantObjectInspector) oi;
    if (constOI.getCategory() != Category.LIST) {
        throw new UDFArgumentException(
            "argument must be an array: " + TypeInfoUtils.getTypeInfoFromObjectInspector(oi));
    }
    StandardConstantListObjectInspector listOI = (StandardConstantListObjectInspector) constOI;
    PrimitiveObjectInspector elemOI =
            HiveUtils.asDoubleCompatibleOI(listOI.getListElementObjectInspector());

    final List<?> lst = listOI.getWritableConstantValue();
    if (lst == null) {
        return null;
    }
    final int size = lst.size();
    final double[] ary = new double[size];
    for (int i = 0; i < size; i++) {
        Object o = lst.get(i);
        if (o == null) {
            ary[i] = Double.NaN;
        } else {
            ary[i] = PrimitiveObjectInspectorUtils.getDouble(o, elemOI);
        }
    }
    return ary;
}
 

@SuppressWarnings("deprecation")
@Override
public void iterate(final AggregationBuffer buf, final Object[] data) throws HiveException {
  if (data[0] == null) { return; }
  @SuppressWarnings("unchecked")
  final ItemsUnionState<T> state = (ItemsUnionState<T>) buf;
  if (!state.isInitialized() && (kObjectInspector != null)) {
    final int k = PrimitiveObjectInspectorUtils.getInt(data[1], kObjectInspector);
    state.init(k);
  }
  state.update(extractValue(data[0], inputObjectInspector));
}
 

private void generateLongSeries(@Nonnull final Object[] args) throws HiveException {
    final long start, end;
    long step = 1L;
    switch (args.length) {
        case 3:
            step = PrimitiveObjectInspectorUtils.getLong(args[2], stepOI);
            if (step == 0) {
                throw new UDFArgumentException("Step MUST NOT be zero");
            }
            // fall through
        case 2:
            start = PrimitiveObjectInspectorUtils.getLong(args[0], startOI);
            end = PrimitiveObjectInspectorUtils.getLong(args[1], endOI);
            break;
        default:
            throw new UDFArgumentException("Expected number of arguments: " + args.length);
    }

    final LongWritable row0 = new LongWritable();
    row[0] = row0;
    if (step > 0) {
        for (long i = start; i <= end; i += step) {
            row0.set(i);
            forward(row);
        }
    } else {
        for (long i = start; i >= end; i += step) {
            row0.set(i);
            forward(row);
        }
    }
}
 

@Override
public void iterate(@SuppressWarnings("deprecation") AggregationBuffer agg,
        Object[] parameters) throws HiveException {
    Preconditions.checkArgument(parameters.length == 2 || parameters.length == 3);

    if (parameters[0] == null || parameters[1] == null) {
        return;
    }
    final BuildBinsAggregationBuffer myAgg = (BuildBinsAggregationBuffer) agg;

    // Get and process the current datum
    myAgg.histogram.add(PrimitiveObjectInspectorUtils.getDouble(parameters[0], weightOI));
}