org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector Java Examples

void merge(int size, @Nonnull Object posterioriObj,
        @Nonnull StandardListObjectInspector posterioriOI) throws HiveException {

    if (size != _k) {
        if (_k == -1) {
            this._k = size;
            this._posteriori = new double[size];
        } else {
            throw new HiveException(
                "Mismatch in the number of elements: _k=" + _k + ", size=" + size);
        }
    }

    final double[] posteriori = _posteriori;
    final DoubleObjectInspector doubleOI =
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector;
    for (int i = 0, len = _k; i < len; i++) {
        Object o2 = posterioriOI.getListElement(posterioriObj, i);
        posteriori[i] += doubleOI.get(o2);
    }
}
 

@Override
public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException {
    super.init(m, parameters);
    if (m == Mode.PARTIAL1) {
        inputOI = parameters[0];
        return ObjectInspectorFactory.getStandardListObjectInspector(
            ObjectInspectorUtils.getStandardObjectInspector(inputOI));
    } else {
        if (!(parameters[0] instanceof StandardListObjectInspector)) {
            inputOI = ObjectInspectorUtils.getStandardObjectInspector(parameters[0]);
            return (StandardListObjectInspector) ObjectInspectorFactory.getStandardListObjectInspector(
                inputOI);
        } else {
            internalMergeOI = (StandardListObjectInspector) parameters[0];
            inputOI = internalMergeOI.getListElementObjectInspector();
            loi = (StandardListObjectInspector) ObjectInspectorUtils.getStandardObjectInspector(
                internalMergeOI);
            return loi;
        }
    }
}
 

void merge(final int o_size, @Nonnull final Object o_sum, @Nonnull final Object o_count,
        @Nonnull final StandardListObjectInspector sumOI,
        @Nonnull final StandardListObjectInspector countOI) throws HiveException {
    final WritableDoubleObjectInspector sumElemOI =
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector;
    final WritableLongObjectInspector countElemOI =
            PrimitiveObjectInspectorFactory.writableLongObjectInspector;

    if (o_size != _size) {
        if (_size == -1) {
            init(o_size);
        } else {
            throw new HiveException("Mismatch in the number of elements");
        }
    }
    final double[] sum = _sum;
    final long[] count = _count;
    for (int i = 0, len = _size; i < len; i++) {
        Object sum_e = sumOI.getListElement(o_sum, i);
        sum[i] += sumElemOI.get(sum_e);
        Object count_e = countOI.getListElement(o_count, i);
        count[i] += countElemOI.get(count_e);
    }
}
 

void merge(final double o_ret, @Nullable final Object o_sumVjXj,
        @Nullable final Object o_sumV2X2,
        @Nonnull final StandardListObjectInspector sumVjXjOI,
        @Nonnull final StandardListObjectInspector sumV2X2OI) throws HiveException {
    this.ret += o_ret;
    if (o_sumVjXj == null) {
        return;
    }

    if (o_sumV2X2 == null) {//sanity check
        throw new HiveException("o_sumV2X2 should not be null");
    }

    final int factors = sumVjXjOI.getListLength(o_sumVjXj);
    if (sumVjXj == null) {
        this.sumVjXj = new double[factors];
        this.sumV2X2 = new double[factors];
    } else if (sumVjXj.length != factors) {//sanity check
        throw new HiveException("Mismatch in the number of factors");
    }

    final WritableDoubleObjectInspector doubleOI =
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector;
    for (int f = 0; f < factors; f++) {
        Object o1 = sumVjXjOI.getListElement(o_sumVjXj, f);
        Object o2 = sumV2X2OI.getListElement(o_sumV2X2, f);
        double d1 = doubleOI.get(o1);
        double d2 = doubleOI.get(o2);
        sumVjXj[f] += d1;
        sumV2X2[f] += d2;
    }
}
 

@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
  if (arguments.length != 1) {
    throw new UDFArgumentException("removeNulls takes an array as argument");
  }
  if (arguments[0].getCategory() != Category.LIST) {
    throw new UDFArgumentException("removeNulls takes an array as argument");
  }
  retValInspector = (StandardListObjectInspector) ObjectInspectorUtils.getStandardObjectInspector(arguments[0]);
  if (retValInspector.getListElementObjectInspector().getCategory() != Category.PRIMITIVE) {
    primitiveObjectInspector = (PrimitiveObjectInspector) retValInspector.getListElementObjectInspector();
  }
  return retValInspector;
}
 

@Override
public ObjectInspector init(Mode mode, ObjectInspector[] OIs) throws HiveException {
    super.init(mode, OIs);

    if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {
        weightOI = HiveUtils.asDoubleCompatibleOI(OIs[0]);

        // set const values
        nBins = HiveUtils.getConstInt(OIs[1]);
        if (OIs.length == 3) {
            autoShrink = HiveUtils.getConstBoolean(OIs[2]);
        }

        // check value of `num_of_bins`
        if (nBins < 2) {
            throw new UDFArgumentException(
                "Only greater than or equal to 2 is accepted but " + nBins
                        + " was passed as `num_of_bins`.");
        }

        quantiles = getQuantiles();
    } else {
        structOI = (StructObjectInspector) OIs[0];
        autoShrinkField = structOI.getStructFieldRef("autoShrink");
        histogramField = structOI.getStructFieldRef("histogram");
        quantilesField = structOI.getStructFieldRef("quantiles");
        autoShrinkOI =
                (WritableBooleanObjectInspector) autoShrinkField.getFieldObjectInspector();
        histogramOI =
                (StandardListObjectInspector) histogramField.getFieldObjectInspector();
        quantilesOI =
                (StandardListObjectInspector) quantilesField.getFieldObjectInspector();
        histogramElOI =
                (WritableDoubleObjectInspector) histogramOI.getListElementObjectInspector();
        quantileOI =
                (WritableDoubleObjectInspector) quantilesOI.getListElementObjectInspector();
    }

    if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {
        final ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableBooleanObjectInspector);
        fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector));
        fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector));

        return ObjectInspectorFactory.getStandardStructObjectInspector(
            Arrays.asList("autoShrink", "histogram", "quantiles"), fieldOIs);
    } else {
        return ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
    }
}
 

@Override
public ObjectInspector init(Mode mode, ObjectInspector[] argOIs) throws HiveException {
    super.init(mode, argOIs);

    // initialize input
    if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data
        // this flag will be used in `processOptions` and `iterate` (= when Mode.PARTIAL1 or Mode.COMPLETE)
        this.sortByKey = (argOIs.length == 2 && !HiveUtils.isConstString(argOIs[1]))
                || (argOIs.length == 3 && HiveUtils.isConstString(argOIs[2]));

        if (sortByKey) {
            this.valueOI = argOIs[0];
            this.keyOI = HiveUtils.asPrimitiveObjectInspector(argOIs[1]);
        } else {
            // sort values by value itself
            this.valueOI = HiveUtils.asPrimitiveObjectInspector(argOIs[0]);
            this.keyOI = HiveUtils.asPrimitiveObjectInspector(argOIs[0]);
        }

        processOptions(argOIs);
    } else {// from partial aggregation
        StructObjectInspector soi = (StructObjectInspector) argOIs[0];
        this.internalMergeOI = soi;

        // re-extract input value OI
        this.valueListField = soi.getStructFieldRef("valueList");
        StandardListObjectInspector valueListOI =
                (StandardListObjectInspector) valueListField.getFieldObjectInspector();
        this.valueOI = valueListOI.getListElementObjectInspector();
        this.valueListOI = ObjectInspectorFactory.getStandardListObjectInspector(valueOI);

        // re-extract input key OI
        this.keyListField = soi.getStructFieldRef("keyList");
        StandardListObjectInspector keyListOI =
                (StandardListObjectInspector) keyListField.getFieldObjectInspector();
        this.keyOI = HiveUtils.asPrimitiveObjectInspector(
            keyListOI.getListElementObjectInspector());
        this.keyListOI = ObjectInspectorFactory.getStandardListObjectInspector(keyOI);

        this.sizeField = soi.getStructFieldRef("size");
        this.reverseOrderField = soi.getStructFieldRef("reverseOrder");

        List<? extends StructField> fieldRefs = soi.getAllStructFieldRefs();


        this.outKVField = HiveUtils.getStructFieldRef("outKV", fieldRefs);
        if (outKVField != null) {
            this.outKV = true;
        }
        this.outVKField = HiveUtils.getStructFieldRef("outVK", fieldRefs);
        if (outVKField != null) {
            this.outVK = true;
        }
    }

    // initialize output
    final ObjectInspector outputOI;
    if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {// terminatePartial
        outputOI = internalMergeOI(valueOI, keyOI, outKV, outVK);
    } else {// terminate
        if (outKV) {
            outputOI = ObjectInspectorFactory.getStandardMapObjectInspector(
                ObjectInspectorUtils.getStandardObjectInspector(keyOI),
                ObjectInspectorUtils.getStandardObjectInspector(valueOI));
        } else if (outVK) {
            outputOI = ObjectInspectorFactory.getStandardMapObjectInspector(
                ObjectInspectorUtils.getStandardObjectInspector(valueOI),
                ObjectInspectorUtils.getStandardObjectInspector(keyOI));
        } else {
            outputOI = ObjectInspectorFactory.getStandardListObjectInspector(
                ObjectInspectorUtils.getStandardObjectInspector(valueOI));
        }
    }

    return outputOI;
}