Java Code Examples for org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector#getTypeName()

The following examples show how to use org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector#getTypeName() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: XGBoostOnlinePredictUDTF.java From incubator-hivemall with Apache License 2.0

6 votes

@Override
public StructObjectInspector initialize(@Nonnull ObjectInspector[] argOIs)
        throws UDFArgumentException {
    if (argOIs.length != 4 && argOIs.length != 5) {
        showHelp("Invalid argment length=" + argOIs.length);
    }
    processOptions(argOIs);

    this.rowIdOI = HiveUtils.asPrimitiveObjectInspector(argOIs, 0);
    ListObjectInspector listOI = HiveUtils.asListOI(argOIs, 1);
    this.featureListOI = listOI;
    ObjectInspector elemOI = listOI.getListElementObjectInspector();
    if (HiveUtils.isNumberOI(elemOI)) {
        this.featureElemOI = HiveUtils.asDoubleCompatibleOI(elemOI);
        this.denseFeatures = true;
    } else if (HiveUtils.isStringOI(elemOI)) {
        this.denseFeatures = false;
    } else {
        throw new UDFArgumentException(
            "Expected array<string|double> for the 2nd argment but got an unexpected features type: "
                    + listOI.getTypeName());
    }
    this.modelIdOI = HiveUtils.asStringOI(argOIs, 2);
    this.modelOI = HiveUtils.asStringOI(argOIs, 3);
    return getReturnOI(rowIdOI);
}

Example 2

Source File: XGBoostTrainUDTF.java From incubator-hivemall with Apache License 2.0

5 votes

@Override
public StructObjectInspector initialize(@Nonnull ObjectInspector[] argOIs)
        throws UDFArgumentException {
    if (argOIs.length != 2 && argOIs.length != 3) {
        showHelp("Invalid argment length=" + argOIs.length);
    }
    processOptions(argOIs);

    ListObjectInspector listOI = HiveUtils.asListOI(argOIs, 0);
    ObjectInspector elemOI = listOI.getListElementObjectInspector();
    this.featureListOI = listOI;
    if (HiveUtils.isNumberOI(elemOI)) {
        this.featureElemOI = HiveUtils.asDoubleCompatibleOI(elemOI);
        this.denseInput = true;
        this.matrixBuilder = new DenseDMatrixBuilder(8192);
    } else if (HiveUtils.isStringOI(elemOI)) {
        this.featureElemOI = HiveUtils.asStringOI(elemOI);
        this.denseInput = false;
        this.matrixBuilder = new SparseDMatrixBuilder(8192);
    } else {
        throw new UDFArgumentException(
            "train_xgboost takes array<double> or array<string> for the first argument: "
                    + listOI.getTypeName());
    }
    this.targetOI = HiveUtils.asDoubleCompatibleOI(argOIs, 1);
    this.labels = new FloatArrayList(1024);

    final List<String> fieldNames = new ArrayList<>(2);
    final List<ObjectInspector> fieldOIs = new ArrayList<>(2);
    fieldNames.add("model_id");
    fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
    fieldNames.add("model");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableStringObjectInspector);
    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}

Example 3

Source File: ArrayUnionUDF.java From incubator-hivemall with Apache License 2.0

5 votes

@Override
public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (argOIs.length < 2) {
        throw new UDFArgumentException("Expecting at least two arrays as arguments");
    }

    ListObjectInspector[] listOIs = new ListObjectInspector[argOIs.length];
    ListObjectInspector arg0OI = HiveUtils.asListOI(argOIs[0]);
    listOIs[0] = arg0OI;
    ObjectInspector arg0ElemOI = arg0OI.getListElementObjectInspector();

    for (int i = 1; i < argOIs.length; ++i) {
        ListObjectInspector checkOI = HiveUtils.asListOI(argOIs[i]);
        if (!ObjectInspectorUtils.compareTypes(arg0ElemOI,
            checkOI.getListElementObjectInspector())) {
            throw new UDFArgumentException("Array types does not match: " + arg0OI.getTypeName()
                    + " != " + checkOI.getTypeName());
        }
        listOIs[i] = checkOI;
    }

    this._listOIs = listOIs;

    return ObjectInspectorFactory.getStandardListObjectInspector(
        ObjectInspectorUtils.getStandardObjectInspector(arg0ElemOI,
            ObjectInspectorCopyOption.WRITABLE));
}

Example 4

Source File: RandomForestClassifierUDTF.java From incubator-hivemall with Apache License 2.0

4 votes

@Override
public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (argOIs.length < 2 || argOIs.length > 4) {
        throw new UDFArgumentException(
            "_FUNC_ takes 2 ~ 4 arguments: array<double|string> features, int label [, const string options, const array<double> classWeight]: "
                    + argOIs.length);
    }

    ListObjectInspector listOI = HiveUtils.asListOI(argOIs, 0);
    ObjectInspector elemOI = listOI.getListElementObjectInspector();
    this.featureListOI = listOI;
    if (HiveUtils.isNumberOI(elemOI)) {
        this.featureElemOI = HiveUtils.asDoubleCompatibleOI(elemOI);
        this.denseInput = true;
        this.matrixBuilder = new RowMajorDenseMatrixBuilder(8192);
    } else if (HiveUtils.isStringOI(elemOI)) {
        this.featureElemOI = HiveUtils.asStringOI(elemOI);
        this.denseInput = false;
        this.matrixBuilder = new CSRMatrixBuilder(8192);
    } else {
        throw new UDFArgumentException(
            "_FUNC_ takes double[] or string[] for the first argument: "
                    + listOI.getTypeName());
    }
    this.labelOI = HiveUtils.asIntCompatibleOI(argOIs, 1);

    processOptions(argOIs);

    this.labels = new IntArrayList(1024);

    final ArrayList<String> fieldNames = new ArrayList<String>(6);
    final ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(6);

    fieldNames.add("model_id");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableStringObjectInspector);
    fieldNames.add("model_weight");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
    fieldNames.add("model");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableStringObjectInspector);
    fieldNames.add("var_importance");
    if (denseInput) {
        fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector));
    } else {
        fieldOIs.add(ObjectInspectorFactory.getStandardMapObjectInspector(
            PrimitiveObjectInspectorFactory.writableIntObjectInspector,
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector));
    }
    fieldNames.add("oob_errors");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
    fieldNames.add("oob_tests");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}

Example 5

Source File: GradientTreeBoostingClassifierUDTF.java From incubator-hivemall with Apache License 2.0

4 votes

@Override
public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (argOIs.length != 2 && argOIs.length != 3) {
        throw new UDFArgumentException(getClass().getSimpleName()
                + " takes 2 or 3 arguments: array<double|string> features, int label [, const string options]: "
                + argOIs.length);
    }

    ListObjectInspector listOI = HiveUtils.asListOI(argOIs, 0);
    ObjectInspector elemOI = listOI.getListElementObjectInspector();
    this.featureListOI = listOI;
    if (HiveUtils.isNumberOI(elemOI)) {
        this.featureElemOI = HiveUtils.asDoubleCompatibleOI(elemOI);
        this.denseInput = true;
        this.matrixBuilder = new RowMajorDenseMatrixBuilder(8192);
    } else if (HiveUtils.isStringOI(elemOI)) {
        this.featureElemOI = HiveUtils.asStringOI(elemOI);
        this.denseInput = false;
        this.matrixBuilder = new CSRMatrixBuilder(8192);
    } else {
        throw new UDFArgumentException(
            "_FUNC_ takes double[] or string[] for the first argument: "
                    + listOI.getTypeName());
    }
    this.labelOI = HiveUtils.asIntCompatibleOI(argOIs, 1);

    processOptions(argOIs);

    this.labels = new IntArrayList(1024);

    ArrayList<String> fieldNames = new ArrayList<String>(6);
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(6);

    fieldNames.add("iteration");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
    fieldNames.add("pred_models");
    fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.writableStringObjectInspector));
    fieldNames.add("intercept");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
    fieldNames.add("shrinkage");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
    fieldNames.add("var_importance");
    if (denseInput) {
        fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector));
    } else {
        fieldOIs.add(ObjectInspectorFactory.getStandardMapObjectInspector(
            PrimitiveObjectInspectorFactory.writableIntObjectInspector,
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector));
    }
    fieldNames.add("oob_error_rate");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}

Example 6

Source File: TreePredictUDF.java From incubator-hivemall with Apache License 2.0

4 votes

@Override
public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (argOIs.length != 3 && argOIs.length != 4) {
        showHelp("tree_predict takes 3 or 4 arguments");
    }

    this.modelOI = HiveUtils.asStringOI(argOIs, 1);
    ListObjectInspector listOI = HiveUtils.asListOI(argOIs, 2);
    this.featureListOI = listOI;
    ObjectInspector elemOI = listOI.getListElementObjectInspector();
    if (HiveUtils.isNumberOI(elemOI)) {
        this.featureElemOI = HiveUtils.asDoubleCompatibleOI(elemOI);
        this.denseInput = true;
    } else if (HiveUtils.isStringOI(elemOI)) {
        this.featureElemOI = HiveUtils.asStringOI(elemOI);
        this.denseInput = false;
    } else {
        throw new UDFArgumentException(
            "tree_predict takes array<double> or array<string> for the second argument: "
                    + listOI.getTypeName());
    }

    if (argOIs.length == 4) {
        ObjectInspector argOI3 = argOIs[3];
        if (HiveUtils.isConstBoolean(argOI3)) {
            this.classification = HiveUtils.getConstBoolean(argOI3);
        } else if (HiveUtils.isConstString(argOI3)) {
            String opts = HiveUtils.getConstString(argOI3);
            processOptions(opts);
        } else {
            throw new UDFArgumentException(
                "tree_predict expects <const boolean> or <const string> for the fourth argument: "
                        + argOI3.getTypeName());
        }
    } else {
        this.classification = false;
    }

    if (classification) {
        List<String> fieldNames = new ArrayList<String>(2);
        List<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(2);
        fieldNames.add("value");
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
        fieldNames.add("posteriori");
        fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector));
        return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
    } else {
        return PrimitiveObjectInspectorFactory.writableDoubleObjectInspector;
    }
}

Example 7

Source File: RandomForestRegressionUDTF.java From incubator-hivemall with Apache License 2.0

4 votes

@Override
public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (argOIs.length != 2 && argOIs.length != 3) {
        throw new UDFArgumentException(getClass().getSimpleName()
                + " takes 2 or 3 arguments: array<double|string> features, double target [, const string options]: "
                + argOIs.length);
    }

    ListObjectInspector listOI = HiveUtils.asListOI(argOIs, 0);
    ObjectInspector elemOI = listOI.getListElementObjectInspector();
    this.featureListOI = listOI;
    if (HiveUtils.isNumberOI(elemOI)) {
        this.featureElemOI = HiveUtils.asDoubleCompatibleOI(elemOI);
        this.denseInput = true;
        this.matrixBuilder = new RowMajorDenseMatrixBuilder(8192);
    } else if (HiveUtils.isStringOI(elemOI)) {
        this.featureElemOI = HiveUtils.asStringOI(elemOI);
        this.denseInput = false;
        this.matrixBuilder = new CSRMatrixBuilder(8192);
    } else {
        throw new UDFArgumentException(
            "_FUNC_ takes double[] or string[] for the first argument: "
                    + listOI.getTypeName());
    }
    this.targetOI = HiveUtils.asDoubleCompatibleOI(argOIs, 1);

    processOptions(argOIs);

    this.targets = new DoubleArrayList(1024);

    final ArrayList<String> fieldNames = new ArrayList<String>(6);
    final ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(6);

    fieldNames.add("model_id");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableStringObjectInspector);
    fieldNames.add("model_err");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
    fieldNames.add("model");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableStringObjectInspector);
    fieldNames.add("var_importance");
    if (denseInput) {
        fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector));
    } else {
        fieldOIs.add(ObjectInspectorFactory.getStandardMapObjectInspector(
            PrimitiveObjectInspectorFactory.writableIntObjectInspector,
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector));
    }
    fieldNames.add("oob_errors");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
    fieldNames.add("oob_tests");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}