org.apache.hadoop.hive.ql.metadata.HiveException Java Examples

private void flushBuffered(int position) throws HiveException {
    final Object[] forwardObjs = new Object[2];
    if (dense) {
        for (int i = 0; i < position; i++) {
            forwardObjs[0] = Float.valueOf(labels[i]);
            forwardObjs[1] = Arrays.asList(featuresFloatArray[i]);
            forward(forwardObjs);
        }
    } else {
        for (int i = 0; i < position; i++) {
            forwardObjs[0] = Float.valueOf(labels[i]);
            forwardObjs[1] = Arrays.asList(featuresArray[i]);
            forward(forwardObjs);
        }
    }
}
 

@Test
public void testTruncate() throws HiveException {
    List<Text> args = WritableUtils.val("a:0.5", "b:1.0", "c:0.2");
    PolynomialFeaturesUDF udf = new PolynomialFeaturesUDF();
    List<Text> actuals = udf.evaluate(args, 3);
    List<Text> expected = WritableUtils.val("a:0.5", "a^a:0.25", "a^a^a:0.125", "a^a^c:0.05",
        "a^c:0.1", "a^c^c:0.020000001", "b:1.0", "c:0.2", "c^c:0.040000003", "c^c^c:0.008");
    Assert.assertEquals(expected, actuals);

    actuals = udf.evaluate(args, 3, false, false);
    expected = WritableUtils.val("a:0.5", "a^a:0.25", "a^a^a:0.125", "a^a^b:0.25", "a^a^c:0.05",
        "a^b:0.5", "a^b^b:0.5", "a^b^c:0.1", "a^c:0.1", "a^c^c:0.020000001", "b:1.0", "b^b:1.0",
        "b^b^b:1.0", "b^b^c:0.2", "b^c:0.2", "b^c^c:0.040000003", "c:0.2", "c^c:0.040000003",
        "c^c^c:0.008");
    Assert.assertEquals(expected, actuals);
}
 

@Override
public IntWritable evaluate(DeferredObject[] arguments) throws HiveException {
    Object arg0 = arguments[0].get();
    Object arg1 = arguments[1].get();

    if (arg0 == null) {
        return null;
    }
    if (arg1 == null) {
        throw new UDFArgumentException("zoom level should not be null");
    }

    double lon = PrimitiveObjectInspectorUtils.getDouble(arg0, lonOI);
    int zoom = PrimitiveObjectInspectorUtils.getInt(arg1, zoomOI);
    Preconditions.checkArgument(zoom >= 0, "Invalid zoom level", UDFArgumentException.class);

    final int x;
    try {
        x = GeoSpatialUtils.lon2tilex(lon, zoom);
    } catch (IllegalArgumentException ex) {
        throw new UDFArgumentException(ex);
    }

    result.set(x);
    return result;
}
 

@Override
public List<Object> evaluate(DeferredObject[] arguments) throws HiveException {
    ret.clear();

    for (int i = 0; i < arguments.length; i++) {
        final Object arrayObject = arguments[i].get();
        if (arrayObject == null) {
            continue;
        }

        final ListObjectInspector arrayOI = argumentOIs[i];
        final ObjectInspector elemOI = arrayOI.getListElementObjectInspector();
        final int arraylength = arrayOI.getListLength(arrayObject);
        for (int j = 0; j < arraylength; j++) {
            Object rawObj = arrayOI.getListElement(arrayObject, j);
            Object obj = ObjectInspectorUtils.copyToStandardObject(rawObj, elemOI,
                ObjectInspectorCopyOption.WRITABLE);
            ret.add(obj);
        }
    }

    return ret;
}
 

@Override
public Map<?, ?> evaluate(DeferredObject[] arguments) throws HiveException {
    Object arg0 = arguments[0].get();
    if (arg0 == null) {
        return null;
    }
    final Map<?, ?> map = (Map<?, ?>) ObjectInspectorUtils.copyToStandardObject(arg0, mapOI,
        ObjectInspectorCopyOption.WRITABLE);

    Object arg1 = arguments[1].get();
    if (arg1 == null) {
        return null;
    }
    final List<?> filterKeys = (List<?>) ObjectInspectorUtils.copyToStandardObject(arg1, listOI,
        ObjectInspectorCopyOption.WRITABLE);

    final Map<Object, Object> result = new HashMap<>();
    for (Object k : filterKeys) {
        Object v = map.get(k);
        if (v != null) {
            result.put(k, v);
        }
    }
    return result;
}
 

@Override
public void process(Object[] args) throws HiveException {
    if (mapToModel == null) {
        this.mapToModel = new HashMap<String, Predictor>();
    }
    if (args[1] == null) {// features is null
        return;
    }

    String modelId =
            PrimitiveObjectInspectorUtils.getString(nonNullArgument(args, 2), modelIdOI);
    Predictor model = mapToModel.get(modelId);
    if (model == null) {
        Text arg3 = modelOI.getPrimitiveWritableObject(nonNullArgument(args, 3));
        model = XGBoostUtils.loadPredictor(arg3);
        mapToModel.put(modelId, model);
    }

    Writable rowId = HiveUtils.copyToWritable(nonNullArgument(args, 0), rowIdOI);
    FVec features = denseFeatures ? parseDenseFeatures(args[1])
            : parseSparseFeatures(featureListOI.getList(args[1]));

    predictAndForward(model, rowId, features);
}
 

@Override
public void iterate(AggregationBuffer aggregate, Object[] parameters) throws HiveException {
    FunnelAggregateBuffer funnelAggregate = (FunnelAggregateBuffer) aggregate;

    // Add the funnel steps if not already stored
    if (funnelAggregate.funnelSteps.isEmpty()) {
        // Funnel steps start at index 2
        addFunnelSteps(funnelAggregate, Arrays.copyOfRange(parameters, 2, parameters.length));
    }

    // Get the action_column value and add it (if it matches a funnel)
    Object action = parameters[0];
    Object timestamp = parameters[1];
    if (action != null && timestamp != null) {
        // Get the action value
        Object actionValue = ObjectInspectorUtils.copyToStandardObject(action, actionObjectInspector);
        // Get the timestamp value
        Object timestampValue = ObjectInspectorUtils.copyToStandardObject(timestamp, timestampObjectInspector);

        // If the action is not null and it is one of the funnels we are looking for, keep it
        if (actionValue != null && timestampValue != null && funnelAggregate.funnelSet.contains(actionValue)) {
            funnelAggregate.actions.add(actionValue);
            funnelAggregate.timestamps.add(timestampValue);
        }
    }
}
 

@Override
public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException {
    super.init(m, parameters);

    // initialize input
    if (m == Mode.PARTIAL1 || m == Mode.COMPLETE) {// from original data
        this.xhOI = HiveUtils.asNumberOI(parameters[0]);
        this.xkOI = HiveUtils.asNumberOI(parameters[1]);
        this.w0OI = HiveUtils.asNumberOI(parameters[2]);
        this.w1OI = HiveUtils.asNumberOI(parameters[3]);
        this.w2OI = HiveUtils.asNumberOI(parameters[4]);
        this.w3OI = HiveUtils.asNumberOI(parameters[5]);
    }

    return PrimitiveObjectInspectorFactory.writableDoubleObjectInspector;
}
 

@Test
public void testSingleRow() throws HiveException {
    LDAUDTF udtf = new LDAUDTF();
    final int numTopics = 2;
    ObjectInspector[] argOIs = new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector),
            ObjectInspectorUtils.getConstantObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                "-topics " + numTopics)};
    udtf.initialize(argOIs);

    String[] doc1 = new String[] {"1", "2", "3"};
    udtf.process(new Object[] {Arrays.asList(doc1)});

    final MutableInt cnt = new MutableInt(0);
    udtf.setCollector(new Collector() {
        @Override
        public void collect(Object arg0) throws HiveException {
            cnt.addValue(1);
        }
    });
    udtf.close();

    Assert.assertEquals(doc1.length * numTopics, cnt.getValue());
}
 

private static int evalPredict(DecisionTree tree, double[] x)
        throws HiveException, IOException {
    String opScript = tree.predictOpCodegen(StackMachine.SEP);
    debugPrint(opScript);

    TreePredictUDFv1 udf = new TreePredictUDFv1();
    udf.initialize(
        new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                ObjectInspectorFactory.getStandardListObjectInspector(
                    PrimitiveObjectInspectorFactory.javaDoubleObjectInspector),
                ObjectInspectorUtils.getConstantObjectInspector(
                    PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, true)});
    DeferredObject[] arguments = new DeferredObject[] {new DeferredJavaObject("model_id#1"),
            new DeferredJavaObject(ModelType.opscode.getId()), new DeferredJavaObject(opScript),
            new DeferredJavaObject(ArrayUtils.toList(x)), new DeferredJavaObject(true)};

    IntWritable result = (IntWritable) udf.evaluate(arguments);
    udf.close();
    return result.get();
}
 

@Nonnull
public DoubleWritable evaluate(@Nonnull final String modelId, @Nonnull final Text script,
        @Nonnull final Vector features) throws HiveException {
    if (!modelId.equals(prevModelId)) {
        this.prevModelId = modelId;
        int length = script.getLength();
        byte[] b = script.getBytes();
        b = Base91.decode(b, 0, length);
        this.rNode = RegressionTree.deserialize(b, b.length, true);
    }
    Preconditions.checkNotNull(rNode);

    double value = rNode.predict(features);
    result.set(value);
    return result;
}
 

@Test
public void testCosineSimilarity() throws HiveException {
    Double result = getResult(ImmutableMap.<String, Double>of("a", 1.0, "b", 2.0), ImmutableMap.<String, Double>of("c", 1.0, "b", 3.0));
    assertEquals(result, 2 * 3 / (Math.sqrt(5) * Math.sqrt(10)), 0.0);
    result = getResult(ImmutableMap.<String, Double>of("a", 1.0, "b", 2.0, "c", -1.0), ImmutableMap.<String, Double>of("c", 1.0, "b", 3.0));
    assertEquals(result, (2 * 3 + (-1) * 1) / (Math.sqrt(1 + 4 + 1) * Math.sqrt(1 + 9)), 0.0);
    result = getResult(ImmutableMap.<String, Double>of("a", 1.0, "b", 2.0, "c", -1.0), ImmutableMap.<String, Double>of("d", 1.0, "e", 3.0));
    assertEquals(result, 0.0, 0.0);
    result = getResult(null, ImmutableMap.<String, Double>of("c", 1.0, "b", 3.0));
    assertEquals(result, null);
    LinkedHashMap<String, Double> leftMap = Maps.newLinkedHashMap();
    leftMap.put("a", 1.0);
    leftMap.put("b", null);
    result = getResult(leftMap, ImmutableMap.<String, Double>of("c", 1.0, "b", 3.0));
    assertEquals(result, null);
}
 

@Test
public void testSerialization() throws HiveException, IOException {
    Map<String, Double> m = new HashMap<>();
    m.put("Tom", 0.1);
    m.put("Jerry", 0.2);
    m.put("Amy", 0.1);
    m.put("Wong", 0.1);
    m.put("Zhao", null);

    TestUtils.testGenericUDFSerialization(MapRouletteUDF.class,
        new ObjectInspector[] {ObjectInspectorFactory.getStandardMapObjectInspector(
            PrimitiveObjectInspectorFactory.javaStringObjectInspector,
            PrimitiveObjectInspectorFactory.javaDoubleObjectInspector)},
        new Object[] {m});
    byte[] serialized = TestUtils.serializeObjectByKryo(new MapRouletteUDFTest());
    TestUtils.deserializeObjectByKryo(serialized, MapRouletteUDFTest.class);
}
 

private static float min_max_normalization(final float value, final float min, final float max)
        throws HiveException {
    if (min > max) {
        throw new HiveException(
            "min value `" + min + "` SHOULD be less than max value `" + max + '`');
    }
    if (min == max) {
        return 0.5f;
    }
    if (value < min) {
        return 0.f;
    }
    if (value > max) {
        return 1.f;
    }
    return (value - min) / (max - min);
}
 

@Test
public void testConvertToConversionWithZeros() throws HiveException {
    Conversion udf = new Conversion();

    ObjectInspector[] inputOiList = new ObjectInspector[]{
            ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector)
    };

    udf.initialize(inputOiList);

    List<Long> inputList = Arrays.asList(10L, 5L, 0L, 0L, 0L);

    DeferredObject obj1 = mock(DeferredObject.class);
    DeferredObject[] objs = new DeferredObject[] { obj1 };
    when(obj1.get()).thenReturn(inputList);

    Assert.assertEquals(Arrays.asList(1.0, 0.5, 0.0, 0.0, 0.0), udf.evaluate(objs));
}
 

@Test
public void testSeed() throws HiveException, IOException {
    MapRouletteUDF udf = new MapRouletteUDF();
    Map<String, Double> m = new HashMap<>();
    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardMapObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                PrimitiveObjectInspectorFactory.javaDoubleObjectInspector),
            ObjectInspectorUtils.getConstantObjectInspector(
                PrimitiveObjectInspectorFactory.javaLongObjectInspector, 43L)});
    m.put("One", 0.7);
    GenericUDF.DeferredObject[] arguments =
            new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(m)};
    Assert.assertEquals("One", udf.evaluate(arguments));

    udf.close();
}
 

@Override
public Object[] evaluate(@Nonnull DeferredObject[] args) throws HiveException {
    Object x = args[0].get();
    if (x == null) {
        return _result;
    }

    _changeFinder.update(x, _scores);

    double outlierScore = _scores[0];
    double changepointScore = _scores[1];
    _outlierScore.set(outlierScore);
    _changepointScore.set(changepointScore);
    if (_isOutlier != null) {
        _isOutlier.set(outlierScore >= _params.outlierThreshold);
        if (_isChangepoint != null) {
            _isChangepoint.set(changepointScore >= _params.changepointThreshold);
        }
    }

    return _result;
}
 

@Test
public void testBasic() throws HiveException {
    // This is an edge case where the webview fields are calulcated AND wiped again.
    String userAgent = "Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI Build/LMY47O) AppleWebKit/537.36 (KHTML, like Gecko) " +
        "Version/4.0 Chrome/41.51020.2250.0246 Mobile Safari/537.36 cordova-amazon-fireos/3.4.0 AmazonWebAppPlatform/3.4.0;2.0";

    ParseUserAgent parseUserAgent = new ParseUserAgent();

    StandardStructObjectInspector resultInspector = (StandardStructObjectInspector) parseUserAgent
        .initialize(new ObjectInspector[]{
            PrimitiveObjectInspectorFactory.javaStringObjectInspector
        });

    for (int i = 0; i < 100000; i++) {
        Object row = parseUserAgent.evaluate(new DeferredObject[]{new DeferredJavaObject(userAgent)});
        checkField(resultInspector, row, "DeviceClass", "Tablet");
        checkField(resultInspector, row, "OperatingSystemNameVersion", "FireOS 3.4.0");
        checkField(resultInspector, row, "WebviewAppName", "Unknown");
    }
}
 

@Override
public Object terminatePartial(@SuppressWarnings("deprecation") AggregationBuffer agg)
        throws HiveException {
    NDCGAggregationBuffer myAggr = (NDCGAggregationBuffer) agg;

    Object[] partialResult = new Object[2];
    partialResult[0] = new DoubleWritable(myAggr.sum);
    partialResult[1] = new LongWritable(myAggr.count);
    return partialResult;
}
 

@Override
public Object evaluate(DeferredObject[] arguments) throws HiveException {
    if (_analyzer == null) {
        CharArraySet stopWords = stopWords(_stopWordsArray);

        UserDictionary userDict = null;
        if (_userDictObj instanceof String[]) {
            userDict = userDictionary((String[]) _userDictObj);
        } else if (_userDictObj instanceof String) {
            userDict = userDictionary((String) _userDictObj);
        }

        this._analyzer = new JapaneseAnalyzer(userDict, _mode, stopWords, _stopTags);
    }

    Object arg0 = arguments[0].get();
    if (arg0 == null) {
        return null;
    }
    String line = arg0.toString();

    if (_returnPos) {
        return parseLine(_analyzer, line, _result);
    } else {
        return parseLine(_analyzer, line);
    }
}
 

@Override
public void iterate(@SuppressWarnings("deprecation") AggregationBuffer agg,
        Object[] parameters) throws HiveException {
    Preconditions.checkArgument(parameters.length == 6, HiveException.class);

    final AggrBuffer aggr = (AggrBuffer) agg;

    if (parameters[0] /* xh */ != null) {
        double xh = HiveUtils.getDouble(parameters[0], xhOI);
        if (parameters[1] /* xk */ != null) {
            if (parameters[5] /* w3hk */ == null) {
                return;
            }
            // xh, xk, w3hk
            double xk = HiveUtils.getDouble(parameters[1], xkOI);
            double w3hk = HiveUtils.getDouble(parameters[5], w3OI);
            aggr.addW3(xh, xk, w3hk);
        } else {
            if (parameters[3] /* w1h */ == null) {
                return;
            }
            // xh, w1h, w2h
            Preconditions.checkNotNull(parameters[4], HiveException.class);
            double w1h = HiveUtils.getDouble(parameters[3], w1OI);
            double w2h = HiveUtils.getDouble(parameters[4], w2OI);
            aggr.addW1W2(xh, w1h, w2h);
        }
    } else if (parameters[2] /* w0 */ != null) {
        // w0
        double w0 = HiveUtils.getDouble(parameters[2], w0OI);
        aggr.addW0(w0);
    }
}
 

private <T> void testFeature(@Nonnull List<T> x, @Nonnull ObjectInspector featureOI,
        @Nonnull Class<T> featureClass, @Nonnull Class<?> modelFeatureClass) throws Exception {
    float y = 1.f;

    GeneralRegressorUDTF udtf = new GeneralRegressorUDTF();
    ObjectInspector valueOI = PrimitiveObjectInspectorFactory.javaFloatObjectInspector;
    ListObjectInspector featureListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(featureOI);

    udtf.initialize(new ObjectInspector[] {featureListOI, valueOI});

    final List<Object> modelFeatures = new ArrayList<Object>();
    udtf.setCollector(new Collector() {
        @Override
        public void collect(Object input) throws HiveException {
            Object[] forwardMapObj = (Object[]) input;
            modelFeatures.add(forwardMapObj[0]);
        }
    });

    udtf.process(new Object[] {x, y});

    udtf.close();

    Assert.assertFalse(modelFeatures.isEmpty());
    for (Object modelFeature : modelFeatures) {
        Assert.assertEquals("All model features must have same type", modelFeatureClass,
            modelFeature.getClass());
    }
}
 

@SuppressWarnings("deprecation")
@Override
public void iterate(AggregationBuffer aggregationBuffer, Object[] parameters)
        throws HiveException {
    Preconditions.checkNotNull(inputElemOIs);

    EncodingBuffer buf = (EncodingBuffer) aggregationBuffer;
    buf.iterate(parameters, inputElemOIs);
}
 

@Override
public void close() throws HiveException {
    this._progressReporter = getReporter();
    this._treeBuildTaskCounter = (_progressReporter == null) ? null
            : _progressReporter.getCounter("hivemall.smile.RandomForestClassifier$Counter",
                "finishedTreeBuildTasks");
    reportProgress(_progressReporter);

    if (!labels.isEmpty()) {
        Matrix x = matrixBuilder.buildMatrix();
        this.matrixBuilder = null;
        int[] y = labels.toArray();
        this.labels = null;

        // sanity checks
        if (x.numColumns() == 0) {
            throw new HiveException(
                "No non-null features in the training examples. Revise training data");
        }
        if (x.numRows() != y.length) {
            throw new HiveException("Illegal condition was met. y.length=" + y.length
                    + ", X.length=" + x.numRows());
        }

        // run training
        train(x, y);
    }

    // clean up
    this.featureListOI = null;
    this.featureElemOI = null;
    this.labelOI = null;
}
 

@Test
public void testSerialization() throws HiveException {
    TestUtils.testGenericUDTFSerialization(MovingAverageUDTF.class,
        new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaFloatObjectInspector,
                ObjectInspectorUtils.getConstantObjectInspector(
                    PrimitiveObjectInspectorFactory.javaIntObjectInspector, 3)},
        new Object[][] {{1.f}, {2.f}, {3.f}, {4.f}, {5.f}});
}
 

@Test
public void testSerialization() throws HiveException, IOException {
    TestUtils.testGenericUDFSerialization(ToJsonUDF.class,
        new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.javaDoubleObjectInspector)},
        new Object[] {Arrays.asList(0.1d, 1.1d, 2.1d)});
}
 

@Override
protected void forwardPredicted(@Nonnull Writable rowId, @Nonnull double[] predicted)
        throws HiveException {
    final Object[] forwardObj = _forwardObj;
    forwardObj[0] = rowId;
    for (int j = 0, ncols = predicted.length; j < ncols; j++) {
        forwardObj[1] = Integer.valueOf(j);
        forwardObj[2] = Double.valueOf(predicted[j]);
        forward(forwardObj);
    }
}
 

@Override
public IntWritable evaluate(DeferredObject[] arguments) throws HiveException {
    Object arg0 = arguments[0].get();
    if (arg0 == null) {
        return null;
    }

    int index = -1;
    Object minObject = null;
    final int size = listOI.getListLength(arg0);
    for (int i = 0; i < size; i++) {
        Object ai = listOI.getListElement(arg0, i);
        if (ai == null) {
            continue;
        }

        if (minObject == null) {
            minObject = ai;
            index = i;
        } else {
            final int cmp = ObjectInspectorUtils.compare(ai, elemOI, minObject, elemOI);
            if (cmp < 0) {
                minObject = ai;
                index = i;
            }
        }
    }

    result.set(index);
    return result;
}
 

@Override
public DoubleWritable terminate(@SuppressWarnings("deprecation") AggregationBuffer agg)
        throws HiveException {
    RecallAggregationBuffer myAggr = (RecallAggregationBuffer) agg;
    double result = myAggr.get();
    return new DoubleWritable(result);
}
 

@Override
public List<Object> evaluate(DeferredObject[] args) throws HiveException {
    result.clear();

    Object arg0 = args[0].get();
    if (arg0 == null) {
        return null;
    }

    final int listLength = listOI.getListLength(arg0);
    for (int i = 0; i < listLength; i++) {
        final Object subarray = listOI.getListElement(arg0, i);
        if (subarray == null) {
            continue;
        }

        final ListObjectInspector subarrayOI =
                HiveUtils.asListOI(listOI.getListElementObjectInspector());
        final ObjectInspector elemOI = subarrayOI.getListElementObjectInspector();
        final int subarrayLength = subarrayOI.getListLength(subarray);
        for (int j = 0; j < subarrayLength; j++) {
            Object rawElem = subarrayOI.getListElement(subarray, j);
            if (rawElem == null) {
                continue;
            }
            Object elem = ObjectInspectorUtils.copyToStandardObject(rawElem, elemOI,
                ObjectInspectorCopyOption.WRITABLE);
            result.add(elem);
        }
    }

    return result;
}