org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils Java Examples

@Nonnull
protected final List<FeatureValue> parseFeatures(@Nonnull final List<?> features,
        @Nonnull final ObjectInspector featureInspector, final boolean parseFeature) {
    final int numFeatures = features.size();
    if (numFeatures == 0) {
        return Collections.emptyList();
    }
    final List<FeatureValue> list = new ArrayList<FeatureValue>(numFeatures);
    for (Object f : features) {
        if (f == null) {
            continue;
        }
        final FeatureValue fv;
        if (parseFeature) {
            fv = FeatureValue.parse(f);
        } else {
            Object o = ObjectInspectorUtils.copyToStandardObject(f, featureInspector,
                ObjectInspectorCopyOption.WRITABLE);
            Writable k = WritableUtils.toWritable(o);
            fv = new FeatureValue(k, 1.f);
        }
        list.add(fv);
    }
    return list;
}
 

@Test
public void testSingleRow() throws HiveException {
    LDAUDTF udtf = new LDAUDTF();
    final int numTopics = 2;
    ObjectInspector[] argOIs = new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector),
            ObjectInspectorUtils.getConstantObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                "-topics " + numTopics)};
    udtf.initialize(argOIs);

    String[] doc1 = new String[] {"1", "2", "3"};
    udtf.process(new Object[] {Arrays.asList(doc1)});

    final MutableInt cnt = new MutableInt(0);
    udtf.setCollector(new Collector() {
        @Override
        public void collect(Object arg0) throws HiveException {
            cnt.addValue(1);
        }
    });
    udtf.close();

    Assert.assertEquals(doc1.length * numTopics, cnt.getValue());
}
 

@Nullable
protected final FeatureValue[] parseFeatures(@Nonnull final List<?> features) {
    final int size = features.size();
    if (size == 0) {
        return null;
    }

    final ObjectInspector featureInspector = featureListOI.getListElementObjectInspector();
    final FeatureValue[] featureVector = new FeatureValue[size];
    for (int i = 0; i < size; i++) {
        Object f = features.get(i);
        if (f == null) {
            continue;
        }
        final FeatureValue fv;
        if (parseFeature) {
            fv = FeatureValue.parse(f);
        } else {
            Object k = ObjectInspectorUtils.copyToStandardObject(f, featureInspector);
            fv = new FeatureValue(k, 1.f);
        }
        featureVector[i] = fv;
    }
    return featureVector;
}
 

@Nonnull
private static StructObjectInspector internalMergeOI(
        @Nonnull PrimitiveObjectInspector keyOI, @Nonnull ObjectInspector valueOI) {
    List<String> fieldNames = new ArrayList<String>();
    List<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();

    fieldNames.add("partialMap");
    fieldOIs.add(ObjectInspectorFactory.getStandardMapObjectInspector(
        ObjectInspectorUtils.getStandardObjectInspector(keyOI),
        ObjectInspectorUtils.getStandardObjectInspector(valueOI)));

    fieldNames.add("size");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 

@Override
public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (argOIs.length != 1) {
        throw new UDFArgumentException(
            "array_flatten expects exactly one argument: " + argOIs.length);
    }

    this.listOI = HiveUtils.asListOI(argOIs[0]);
    ObjectInspector listElemOI = listOI.getListElementObjectInspector();
    if (listElemOI.getCategory() != Category.LIST) {
        throw new UDFArgumentException(
            "array_flatten takes array of array for the argument: " + listOI.toString());
    }

    ListObjectInspector nestedListOI = HiveUtils.asListOI(listElemOI);
    ObjectInspector elemOI = nestedListOI.getListElementObjectInspector();

    return ObjectInspectorFactory.getStandardListObjectInspector(
        ObjectInspectorUtils.getStandardObjectInspector(elemOI,
            ObjectInspectorCopyOption.WRITABLE));
}
 

@Test
public void testTop2IntNuturalOrder() throws Exception {
    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                    ObjectInspectorUtils.getConstantObjectInspector(
                        PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-k 2")};

    final Integer[] values = new Integer[] {3, -1, 4, 4, 2, 5};

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    for (int i = 0; i < values.length; i++) {
        evaluator.iterate(agg, new Object[] {values[i]});
    }

    @SuppressWarnings("unchecked")
    List<Object> res = (List<Object>) evaluator.terminate(agg);

    Assert.assertEquals(2, res.size());
    Assert.assertEquals(5, res.get(0));
    Assert.assertEquals(4, res.get(1));
}
 

@Override
public Object evaluate(DeferredObject[] arguments) throws HiveException {
    result.set(0L);

    Object array = arguments[ARRAY_IDX].get();
    Object value = arguments[VALUE_IDX].get();

    int arrayLength = arrayOI.getListLength(array);

    // Check if array is null or empty or value is null
    if (array == null || arrayLength <= 0) {
        return result;
    }

    long count = 0L;
    for (int i = 0; i < arrayLength; ++i) {
        Object listElement = arrayOI.getListElement(array, i);
        if (ObjectInspectorUtils.compare(value, valueOI, listElement, arrayElementOI) == 0) {
            count++;
        }
    }
    result.set(count);

    return result;
}
 

@Test
public void testFeatureOnly() throws IOException, HiveException {
    ToLibSVMFormatUDF udf = new ToLibSVMFormatUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector),
            ObjectInspectorUtils.getConstantObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-features 10")});

    Assert.assertEquals("3:2.1 7:3.4", udf.evaluate(new DeferredObject[] {
            new DeferredJavaObject(Arrays.asList("apple:3.4", "orange:2.1"))}));

    Assert.assertEquals("3:2.1 7:3.4", udf.evaluate(
        new DeferredObject[] {new DeferredJavaObject(Arrays.asList("7:3.4", "3:2.1"))}));

    udf.close();
}
 

@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    // Check if two arguments were passed
    if (arguments.length != ARG_COUNT) {
        throw new UDFArgumentLengthException(
                "The function split_to_multimap(string, string, string) takes exactly " + ARG_COUNT + " arguments.");
    }

    // Check if two argument is of string
    for (int i = 0; i < 3; i++) {
        if (!ObjectInspectorUtils.compareTypes(PrimitiveObjectInspectorFactory.javaStringObjectInspector, arguments[i])) {
            throw new UDFArgumentTypeException(i,
                    "\"" + PrimitiveObjectInspectorFactory.javaStringObjectInspector.getTypeName() + "\" "
                            + "expected at function split_to_multimap, but "
                            + "\"" + arguments[i].getTypeName() + "\" "
                            + "is found");
        }
    }

    ObjectInspector mapKeyOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ObjectInspector mapValueOI = ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector);

    return ObjectInspectorFactory.getStandardMapObjectInspector(mapKeyOI, mapValueOI);
}
 

@Nonnull
private static StructObjectInspector internalMergeOutputOI(
        @CheckForNull PrimitiveObjectInspector[] inputOIs) throws UDFArgumentException {
    Preconditions.checkNotNull(inputOIs);

    final int numOIs = inputOIs.length;
    final List<String> fieldNames = new ArrayList<String>(numOIs);
    final List<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(numOIs);
    for (int i = 0; i < numOIs; i++) {
        fieldNames.add("f" + String.valueOf(i));
        ObjectInspector elemOI = ObjectInspectorUtils.getStandardObjectInspector(
            inputOIs[i], ObjectInspectorCopyOption.WRITABLE);
        ListObjectInspector listOI =
                ObjectInspectorFactory.getStandardListObjectInspector(elemOI);
        fieldOIs.add(listOI);
    }
    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 

@Test
public void testTailK() throws Exception {
    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                    ObjectInspectorUtils.getConstantObjectInspector(
                        PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-k -2")};

    final String[] values = new String[] {"banana", "apple", "candy"};

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    for (int i = 0; i < values.length; i++) {
        evaluator.iterate(agg, new Object[] {values[i]});
    }

    @SuppressWarnings("unchecked")
    List<Object> res = (List<Object>) evaluator.terminate(agg);

    Assert.assertEquals(2, res.size());
    Assert.assertEquals("apple", res.get(0));
    Assert.assertEquals("banana", res.get(1));
}
 

@Override
public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (argOIs.length != 2) {
        throw new UDFArgumentLengthException(
            "Expected two arguments for map_filter_keys: " + argOIs.length);
    }

    this.mapOI = HiveUtils.asMapOI(argOIs[0]);
    this.listOI = HiveUtils.asListOI(argOIs[1]);

    ObjectInspector mapKeyOI = mapOI.getMapKeyObjectInspector();
    ObjectInspector filterKeyOI = listOI.getListElementObjectInspector();
    if (!ObjectInspectorUtils.compareTypes(mapKeyOI, filterKeyOI)) {
        throw new UDFArgumentException("Element types does not match: mapKey "
                + mapKeyOI.getTypeName() + ", filterKey" + filterKeyOI.getTypeName());
    }

    return ObjectInspectorUtils.getStandardObjectInspector(mapOI,
        ObjectInspectorCopyOption.WRITABLE);
}
 

@Nullable
protected final FeatureValue[] parseFeatures(@Nonnull final List<?> features) {
    final int size = features.size();
    if (size == 0) {
        return null;
    }

    final ObjectInspector featureInspector = featureListOI.getListElementObjectInspector();
    final FeatureValue[] featureVector = new FeatureValue[size];
    for (int i = 0; i < size; i++) {
        Object f = features.get(i);
        if (f == null) {
            continue;
        }
        final FeatureValue fv;
        if (parseFeature) {
            fv = FeatureValue.parse(f);
        } else {
            Object k = ObjectInspectorUtils.copyToStandardObject(f, featureInspector);
            fv = new FeatureValue(k, 1.f);
        }
        featureVector[i] = fv;
    }
    return featureVector;
}
 

@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    // Check if two arguments were passed
    if (arguments.length != ARG_COUNT) {
        throw new UDFArgumentLengthException(
                "The function regexp_split(string, pattern) takes exactly " + ARG_COUNT + " arguments.");
    }

    for (int i = 0; i < ARG_COUNT; i++) {
        if (!ObjectInspectorUtils.compareTypes(PrimitiveObjectInspectorFactory.javaStringObjectInspector, arguments[i])) {
            throw new UDFArgumentTypeException(i,
                    "\"" + PrimitiveObjectInspectorFactory.javaStringObjectInspector.getTypeName() + "\" "
                            + "expected at function regexp_split, but "
                            + "\"" + arguments[i].getTypeName() + "\" "
                            + "is found");
        }
    }

    ObjectInspector expectOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;

    return ObjectInspectorFactory.getStandardListObjectInspector(expectOI);
}
 

@Override
public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    final StringBuilder buf = new StringBuilder(128);
    final int numArgs = argOIs.length;
    final int last = numArgs - 1;
    for (int i = 0; i < numArgs; i++) {
        if (HiveUtils.isNumberOI(argOIs[i])) {
            buf.append('Q'); // quantitative
        } else {
            buf.append('C'); // categorical            
        }
        if (i != last) {
            buf.append(',');
        }
    }
    String value = buf.toString();
    return ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, value);
}
 

@Test
public void testSeed() throws HiveException, IOException {
    MapRouletteUDF udf = new MapRouletteUDF();
    Map<String, Double> m = new HashMap<>();
    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardMapObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                PrimitiveObjectInspectorFactory.javaDoubleObjectInspector),
            ObjectInspectorUtils.getConstantObjectInspector(
                PrimitiveObjectInspectorFactory.javaLongObjectInspector, 43L)});
    m.put("One", 0.7);
    GenericUDF.DeferredObject[] arguments =
            new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(m)};
    Assert.assertEquals("One", udf.evaluate(arguments));

    udf.close();
}
 

@Test
public void testReverseTopK() throws Exception {
    // = tail-k
    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                    ObjectInspectorUtils.getConstantObjectInspector(
                        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                        "-k 2 -reverse")};

    final String[] values = new String[] {"banana", "apple", "candy"};

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    for (int i = 0; i < values.length; i++) {
        evaluator.iterate(agg, new Object[] {values[i]});
    }

    @SuppressWarnings("unchecked")
    List<Object> res = (List<Object>) evaluator.terminate(agg);

    Assert.assertEquals(2, res.size());
    Assert.assertEquals("apple", res.get(0));
    Assert.assertEquals("banana", res.get(1));
}
 

@Nonnull
private static StructObjectInspector terminalOutputOI(
        @CheckForNull PrimitiveObjectInspector[] inputOIs) {
    Preconditions.checkNotNull(inputOIs);
    Preconditions.checkArgument(inputOIs.length >= 1, inputOIs.length);

    final List<String> fieldNames = new ArrayList<>(inputOIs.length);
    final List<ObjectInspector> fieldOIs = new ArrayList<>(inputOIs.length);
    for (int i = 0; i < inputOIs.length; i++) {
        fieldNames.add("f" + String.valueOf(i + 1));
        ObjectInspector keyOI = ObjectInspectorUtils.getStandardObjectInspector(inputOIs[i],
            ObjectInspectorCopyOption.WRITABLE);
        MapObjectInspector mapOI = ObjectInspectorFactory.getStandardMapObjectInspector(
            keyOI, PrimitiveObjectInspectorFactory.javaIntObjectInspector);
        fieldOIs.add(mapOI);
    }
    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 

@Test
public void testSGD() throws IOException, HiveException {
    String filePath = "adam_test_10000.tsv.gz";
    String options =
            "-loss logloss -opt sgd -reg l1 -lambda 0.0001 -iter 10 -mini_batch 1 -cv_rate 0.00005";

    GeneralClassifierUDTF udtf = new GeneralClassifierUDTF();

    ListObjectInspector stringListOI = ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector);
    ObjectInspector params = ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, options);

    udtf.initialize(new ObjectInspector[] {stringListOI,
            PrimitiveObjectInspectorFactory.javaIntObjectInspector, params});

    BufferedReader reader = readFile(filePath);
    for (String line = reader.readLine(); line != null; line = reader.readLine()) {
        StringTokenizer tokenizer = new StringTokenizer(line, " ");

        String featureLine = tokenizer.nextToken();
        List<String> X = Arrays.asList(featureLine.split(","));

        String labelLine = tokenizer.nextToken();
        Integer y = Integer.valueOf(labelLine);

        udtf.process(new Object[] {X, y});
    }

    udtf.finalizeTraining();

    Assert.assertTrue(
        "CumulativeLoss is expected to be less than 1300: " + udtf.getCumulativeLoss(),
        udtf.getCumulativeLoss() < 1300);
}
 

public static int getHiveBucketHashCode(BucketingVersion bucketingVersion, List<Entry<ObjectInspector, Object>> columnBindings)
{
    ObjectInspector[] objectInspectors = new ObjectInspector[columnBindings.size()];
    Object[] objects = new Object[columnBindings.size()];

    int i = 0;
    for (Entry<ObjectInspector, Object> entry : columnBindings) {
        objectInspectors[i] = entry.getKey();
        if (entry.getValue() != null && entry.getKey() instanceof JavaHiveVarcharObjectInspector) {
            JavaHiveVarcharObjectInspector varcharObjectInspector = (JavaHiveVarcharObjectInspector) entry.getKey();
            objects[i] = new HiveVarchar(((String) entry.getValue()), varcharObjectInspector.getMaxLength());
        }
        else {
            objects[i] = entry.getValue();
        }
        i++;
    }

    switch (bucketingVersion) {
        case BUCKETING_V1:
            @SuppressWarnings("deprecation")
            int hashCodeOld = ObjectInspectorUtils.getBucketHashCodeOld(objects, objectInspectors);
            return hashCodeOld;
        case BUCKETING_V2:
            return ObjectInspectorUtils.getBucketHashCode(objects, objectInspectors);
        default:
            throw new IllegalArgumentException("Unsupported bucketing version: " + bucketingVersion);
    }
}
 

private void setUpWithArguments(double beta, String average) throws Exception {
    fmeasure = new FMeasureUDAF();
    inputOIs = new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableLongObjectInspector),
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableLongObjectInspector),
            ObjectInspectorUtils.getConstantObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                "-beta " + beta + " -average " + average)};

    evaluator =
            fmeasure.getEvaluator(new SimpleGenericUDAFParameterInfo(inputOIs, false, false));
    agg = (FMeasureUDAF.FMeasureAggregationBuffer) evaluator.getNewAggregationBuffer();
}
 

/**
 * Perform argument count check and argument type checking, returns an
 * appropriate evaluator to perform based on input type (which should always
 * be BINARY sketch). Also check lgK and seed parameters if they are passed in.
 *
 * @see org.apache.hadoop.hive.ql.udf.generic.AbstractGenericUDAFResolver
 * #getEvaluator(org.apache.hadoop.hive.ql.udf.generic.GenericUDAFParameterInfo)
 *
 * @param info The parameter info to validate
 * @return The GenericUDAFEvaluator to use to compute the function.
 */
@Override
public GenericUDAFEvaluator getEvaluator(final GenericUDAFParameterInfo info) throws SemanticException {
  final ObjectInspector[] inspectors = info.getParameterObjectInspectors();

  if (inspectors.length < 1) {
    throw new UDFArgumentException("Please specify at least 1 argument");
  }

  if (inspectors.length > 3) {
    throw new UDFArgumentTypeException(inspectors.length - 1, "Please specify no more than 3 arguments");
  }

  ObjectInspectorValidator.validateGivenPrimitiveCategory(inspectors[0], 0, PrimitiveCategory.BINARY);

  // Validate second argument if present
  if (inspectors.length > 1) {
    ObjectInspectorValidator.validateIntegralParameter(inspectors[1], 1);
    if (!ObjectInspectorUtils.isConstantObjectInspector(inspectors[1])) {
      throw new UDFArgumentTypeException(1, "The second argument must be a constant");
    }
  }

  // Validate third argument if present
  if (inspectors.length > 2) {
    ObjectInspectorValidator.validateIntegralParameter(inspectors[2], 2);
    if (!ObjectInspectorUtils.isConstantObjectInspector(inspectors[2])) {
      throw new UDFArgumentTypeException(2, "The third argument must be a constant");
    }
  }

  return new UnionSketchUDAFEvaluator();
}
 

@Override
public void merge(@SuppressWarnings("deprecation") AggregationBuffer agg, Object partial)
        throws HiveException {
    if (partial == null) {
        return;
    }

    final MaxAgg maxagg = (MaxAgg) agg;

    final List<Object> otherObjects;
    if (partial instanceof Object[]) {
        otherObjects = Arrays.asList((Object[]) partial);
    } else if (partial instanceof LazyBinaryStruct) {
        otherObjects = ((LazyBinaryStruct) partial).getFieldsAsList();
    } else if (inputStructOI != null) {
        otherObjects = inputStructOI.getStructFieldsDataAsList(partial);
    } else {
        throw new HiveException("Invalid type: " + partial.getClass().getName());
    }

    boolean isMax = false;
    if (maxagg.objects == null) {
        isMax = true;
    } else {
        int cmp = ObjectInspectorUtils.compare(maxagg.objects[0], outputOIs[0],
            otherObjects.get(0), inputOIs[0]);
        if (cmp < 0) {
            isMax = true;
        }
    }

    if (isMax) {
        int length = otherObjects.size();
        maxagg.objects = new Object[length];
        for (int i = 0; i < length; i++) {
            maxagg.objects[i] = ObjectInspectorUtils.copyToStandardObject(
                otherObjects.get(i), inputOIs[i]);
        }
    }
}
 

@Test
public void testVKMapOptionBananaOverlap2() throws Exception {
    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                    PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                    ObjectInspectorUtils.getConstantObjectInspector(
                        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                        "-k 2 -vk_map")};

    final String[] values = new String[] {"banana", "banana", "candy"};
    final double[] keys = new double[] {0.8, 0.8, 0.7};

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    for (int i = 0; i < values.length; i++) {
        evaluator.iterate(agg, new Object[] {values[i], keys[i]});
    }

    Object result = evaluator.terminate(agg);

    Assert.assertEquals(LinkedHashMap.class, result.getClass());
    Map<?, ?> map = (Map<?, ?>) result;
    Assert.assertEquals(1, map.size());

    Assert.assertEquals(0.8d, map.get("banana"));
}
 

void merge(final Object newX, final Object newY,
        @Nonnull final ObjectInspector xInputOI,
        @Nonnull final ObjectInspector yInputOI,
        @Nonnull final ObjectInspector yOutputOI) {
    final int cmp = ObjectInspectorUtils.compare(y, yOutputOI, newY, yInputOI);
    if (x == null || cmp > 0) {// found smaller y
        this.x = ObjectInspectorUtils.copyToStandardObject(newX, xInputOI,
            ObjectInspectorCopyOption.JAVA);
        this.y = ObjectInspectorUtils.copyToStandardObject(newY, yInputOI,
            ObjectInspectorCopyOption.JAVA);
    }
}
 

@Test
public void testSerialization() throws HiveException {
    TestUtils.testGenericUDTFSerialization(MatrixFactorizationAdaGradUDTF.class,
        new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaFloatObjectInspector,
                ObjectInspectorUtils.getConstantObjectInspector(
                    PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-factor 10")},
        new Object[][] {{0, 0, 5.f}});
}
 

@Test(expected = UDFArgumentException.class)
public void testUnsupportedOptimizer() throws Exception {
    GeneralClassifierUDTF udtf = new GeneralClassifierUDTF();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ObjectInspector stringOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ListObjectInspector stringListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(stringOI);
    ObjectInspector params = ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-opt UnsupportedOpt");

    udtf.initialize(new ObjectInspector[] {stringListOI, intOI, params});
}
 

@Test
public void testVKMapOptionBananaOverlap() throws Exception {
    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                    PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                    ObjectInspectorUtils.getConstantObjectInspector(
                        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                        "-k 2 -vk_map")};

    final String[] values = new String[] {"banana", "banana", "candy"};
    final double[] keys = new double[] {0.7, 0.8, 0.81};

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    for (int i = 0; i < values.length; i++) {
        evaluator.iterate(agg, new Object[] {values[i], keys[i]});
    }

    Object result = evaluator.terminate(agg);

    Assert.assertEquals(LinkedHashMap.class, result.getClass());
    Map<?, ?> map = (Map<?, ?>) result;
    Assert.assertEquals(2, map.size());

    Assert.assertEquals(0.81d, map.get("candy"));
    Assert.assertEquals(0.8d, map.get("banana"));
}
 

@Override
public Object evaluate(DeferredObject[] args) throws HiveException {
    Object arg0 = args[0].get();
    if (arg0 == null) {
        return null;
    }

    Object input = ObjectInspectorUtils.copyToStandardObject(arg0, inputOI);
    try {
        return converter.convert(input);
    } catch (Exception e) {
        return null;
    }
}
 

@Test
public void testDefaultInit() throws HiveException {
    println("--------------------------\n testGaussian()");
    OnlineMatrixFactorizationUDTF mf = new MatrixFactorizationSGDUDTF();

    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ObjectInspector floatOI = PrimitiveObjectInspectorFactory.javaFloatObjectInspector;
    //ObjectInspector param = ObjectInspectorUtils.getConstantObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector, new String("-factor 3 -eta 0.0002"));
    ObjectInspector param = ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, new String("-factor 3"));
    ObjectInspector[] argOIs = new ObjectInspector[] {intOI, intOI, floatOI, param};
    mf.initialize(argOIs);
    Assert.assertTrue(mf.rankInit == RankInitScheme.random);

    float[][] rating = {{5, 3, 0, 1}, {4, 0, 0, 1}, {1, 1, 0, 5}, {1, 0, 0, 4}, {0, 1, 5, 4}};
    Object[] args = new Object[3];
    final int num_iters = 100;
    for (int iter = 0; iter < num_iters; iter++) {
        for (int row = 0; row < rating.length; row++) {
            for (int col = 0, size = rating[row].length; col < size; col++) {
                //print(row + "," + col + ",");
                args[0] = row;
                args[1] = col;
                args[2] = (float) rating[row][col];
                //println((float) rating[row][col]);
                mf.process(args);
            }
        }
    }
    for (int row = 0; row < rating.length; row++) {
        for (int col = 0, size = rating[row].length; col < size; col++) {
            double predicted = mf.predict(row, col);
            print(rating[row][col] + "[" + predicted + "]\t");
            Assert.assertEquals(rating[row][col], predicted, 0.2d);
        }
        println();
    }
}