org.apache.hadoop.hive.ql.udf.generic.GenericUDF#DeferredJavaObject

Source File: ArrayToStrUDFTest.java From incubator-hivemall with Apache License 2.0

6 votes

@Test
public void testSimpleCase() throws HiveException, IOException {
    ArrayToStrUDF udf = new ArrayToStrUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.javaIntObjectInspector),
            PrimitiveObjectInspectorFactory.writableStringObjectInspector});

    Text sep = new Text("#");
    DeferredObject[] args =
            new DeferredObject[] {new GenericUDF.DeferredJavaObject(Arrays.asList(1, 2, 3)),
                    new GenericUDF.DeferredJavaObject(sep)};
    Assert.assertEquals("1#2#3", udf.evaluate(args));

    args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(Arrays.asList(1, 2, 3)),
            new GenericUDF.DeferredJavaObject(null)};
    Assert.assertEquals("1,2,3", udf.evaluate(args));

    udf.close();
}

Source File: MapRouletteUDFTest.java From incubator-hivemall with Apache License 2.0

6 votes

@Test
public void testEmptyMapAndAllNullMap() throws HiveException, IOException {
    MapRouletteUDF udf = new MapRouletteUDF();
    Map<Object, Double> m = new HashMap<>();
    udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardMapObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
        PrimitiveObjectInspectorFactory.javaDoubleObjectInspector)});
    GenericUDF.DeferredObject[] arguments =
            new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(m)};
    Assert.assertNull(udf.evaluate(arguments));
    m.put(null, null);
    arguments = new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(m)};
    Assert.assertNull(udf.evaluate(arguments));

    udf.close();
}

Source File: VectorAddUDFTest.java From incubator-hivemall with Apache License 2.0

6 votes

@Test
public void testAddLong() throws IOException, HiveException {
    VectorAddUDF udf = new VectorAddUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableLongObjectInspector),
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableIntObjectInspector)});

    DeferredObject[] args = new DeferredObject[] {
            new GenericUDF.DeferredJavaObject(
                WritableUtils.toWritableList(new long[] {1, 2, 3})),
            new GenericUDF.DeferredJavaObject(
                WritableUtils.toWritableList(new int[] {2, 3, 4}))};

    List<?> actual = udf.evaluate(args);
    List<Long> expected = Arrays.asList(3L, 5L, 7L);

    Assert.assertEquals(expected, actual);

    udf.close();
}

Source File: UDFStringSplitToMultimapTest.java From hive-third-functions with Apache License 2.0

6 votes

@Test
public void testStringSplitToMultimap() throws Exception {
    UDFStringSplitToMultimap udf = new UDFStringSplitToMultimap();

    GenericUDF.DeferredObject string = new GenericUDF.DeferredJavaObject("a=123,b=0.4,a=124");
    GenericUDF.DeferredObject entryDelimiter = new GenericUDF.DeferredJavaObject(",");
    GenericUDF.DeferredObject keyValueDelimiter = new GenericUDF.DeferredJavaObject("=");
    GenericUDF.DeferredObject[] args = {string, entryDelimiter, keyValueDelimiter};

    HashMap<String, List<String>> output = (HashMap<String, List<String>>) udf.evaluate(args);

    HashMap<String, List<String>> expect = Maps.newHashMap();
    expect.putAll(ImmutableMap.<String, List<String>>of("a", ImmutableList.<String>of("123", "124"), "b", ImmutableList.<String>of("0.4")));

    Assert.assertEquals("split_to_multimap() test", true, MapUtils.mapEquals(output, expect));
}

Source File: ToJsonUDFTest.java From incubator-hivemall with Apache License 2.0

6 votes

@Test
public void testDoubleArray() throws Exception {
    ToJsonUDF udf = new ToJsonUDF();

    ObjectInspector[] argOIs =
            new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)};
    DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(
        WritableUtils.toWritableList(new double[] {0.1, 1.1, 2.1}))};

    udf.initialize(argOIs);
    Text serialized = udf.evaluate(args);

    Assert.assertEquals("[0.1,1.1,2.1]", serialized.toString());

    udf.close();
}

Source File: NamecoinUDFTest.java From hadoopcryptoledger with Apache License 2.0

6 votes

@Test
public void extractNamecoinFieldFirstUpdate() throws HiveException {
	String firstUpdateScript ="520A642F666C6173687570641460C7B068EDEA60281DAF424C38D8DAB87C96CF993D7B226970223A223134352E3234392E3130362E323238222C226D6170223A7B222A223A7B226970223A223134352E3234392E3130362E323238227D7D7D6D6D76A91451B4FC93AAB8CBDBD0AC9BC8EAF824643FC1E29B88AC";
	byte[] firstUpdateScriptBytes = BitcoinUtil.convertHexStringToByteArray(firstUpdateScript);
	NamecoinExtractFieldUDF nefu = new NamecoinExtractFieldUDF();
	ObjectInspector[] arguments = new ObjectInspector[1];
	arguments[0] =  PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;;
	nefu.initialize(arguments);	
	
	GenericUDF.DeferredObject[] doa = new GenericUDF.DeferredObject[1];
	
	doa[0]=new GenericUDF.DeferredJavaObject(new BytesWritable(firstUpdateScriptBytes));
	List<Text> resultList = (List<Text>) nefu.evaluate(doa);
	
	Text[] result=resultList.toArray(new Text[resultList.size()]);
	assertNotNull( result,"Valid result obtained");
	// test for domain name
	assertEquals("d/flashupd",result[0].toString(),"Domain name of first update detected correctly");
	// test for domain value
	assertEquals("{\"ip\":\"145.249.106.228\",\"map\":{\"*\":{\"ip\":\"145.249.106.228\"}}}",result[1].toString(),"Domain value of first update detected correctly");
	
}

Source File: NamecoinUDFTest.java From hadoopcryptoledger with Apache License 2.0

6 votes

@Test
public void extractNamecoinFieldUpdate() throws HiveException {
	String updateScript = "5309642F70616E656C6B612D7B226970223A22382E382E382E38222C226D6170223A7B222A223A7B226970223A22382E382E382E38227D7D7D6D7576A9148D804B079AC79AD0CA108A4E5B679DB591FF069B88AC";
	byte[] updateScriptBytes = BitcoinUtil.convertHexStringToByteArray(updateScript);
	NamecoinExtractFieldUDF nefu = new NamecoinExtractFieldUDF();
	ObjectInspector[] arguments = new ObjectInspector[1];
	arguments[0] =  PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;;
	nefu.initialize(arguments);	
	
	GenericUDF.DeferredObject[] doa = new GenericUDF.DeferredObject[1];
	
	doa[0]=new GenericUDF.DeferredJavaObject(new BytesWritable(updateScriptBytes));
	List<Text> resultList = (List<Text>) nefu.evaluate(doa);
	Text[] result=resultList.toArray(new Text[resultList.size()]);
	assertNotNull( result,"Valid result obtained");
	// test for domain name
	assertEquals("d/panelka",result[0].toString(),"Domain name of first update detected correctly");
	// test for domain value
	assertEquals("{\"ip\":\"8.8.8.8\",\"map\":{\"*\":{\"ip\":\"8.8.8.8\"}}}",result[1].toString(),"Domain value of first update detected correctly");
	
}

Source File: HiveUDFImplementor.java From marble with Apache License 2.0

6 votes

public static Object callGenericUDF(GenericUDF udfInstance, Object[] args,
    RelDataTypeHolder[] argsType, ObjectInspector outputObjectInspector) {
  try {
    GenericUDF.DeferredJavaObject[] deferredJavaObjectArray =
        new GenericUDF.DeferredJavaObject[args.length];
    for (int i = 0; i < args.length; i++) {
      deferredJavaObjectArray[i] = new GenericUDF.DeferredJavaObject(
          TypeInferenceUtil.convertCalciteObject2HiveWritableObject(
              argsType[i], args[i]));
    }
    Object result = udfInstance.evaluate(
        deferredJavaObjectArray);
    return TypeInferenceUtil.convertHiveObject2CalciteObject(
        outputObjectInspector,
        result);
  } catch (Exception e) {
    throw new RuntimeException("call hive udf error", e);
  }
}

Source File: ToStringArrayUDFTest.java From incubator-hivemall with Apache License 2.0

6 votes

@Test
public void testTextArrayInput() throws HiveException, IOException {
    List<String> input = new ArrayList<String>(2);
    input.add("1");
    input.add("2");

    ToStringArrayUDF udf = new ToStringArrayUDF();
    udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector)});

    DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(input)};
    List<String> output = udf.evaluate(args);

    Assert.assertEquals(input, output);

    udf.close();
}

Source File: FeatureHashingUDFTest.java From incubator-hivemall with Apache License 2.0

6 votes

@Test
public void testEvaluateListLibsvm() throws HiveException, IOException {
    FeatureHashingUDF udf = new FeatureHashingUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableStringObjectInspector),
            HiveUtils.getConstStringObjectInspector("-libsvm")});

    DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(
        WritableUtils.val("apple:3", "orange:2", "banana", "0:1"))};

    List<String> expected = Arrays.asList(
        FeatureHashingUDF.mhash("apple", MurmurHash3.DEFAULT_NUM_FEATURES) + ":3",
        FeatureHashingUDF.mhash("orange", MurmurHash3.DEFAULT_NUM_FEATURES) + ":2",
        FeatureHashingUDF.mhash("banana", MurmurHash3.DEFAULT_NUM_FEATURES) + ":1", "0:1");
    Collections.sort(expected);
    Assert.assertEquals(expected, udf.evaluate(args));

    udf.close();
}

Source File: ArrayElementAtUDFTest.java From incubator-hivemall with Apache License 2.0

6 votes

@Test
public void testString() throws IOException, HiveException {
    ArrayElementAtUDF udf = new ArrayElementAtUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableStringObjectInspector),
            PrimitiveObjectInspectorFactory.javaIntObjectInspector});

    DeferredObject[] args = new DeferredObject[] {
            new GenericUDF.DeferredJavaObject(WritableUtils.val("s0", "s1", "s2")),
            new GenericUDF.DeferredJavaObject(1)};

    Assert.assertEquals(WritableUtils.val("s1"), udf.evaluate(args));

    udf.close();
}

Source File: LastElementUDFTest.java From incubator-hivemall with Apache License 2.0

5 votes

@Test
public void testNull() throws IOException, HiveException {
    LastElementUDF udf = new LastElementUDF();

    udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)});

    DeferredObject[] args = new DeferredObject[] {
            new GenericUDF.DeferredJavaObject(WritableUtils.toWritableList(new double[] {}))};

    Assert.assertNull(udf.evaluate(args));

    udf.close();
}

Source File: OkapiBM25UDFTest.java From incubator-hivemall with Apache License 2.0

5 votes

@Test(expected = HiveException.class)
public void testNumDocsWithTermIsLessThanOne() throws Exception {
    initializeUDFWithoutOptions();

    GenericUDF.DeferredObject[] args =
            new GenericUDF.DeferredObject[] {VALID_TERM_FREQ, VALID_DOC_LEN, VALID_AVG_DOC_LEN,
                    VALID_NUM_DOCS, new GenericUDF.DeferredJavaObject(new Integer(0))};

    udf.evaluate(args);
}

Source File: OkapiBM25UDFTest.java From incubator-hivemall with Apache License 2.0

5 votes

@Test(expected = HiveException.class)
public void testAvgDocLengthIsZero() throws Exception {
    initializeUDFWithoutOptions();

    GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[] {VALID_TERM_FREQ,
            VALID_DOC_LEN, new GenericUDF.DeferredJavaObject(new Double(0.0)), VALID_NUM_DOCS,
            VALID_NUM_DOCS_WITH_TERM};

    udf.evaluate(args);
}

Source File: MapKeyValuesUDFTest.java From incubator-hivemall with Apache License 2.0

5 votes

@Test
public void testStringDouble() throws HiveException, IOException {
    MapKeyValuesUDF udf = new MapKeyValuesUDF();

    udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardMapObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
        PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)});

    Map<String, DoubleWritable> input = new HashMap<>();
    for (int i = 0; i < 10; i++) {
        input.put("k" + i, new DoubleWritable(i));
    }

    GenericUDF.DeferredObject[] arguments =
            new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(input)};

    List<Object[]> actual = udf.evaluate(arguments);

    Assert.assertEquals(input.size(), actual.size());
    for (Object[] e : actual) {
        Assert.assertEquals(2, e.length);
        Object v = input.get(e[0]);
        Assert.assertEquals(e[1], v);
    }

    udf.close();
}

Source File: ArraySliceUDFTest.java From incubator-hivemall with Apache License 2.0

5 votes

@Test
public void testNullReturn() throws IOException, HiveException {
    ArraySliceUDF udf = new ArraySliceUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector),
            PrimitiveObjectInspectorFactory.writableIntObjectInspector,
            PrimitiveObjectInspectorFactory.writableIntObjectInspector});

    IntWritable offset = new IntWritable();
    IntWritable length = new IntWritable();
    DeferredObject arg1 = new GenericUDF.DeferredJavaObject(offset);
    DeferredObject arg2 = new GenericUDF.DeferredJavaObject(length);

    DeferredObject[] args =
            new DeferredObject[] {
                    new GenericUDF.DeferredJavaObject(Arrays.asList("zero", "one", "two",
                        "three", "four", "five", "six", "seven", "eight", "nine", "ten")),
                    arg1, arg2};


    offset.set(-12);
    length.set(0);
    List<Object> actual = udf.evaluate(args);
    Assert.assertNull(actual);

    udf.close();

}

Source File: FirstElementUDFTest.java From incubator-hivemall with Apache License 2.0

5 votes

@Test
public void testNull() throws IOException, HiveException {
    FirstElementUDF udf = new FirstElementUDF();

    udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)});

    DeferredObject[] args = new DeferredObject[] {
            new GenericUDF.DeferredJavaObject(WritableUtils.toWritableList(new double[] {}))};

    Assert.assertNull(udf.evaluate(args));

    udf.close();
}

Source File: TestUtils.java From incubator-hivemall with Apache License 2.0

5 votes

public static <T extends GenericUDF> void testGenericUDFSerialization(@Nonnull Class<T> clazz,
        @Nonnull ObjectInspector[] ois, @Nonnull Object[] row)
        throws HiveException, IOException {
    final T udf;
    try {
        udf = clazz.newInstance();
    } catch (InstantiationException | IllegalAccessException e) {
        throw new HiveException(e);
    }

    udf.initialize(ois);

    // serialization after initialization
    byte[] serialized = serializeObjectByKryo(udf);
    deserializeObjectByKryo(serialized, clazz);

    int size = row.length;
    GenericUDF.DeferredObject[] rowDeferred = new GenericUDF.DeferredObject[size];
    for (int i = 0; i < size; i++) {
        rowDeferred[i] = new GenericUDF.DeferredJavaObject(row[i]);
    }

    udf.evaluate(rowDeferred);

    // serialization after evaluating row
    serialized = serializeObjectByKryo(udf);
    TestUtils.deserializeObjectByKryo(serialized, clazz);

    udf.close();
}

Source File: ArrayToStrUDFTest.java From incubator-hivemall with Apache License 2.0

5 votes

@Test
public void testNoSep() throws HiveException, IOException {
    ArrayToStrUDF udf = new ArrayToStrUDF();

    udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.javaIntObjectInspector)});

    DeferredObject[] args =
            new DeferredObject[] {new GenericUDF.DeferredJavaObject(Arrays.asList(1, 2, 3))};

    Assert.assertEquals("1,2,3", udf.evaluate(args));

    udf.close();
}

Source File: ArraySliceUDFTest.java From incubator-hivemall with Apache License 2.0

4 votes

@Test
public void testNonNullReturn() throws IOException, HiveException {
    ArraySliceUDF udf = new ArraySliceUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector),
            PrimitiveObjectInspectorFactory.writableIntObjectInspector,
            PrimitiveObjectInspectorFactory.writableIntObjectInspector});

    IntWritable offset = new IntWritable();
    IntWritable length = new IntWritable();
    DeferredObject arg1 = new GenericUDF.DeferredJavaObject(offset);
    DeferredObject arg2 = new GenericUDF.DeferredJavaObject(length);
    DeferredObject nullarg = new GenericUDF.DeferredJavaObject(null);

    DeferredObject[] args =
            new DeferredObject[] {
                    new GenericUDF.DeferredJavaObject(Arrays.asList("zero", "one", "two",
                        "three", "four", "five", "six", "seven", "eight", "nine", "ten")),
                    arg1, arg2};

    offset.set(0);
    length.set(3);
    List<Object> actual = udf.evaluate(args);
    Assert.assertEquals(Arrays.asList("zero", "one", "two"), actual);

    offset.set(1);
    length.set(-2);
    actual = udf.evaluate(args);
    Assert.assertEquals(
        Arrays.asList("one", "two", "three", "four", "five", "six", "seven", "eight"), actual);

    offset.set(1);
    length.set(0);
    actual = udf.evaluate(args);
    Assert.assertEquals(Collections.emptyList(), actual);

    offset.set(-1);
    length.set(0);
    actual = udf.evaluate(args);
    Assert.assertEquals(Collections.emptyList(), actual);

    offset.set(6);
    args[2] = nullarg;
    actual = udf.evaluate(args);
    Assert.assertEquals(Arrays.asList("six", "seven", "eight", "nine", "ten"), actual);

    udf.close();
}

Java Code Examples for org.apache.hadoop.hive.ql.udf.generic.GenericUDF#DeferredJavaObject