Java Code Examples for org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator#merge()

The following examples show how to use org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator#merge() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: MergeTest.java From hive-funnel-udf with Apache License 2.0

5 votes

@Test
public void testPartial2() throws HiveException {
    Merge udaf = new Merge();
    ObjectInspector[] inputObjectInspectorList = new ObjectInspector[]{
            ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector)
    };

    GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(inputObjectInspectorList, false, false);
    GenericUDAFEvaluator udafEvaluator = udaf.getEvaluator(paramInfo);

    ObjectInspector outputObjectInspector = udafEvaluator.init(Mode.PARTIAL2, inputObjectInspectorList);

    // Setup the two partial results
    List<Long> partialResults1 = new ArrayList<>();
    partialResults1.add(1L);
    partialResults1.add(1L);
    partialResults1.add(0L);

    List<Long> partialResults2 = new ArrayList<>();
    partialResults2.add(1L);
    partialResults2.add(1L);
    partialResults2.add(1L);

    // Merge the partial results
    MergeAggregateBuffer agg = (MergeAggregateBuffer) udafEvaluator.getNewAggregationBuffer();
    udafEvaluator.reset(agg);
    udafEvaluator.merge(agg, partialResults1);
    udafEvaluator.merge(agg, partialResults2);
    Object result = udafEvaluator.terminatePartial(agg);

    // Expected results
    List<Long> expected = new ArrayList<>();
    expected.add(2L);
    expected.add(2L);
    expected.add(1L);

    Assert.assertEquals(expected, result);
}

Example 2

Source File: MergeTest.java From hive-funnel-udf with Apache License 2.0

5 votes

@Test(expected = UDFArgumentTypeException.class)
public void testPartial2FunnelSizeMismatch() throws HiveException {
    Merge udaf = new Merge();
    ObjectInspector[] inputObjectInspectorList = new ObjectInspector[]{
            ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector)
    };

    GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(inputObjectInspectorList, false, false);
    GenericUDAFEvaluator udafEvaluator = udaf.getEvaluator(paramInfo);

    ObjectInspector outputObjectInspector = udafEvaluator.init(Mode.PARTIAL2, inputObjectInspectorList);

    // Setup the two partial results, should fail when merging list of different sizes
    List<Long> partialResults1 = new ArrayList<>();
    partialResults1.add(1L);
    partialResults1.add(1L);

    List<Long> partialResults2 = new ArrayList<>();
    partialResults2.add(1L);
    partialResults2.add(0L);
    partialResults2.add(0L);

    // Merge the partial results, should throw error due to list size difference
    MergeAggregateBuffer agg = (MergeAggregateBuffer) udafEvaluator.getNewAggregationBuffer();
    udafEvaluator.reset(agg);
    udafEvaluator.merge(agg, partialResults1);
    udafEvaluator.merge(agg, partialResults2);
}

Example 3

Source File: FunnelTest.java From hive-funnel-udf with Apache License 2.0

4 votes

@Test
public void testPartial2() throws HiveException {
    Funnel udaf = new Funnel();

    // Construct the object inspector for udaf evaluator
    ObjectInspector[] inputObjectInspectorList = new ObjectInspector[]{
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, // action_column
        PrimitiveObjectInspectorFactory.javaLongObjectInspector,   // timestamp_column
        ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector), // funnel_step_1
        ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector) // funnel_step_1
    };
    GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(inputObjectInspectorList, false, false);
    GenericUDAFEvaluator udafEvaluator = udaf.getEvaluator(paramInfo);

    // Construct the struct object inspector
    List<String> fieldNames = new ArrayList<>();
    fieldNames.add("action");
    fieldNames.add("timestamp");
    fieldNames.add("funnel");
    List<ObjectInspector> fieldInspectors = new ArrayList<>();
    fieldInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector));
    fieldInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector));
    fieldInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector));
    ObjectInspector structObjectInspector = ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldInspectors);
    ObjectInspector[] evaluatorInputObjectInspectorList = new ObjectInspector[]{structObjectInspector};

    ObjectInspector outputObjectInspector = udafEvaluator.init(Mode.PARTIAL2, evaluatorInputObjectInspectorList);

    // Create the two structs to merge
    List<Object> parameter1 = new ArrayList<>();
    parameter1.add(Arrays.asList("beta"));
    parameter1.add(Arrays.asList(300L));
    parameter1.add(Arrays.asList("alpha", null, "beta", null, "gamma", null, "epsilon", null));

    List<Object> parameter2 = new ArrayList<>();
    parameter2.add(Arrays.asList("gamma", "alpha"));
    parameter2.add(Arrays.asList(400L, 200L));
    parameter1.add(Arrays.asList("alpha", null, "beta", null, "gamma", null, "epsilon", null));

    // Process the data
    AggregationBuffer agg = udafEvaluator.getNewAggregationBuffer();
    udafEvaluator.reset(agg);
    udafEvaluator.merge(agg, parameter1);
    udafEvaluator.merge(agg, parameter2);
    Object result = udafEvaluator.terminatePartial(agg);

    // Expected
    List<Object> expected = new ArrayList<>();
    expected.add(Arrays.asList("beta", "gamma", "alpha"));
    expected.add(Arrays.asList(300L, 400L, 200L));
    expected.add(Arrays.asList("alpha", null, "beta", null, "gamma", null, "epsilon", null));

    Assert.assertEquals(expected, result);
}

Example 4

Source File: FunnelTest.java From hive-funnel-udf with Apache License 2.0

4 votes

@Test
public void testFinal() throws HiveException {
    Funnel udaf = new Funnel();

    // Construct the object inspector for udaf evaluator
    ObjectInspector[] inputObjectInspectorList = new ObjectInspector[]{
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, // action_column
        PrimitiveObjectInspectorFactory.javaLongObjectInspector,   // timestamp_column
        ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector), // funnel_step_1
        ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector) // funnel_step_1
    };
    GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(inputObjectInspectorList, false, false);
    GenericUDAFEvaluator udafEvaluator = udaf.getEvaluator(paramInfo);

    // Construct the struct object inspector
    List<String> fieldNames = new ArrayList<>();
    fieldNames.add("action");
    fieldNames.add("timestamp");
    fieldNames.add("funnel");
    List<ObjectInspector> fieldInspectors = new ArrayList<>();
    fieldInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector));
    fieldInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector));
    fieldInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector));
    ObjectInspector structObjectInspector = ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldInspectors);
    ObjectInspector[] evaluatorInputObjectInspectorList = new ObjectInspector[]{structObjectInspector};

    ObjectInspector outputObjectInspector = udafEvaluator.init(Mode.FINAL, evaluatorInputObjectInspectorList);

    // Create the two structs to merge
    List<Object> parameter1 = new ArrayList<>();
    parameter1.add(Arrays.asList("beta"));
    parameter1.add(Arrays.asList(300L));
    parameter1.add(Arrays.asList("alpha", null, "beta", null, "gamma", null, "epsilon", null));

    List<Object> parameter2 = new ArrayList<>();
    parameter2.add(Arrays.asList("gamma", "alpha"));
    parameter2.add(Arrays.asList(400L, 200L));
    parameter2.add(Arrays.asList("alpha", null, "beta", null, "gamma", null, "epsilon", null));

    // Process the data
    AggregationBuffer agg = udafEvaluator.getNewAggregationBuffer();
    udafEvaluator.reset(agg);
    udafEvaluator.merge(agg, parameter1);
    udafEvaluator.merge(agg, parameter2);
    Object result = udafEvaluator.terminate(agg);

    // Expected
    List<Long> expected = Arrays.asList(1L, 1L, 1L, 0L);

    Assert.assertEquals(expected, result);
}