Java Code Examples for org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator#iterate()

The following examples show how to use org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator#iterate() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TransposeAndDotUDAFTest.java From incubator-hivemall with Apache License 2.0

5 votes

@Test
public void test() throws Exception {
    final TransposeAndDotUDAF tad = new TransposeAndDotUDAF();

    final double[][] matrix0 = new double[][] {{1, -2}, {-1, 3}};
    final double[][] matrix1 = new double[][] {{1, 2}, {3, 4}};

    final ObjectInspector[] OIs = new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableDoubleObjectInspector),
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)};
    final GenericUDAFEvaluator evaluator =
            tad.getEvaluator(new SimpleGenericUDAFParameterInfo(OIs, false, false));
    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, OIs);
    TransposeAndDotUDAF.TransposeAndDotUDAFEvaluator.TransposeAndDotAggregationBuffer agg =
            (TransposeAndDotUDAF.TransposeAndDotUDAFEvaluator.TransposeAndDotAggregationBuffer) evaluator.getNewAggregationBuffer();
    evaluator.reset(agg);
    for (int i = 0; i < matrix0.length; i++) {
        evaluator.iterate(agg, new Object[] {WritableUtils.toWritableList(matrix0[i]),
                WritableUtils.toWritableList(matrix1[i])});
    }

    final double[][] answer = new double[][] {{-2.0, -2.0}, {7.0, 8.0}};

    for (int i = 0; i < answer.length; i++) {
        Assert.assertArrayEquals(answer[i], agg.aggMatrix[i], 0.d);
    }
}

Example 2

Source File: MergeTest.java From hive-funnel-udf with Apache License 2.0

5 votes

@Test(expected = UDFArgumentTypeException.class)
public void testCompleteFunnelSizeMismatch() throws HiveException {
    Merge udaf = new Merge();
    ObjectInspector[] inputObjectInspectorList = new ObjectInspector[]{
            ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector)
    };

    GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(inputObjectInspectorList, false, false);
    GenericUDAFEvaluator udafEvaluator = udaf.getEvaluator(paramInfo);

    ObjectInspector outputObjectInspector = udafEvaluator.init(Mode.COMPLETE, inputObjectInspectorList);

    // Setup two funnels, different sizes.
    List<Long> funnel1 = new ArrayList<>();
    funnel1.add(1L);
    funnel1.add(1L);
    funnel1.add(0L);

    List<Long> funnel2 = new ArrayList<>();
    funnel2.add(1L);
    funnel2.add(0L);

    Object[] parameters1 = new Object[]{funnel1};
    Object[] parameters2 = new Object[]{funnel2};

    // Should cause an error when merging funnels of different sizes
    AggregationBuffer agg = udafEvaluator.getNewAggregationBuffer();
    udafEvaluator.reset(agg);
    udafEvaluator.iterate(agg, parameters1);
    udafEvaluator.iterate(agg, parameters2);
}

Example 3

Source File: FunnelTest.java From hive-funnel-udf with Apache License 2.0

4 votes

@Test
public void testComplete() throws HiveException {
    Funnel udaf = new Funnel();

    ObjectInspector[] inputObjectInspectorList = new ObjectInspector[]{
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, // action_column
        PrimitiveObjectInspectorFactory.javaLongObjectInspector,   // timestamp_column
        ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector), // funnel_step_1
        ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector) // funnel_step_1
    };

    GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(inputObjectInspectorList, false, false);
    GenericUDAFEvaluator udafEvaluator = udaf.getEvaluator(paramInfo);

    ObjectInspector outputObjectInspector = udafEvaluator.init(Mode.COMPLETE, inputObjectInspectorList);

    // Order will be "alpha, beta, gamma, delta" when ordered on timestamp_column
    // Funnel is "beta" -> "gamma" -> "epsilon"
    // Should return [1, 1, 0] as we don't have an epsilon
    Object[] parameters1 = new Object[]{ "beta", 200L, new ArrayList<Object>(), Arrays.asList("beta", "BAD"), null, "gamma", Arrays.asList("epsilon")}; // Test empty list funnel step, and null in funnel step
    Object[] parameters2 = new Object[]{"alpha", 100L,   Arrays.asList("beta", "BAD"), "gamma", Arrays.asList("epsilon")};
    Object[] parameters3 = new Object[]{"delta", 400L,   Arrays.asList("beta", "BAD"), "gamma", Arrays.asList("epsilon")};
    Object[] parameters4 = new Object[]{"gamma", 200L,   Arrays.asList("beta", "BAD"), "gamma", Arrays.asList("epsilon")}; // gamma and beta happen at the same time, beta should come first (sorted on action after timestamp)
    Object[] parameters5 = new Object[]{   null, 800L,   Arrays.asList("beta", "BAD"), "gamma", Arrays.asList("epsilon")}; // Check null action_column
    Object[] parameters6 = new Object[]{"omega", null,   Arrays.asList("beta", "BAD"), "gamma", Arrays.asList("epsilon")}; // Check null timestamp

    // Process the data
    AggregationBuffer agg = udafEvaluator.getNewAggregationBuffer();
    udafEvaluator.reset(agg);
    udafEvaluator.iterate(agg, parameters1);
    udafEvaluator.iterate(agg, parameters2);
    udafEvaluator.iterate(agg, parameters3);
    udafEvaluator.iterate(agg, parameters4);
    udafEvaluator.iterate(agg, parameters5);
    udafEvaluator.iterate(agg, parameters6);
    Object result = udafEvaluator.terminate(agg);

    // Expected
    List<Long> expected = new ArrayList<>();
    expected.add(1L);
    expected.add(1L);
    expected.add(0L);

    Assert.assertEquals(expected, result);
}

Example 4

Source File: FunnelTest.java From hive-funnel-udf with Apache License 2.0

4 votes

@Test
public void testPartial1() throws HiveException {
    Funnel udaf = new Funnel();

    ObjectInspector[] inputObjectInspectorList = new ObjectInspector[]{
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, // action_column
        PrimitiveObjectInspectorFactory.javaLongObjectInspector,   // timestamp_column
        ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector), // funnel_step_1
        ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector) // funnel_step_1
    };

    GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(inputObjectInspectorList, false, false);
    GenericUDAFEvaluator udafEvaluator = udaf.getEvaluator(paramInfo);

    ObjectInspector outputObjectInspector = udafEvaluator.init(Mode.PARTIAL1, inputObjectInspectorList);

    // Order will be "alpha, beta, gamma, delta" when ordered on timestamp_column
    // Funnel is "beta" -> "gamma" -> "epsilon"
    // Should return [1, 1, 0] as we don't have an epsilon
    Object[] parameters1 = new Object[]{ "beta", 200L, Arrays.asList("beta"), "gamma", Arrays.asList("epsilon")};
    Object[] parameters2 = new Object[]{"alpha", 100L, Arrays.asList("beta"), "gamma", Arrays.asList("epsilon")};
    Object[] parameters3 = new Object[]{"delta", 400L, Arrays.asList("beta"), "gamma", Arrays.asList("epsilon")};
    Object[] parameters4 = new Object[]{"gamma", 300L, Arrays.asList("beta"), "gamma", Arrays.asList("epsilon")};

    // Process the data
    AggregationBuffer agg = udafEvaluator.getNewAggregationBuffer();
    udafEvaluator.reset(agg);
    udafEvaluator.iterate(agg, parameters1);
    udafEvaluator.iterate(agg, parameters2);
    udafEvaluator.iterate(agg, parameters3);
    udafEvaluator.iterate(agg, parameters4);
    Object result = udafEvaluator.terminatePartial(agg);

    // Expected partial output
    List<Object> expected = new ArrayList<>();
    expected.add(Arrays.asList("beta", "gamma"));
    expected.add(Arrays.asList(200L, 300L));
    expected.add(Arrays.asList("beta", null, "gamma", null, "epsilon", null));

    Assert.assertEquals(expected, result);
}

Example 5

Source File: MergeTest.java From hive-funnel-udf with Apache License 2.0

4 votes

@Test
public void testComplete() throws HiveException {
    Merge udaf = new Merge();
    ObjectInspector[] inputObjectInspectorList = new ObjectInspector[]{
            ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector)
    };

    GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(inputObjectInspectorList, false, false);
    GenericUDAFEvaluator udafEvaluator = udaf.getEvaluator(paramInfo);

    ObjectInspector outputObjectInspector = udafEvaluator.init(Mode.COMPLETE, inputObjectInspectorList);

    List<Long> funnel1 = new ArrayList<>();
    funnel1.add(1L);
    funnel1.add(1L);
    funnel1.add(0L);

    List<Long> funnel2 = new ArrayList<>();
    funnel2.add(1L);
    funnel2.add(0L);
    funnel2.add(0L);

    Object[] parameters1 = new Object[]{null};
    Object[] parameters2 = new Object[]{funnel1};
    Object[] parameters3 = new Object[]{funnel2};

    AggregationBuffer agg = udafEvaluator.getNewAggregationBuffer();
    udafEvaluator.reset(agg);
    udafEvaluator.iterate(agg, parameters1);
    udafEvaluator.iterate(agg, parameters2);
    udafEvaluator.iterate(agg, parameters3);
    Object result = udafEvaluator.terminate(agg);

    // Expected
    List<Long> expected = new ArrayList<>();
    expected.add(2L);
    expected.add(1L);
    expected.add(0L);

    Assert.assertEquals(expected, result);
}