Java Code Examples for org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory#getStandardStructObjectInspector()

@Nonnull
protected StructObjectInspector getReturnOI(@Nonnull ObjectInspector featureOutputOI) {
    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();

    fieldNames.add("feature");
    fieldOIs.add(featureOutputOI);
    fieldNames.add("weight");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);
    if (useCovariance()) {
        fieldNames.add("covar");
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);
    }

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 

@Nonnull
private static StructObjectInspector internalMergeOI() {
    List<String> fieldNames = new ArrayList<>();
    List<ObjectInspector> fieldOIs = new ArrayList<>();

    fieldNames.add("tp");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
    fieldNames.add("totalActual");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
    fieldNames.add("totalPredicted");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
    fieldNames.add("beta");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
    fieldNames.add("average");
    fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 

@Override
public ObjectInspector init(final Mode mode, final ObjectInspector[] inspectors) throws HiveException {
  final ObjectInspector resultInspector = super.init(mode, inspectors);
  if ((mode == Mode.PARTIAL1) || (mode == Mode.COMPLETE)) {
    // input is original data
    if (inspectors.length > 2) {
      summaryModeInspector_ = (PrimitiveObjectInspector) inspectors[2];
    }
  }
  if ((mode == Mode.PARTIAL1) || (mode == Mode.PARTIAL2)) {
    // intermediate results need to include the nominal number of entries and the summary mode
    return ObjectInspectorFactory.getStandardStructObjectInspector(
      Arrays.asList(NOMINAL_NUM_ENTRIES_FIELD, SUMMARY_MODE_FIELD, SKETCH_FIELD),
      Arrays.asList(
        PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.INT),
        PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.STRING),
        PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.BINARY)
      )
    );
  }
  return resultInspector;
}
 

@Override
public ObjectInspector init(final Mode mode, final ObjectInspector[] inspectors) throws HiveException {
  super.init(mode, inspectors);
  if ((mode == Mode.PARTIAL1) || (mode == Mode.COMPLETE)) {
    // input is original data
    sketchInspector_ = (PrimitiveObjectInspector) inspectors[0];
    if (inspectors.length > 1) {
      nominalNumEntriesInspector_ = (PrimitiveObjectInspector) inspectors[1];
    }
  } else {
    // input for PARTIAL2 and FINAL is the output from PARTIAL1
    intermediateInspector_ = (StructObjectInspector) inspectors[0];
  }

  if ((mode == Mode.PARTIAL1) || (mode == Mode.PARTIAL2)) {
    // intermediate results need to include the the nominal number of entries
    return ObjectInspectorFactory.getStandardStructObjectInspector(
      Arrays.asList(NOMINAL_NUM_ENTRIES_FIELD, SKETCH_FIELD),
      Arrays.asList(
        PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.INT),
        PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.BINARY)
      )
    );
  }
  // final results include just the sketch
  return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.BINARY);
}
 

public static StructObjectInspector getObjectInspectorFor(final ImmutableList<String> names,
                                                          final ImmutableList<String> columnTypeNames) {
  List<ObjectInspector> inspectors = new ArrayList<>();
  for (int i = 0; i < columnTypeNames.size(); i++) {
    inspectors.add(createJavaObjectInspectorFromFieldSchema(columnTypeNames.get(i)));
  }
  return ObjectInspectorFactory.getStandardStructObjectInspector(
      names,
      inspectors);
}
 

private ObjectInspector createObjectInspector() throws TsFileSerDeException {
  List<ObjectInspector> columnOIs = new ArrayList<>(columnNames.size());

  // At this point we've verified the types are correct.
  for(int i = 0; i < columnNames.size(); i++) {
    columnOIs.add(i, createObjectInspectorWorker(columnTypes.get(i)));
  }
  return ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, columnOIs);
}
 

private static StructObjectInspector internalMergeOI() {
    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();

    fieldNames.add("size");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
    fieldNames.add("sum");
    fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.writableDoubleObjectInspector));
    fieldNames.add("count");
    fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.writableLongObjectInspector));

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 

@Override
public ObjectInspector init(@Nonnull Mode mode, @Nonnull ObjectInspector[] argOIs)
        throws HiveException {
    super.init(mode, argOIs);

    // initialize input
    if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data
        this.yhatOI = HiveUtils.asIntegerOI(argOIs, 0);
    } else {// from partial aggregation
        this.internalMergeOI = (StandardMapObjectInspector) argOIs[0];
        this.keyOI = HiveUtils.asIntOI(internalMergeOI.getMapKeyObjectInspector());
        this.valueOI = HiveUtils.asIntOI(internalMergeOI.getMapValueObjectInspector());
    }

    // initialize output
    final ObjectInspector outputOI;
    if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {// terminatePartial
        outputOI = ObjectInspectorFactory.getStandardMapObjectInspector(
            PrimitiveObjectInspectorFactory.javaIntObjectInspector,
            PrimitiveObjectInspectorFactory.javaIntObjectInspector);
    } else {// terminate
        List<String> fieldNames = new ArrayList<>(3);
        List<ObjectInspector> fieldOIs = new ArrayList<>(3);
        fieldNames.add("label");
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
        fieldNames.add("probability");
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
        fieldNames.add("probabilities");
        fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector));
        outputOI = ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames,
            fieldOIs);
    }
    return outputOI;
}
 

@Before
public void setUp() throws Exception {
    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();

    fieldNames.add("wcList");
    fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector));

    fieldNames.add("lambdaMap");
    fieldOIs.add(ObjectInspectorFactory.getStandardMapObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
        ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.javaFloatObjectInspector)));

    fieldNames.add("topics");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);

    fieldNames.add("alpha");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);

    fieldNames.add("delta");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);

    partialOI = new ObjectInspector[4];
    partialOI[0] =
            ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);

    words = new String[] {"fruits", "vegetables", "healthy", "flu", "apples", "oranges", "like",
            "avocados", "colds", "colds", "avocados", "oranges", "like", "apples", "flu",
            "healthy", "vegetables", "fruits"};
    labels = new int[] {0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1};
    lambdas = new float[] {0.3339331f, 0.3324783f, 0.33209667f, 3.2804057E-4f, 3.0303953E-4f,
            2.4860457E-4f, 2.41481E-4f, 2.3554532E-4f, 1.352576E-4f, 0.1660153f, 0.16596903f,
            0.1659654f, 0.1659627f, 0.16593699f, 0.1659259f, 0.0017611005f, 0.0015791848f,
            8.84464E-4f};
}
 

/**
 * Receives the passed in argument object inspectors and returns the desired
 * return type's object inspector to inform hive of return type of UDAF.
 *
 * @param mode
 *          Mode (i.e. PARTIAL 1, COMPLETE...) for determining input and output
 *          object inspector type.
 * @param parameters
 *          List of object inspectors for input arguments.
 * @return The object inspector type indicates the UDAF return type (i.e.
 *         returned type of terminate(...)).
 */
@Override
public ObjectInspector init(final Mode mode, final ObjectInspector[] parameters) throws HiveException {
  super.init(mode, parameters);

  if ((mode == Mode.PARTIAL1) || (mode == Mode.COMPLETE)) {
    inputInspector_ = (PrimitiveObjectInspector) parameters[0];
    if (parameters.length > 1) {
      lgKInspector_ = (PrimitiveObjectInspector) parameters[1];
    }
    if (parameters.length > 2) {
      seedInspector_ = (PrimitiveObjectInspector) parameters[2];
    }
  } else {
    // mode = partial2 || final
    intermediateInspector_ = (StandardStructObjectInspector) parameters[0];
  }

  if ((mode == Mode.PARTIAL1) || (mode == Mode.PARTIAL2)) {
    // intermediate results need to include the lgK and the target HLL type
    return ObjectInspectorFactory.getStandardStructObjectInspector(
      Arrays.asList(LG_K_FIELD, SEED_FIELD, SKETCH_FIELD),
      Arrays.asList(
        PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.INT),
        PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.LONG),
        PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.BINARY)
      )
    );
  }
  // final results include just the sketch
  return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.BINARY);
}
 

private Map<String, AttributeValue> getSerializedItem(List<String> attributeNames, List<ObjectInspector> colOIs,
                                                      Map<String, String> typeMapping, List<Object> rowData,
                                                      boolean nullSerialization)
    throws SerDeException {
  List<String> colTypes = Lists.newArrayList();
  List<String> colMappings = Lists.newArrayList();
  for (int i = 0; i < attributeNames.size(); i++) {
    String attributeName = attributeNames.get(i);
    colTypes.add(colOIs.get(i).getTypeName());
    if (!HiveDynamoDBTypeFactory.isHiveDynamoDBItemMapType(colOIs.get(i)) ||
        typeMapping.containsKey(attributeName)) {
      colMappings.add(attributeName + ":" + attributeName);
    }
  }

  List<String> typeMapList = Lists.newArrayList();
  for (Map.Entry<String, String> colType : typeMapping.entrySet()) {
    typeMapList.add(colType.getKey() + ":" + colType.getValue());
  }

  Properties props = new Properties();
  props.setProperty(serdeConstants.LIST_COLUMNS, StringUtils.join(attributeNames, ","));
  props.setProperty(serdeConstants.LIST_COLUMN_TYPES, StringUtils.join(colTypes, ","));
  props.setProperty(DynamoDBConstants.DYNAMODB_COLUMN_MAPPING, StringUtils.join(colMappings, ","));
  props.setProperty(DynamoDBConstants.DYNAMODB_TYPE_MAPPING, StringUtils.join(typeMapList, ","));
  props.setProperty(DynamoDBConstants.DYNAMODB_NULL_SERIALIZATION, Boolean.toString(nullSerialization));

  DynamoDBSerDe serde = new DynamoDBSerDe();
  serde.initialize(null, props);

  StructObjectInspector rowOI = ObjectInspectorFactory.getStandardStructObjectInspector(attributeNames, colOIs);
  DynamoDBItemWritable item = (DynamoDBItemWritable) serde.serialize(rowData, rowOI);
  return item.getItem();
}
 

@Override
public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (argOIs.length < 3) {
        throw new UDFArgumentException("binarize_label(int/long positive, "
                + "int/long negative, *) takes at least three arguments");
    }
    this.positiveOI = HiveUtils.asIntCompatibleOI(argOIs[0]);
    this.negativeOI = HiveUtils.asIntCompatibleOI(argOIs[1]);

    this.positiveObjs = new Object[argOIs.length - 1];
    positiveObjs[positiveObjs.length - 1] = 1;
    this.negativeObjs = new Object[argOIs.length - 1];
    negativeObjs[negativeObjs.length - 1] = 0;

    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
    for (int i = 2; i < argOIs.length; i++) {
        fieldNames.add("c" + (i - 2));
        // Use negative label ObjectInspector here. OIs for positive
        // label and negative labels must be same.
        fieldOIs.add(argOIs[i]);
    }
    fieldNames.add("c" + (argOIs.length - 2));
    fieldOIs.add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 

private void checkInitialize() throws Exception {
	if (initialized) {
		return;
	}

	JobConf jobConf = confWrapper.conf();
	Object serdeLib = Class.forName(serDeInfo.getSerializationLib()).newInstance();
	Preconditions.checkArgument(serdeLib instanceof Serializer && serdeLib instanceof Deserializer,
			"Expect a SerDe lib implementing both Serializer and Deserializer, but actually got "
					+ serdeLib.getClass().getName());
	this.recordSerDe = (Serializer) serdeLib;
	ReflectionUtils.setConf(recordSerDe, jobConf);

	// TODO: support partition properties, for now assume they're same as table properties
	SerDeUtils.initializeSerDe((Deserializer) recordSerDe, jobConf, tableProperties, null);

	this.formatFields = allColumns.length - partitionColumns.length;
	this.hiveConversions = new HiveObjectConversion[formatFields];
	this.converters = new DataFormatConverter[formatFields];
	List<ObjectInspector> objectInspectors = new ArrayList<>(hiveConversions.length);
	for (int i = 0; i < formatFields; i++) {
		DataType type = allTypes[i];
		ObjectInspector objectInspector = HiveInspectors.getObjectInspector(type);
		objectInspectors.add(objectInspector);
		hiveConversions[i] = HiveInspectors.getConversion(
				objectInspector, type.getLogicalType(), hiveShim);
		converters[i] = DataFormatConverters.getConverterForDataType(type);
	}

	this.formatInspector = ObjectInspectorFactory.getStandardStructObjectInspector(
			Arrays.asList(allColumns).subList(0, formatFields),
			objectInspectors);
	this.initialized = true;
}
 

@Override
public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (argOIs.length != 2 && argOIs.length != 3) {
        throw new UDFArgumentException(getClass().getSimpleName()
                + " takes 2 or 3 arguments: array<string> x, map<long, double> colNorms "
                + "[, CONSTANT STRING options]: " + Arrays.toString(argOIs));
    }

    this.rowOI = HiveUtils.asListOI(argOIs[0]);
    HiveUtils.validateFeatureOI(rowOI.getListElementObjectInspector());

    this.colNormsOI = HiveUtils.asMapOI(argOIs[1]);

    processOptions(argOIs);

    this.rnd = RandomNumberGeneratorFactory.createPRNG(1001);
    this.colNorms = null;
    this.colProbs = null;

    ArrayList<String> fieldNames = new ArrayList<String>();
    fieldNames.add("j");
    fieldNames.add("k");
    fieldNames.add("b_jk");

    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
    if (parseFeatureAsInt) {
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
    } else {
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableStringObjectInspector);
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableStringObjectInspector);
    }
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 

@Override
public StructObjectInspector initialize(@Nonnull ObjectInspector[] argOIs)
        throws UDFArgumentException {
    if (argOIs.length != 2 && argOIs.length != 3) {
        showHelp("Invalid argment length=" + argOIs.length);
    }
    processOptions(argOIs);

    ListObjectInspector listOI = HiveUtils.asListOI(argOIs, 0);
    ObjectInspector elemOI = listOI.getListElementObjectInspector();
    this.featureListOI = listOI;
    if (HiveUtils.isNumberOI(elemOI)) {
        this.featureElemOI = HiveUtils.asDoubleCompatibleOI(elemOI);
        this.denseInput = true;
        this.matrixBuilder = new DenseDMatrixBuilder(8192);
    } else if (HiveUtils.isStringOI(elemOI)) {
        this.featureElemOI = HiveUtils.asStringOI(elemOI);
        this.denseInput = false;
        this.matrixBuilder = new SparseDMatrixBuilder(8192);
    } else {
        throw new UDFArgumentException(
            "train_xgboost takes array<double> or array<string> for the first argument: "
                    + listOI.getTypeName());
    }
    this.targetOI = HiveUtils.asDoubleCompatibleOI(argOIs, 1);
    this.labels = new FloatArrayList(1024);

    final List<String> fieldNames = new ArrayList<>(2);
    final List<ObjectInspector> fieldOIs = new ArrayList<>(2);
    fieldNames.add("model_id");
    fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
    fieldNames.add("model");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableStringObjectInspector);
    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 

@Override
public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
	return ObjectInspectorFactory.getStandardStructObjectInspector(
		Collections.singletonList("col1"),
		Collections.singletonList(PrimitiveObjectInspectorFactory.javaStringObjectInspector));
}
 

@Override
public ObjectInspector initialize(@Nonnull ObjectInspector[] argOIs)
        throws UDFArgumentException {
    if (argOIs.length < 1 || argOIs.length > 2) {
        throw new UDFArgumentException(
            "_FUNC_(double|array<double> x [, const string options]) takes 1 or 2 arguments: "
                    + Arrays.toString(argOIs));
    }

    this._params = new Parameters();
    if (argOIs.length == 2) {
        String options = HiveUtils.getConstString(argOIs[1]);
        processOptions(options);
    }

    ObjectInspector argOI0 = argOIs[0];
    PrimitiveObjectInspector xOI = HiveUtils.asDoubleCompatibleOI(argOI0);
    this._sst = new SingularSpectrumTransform(_params, xOI);

    this._scores = new double[1];

    final Object[] result;
    final ArrayList<String> fieldNames = new ArrayList<String>();
    final ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
    fieldNames.add("changepoint_score");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
    if (_params.changepointThreshold != -1d) {
        fieldNames.add("is_changepoint");
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableBooleanObjectInspector);
        result = new Object[2];
        this._isChangepoint = new BooleanWritable(false);
        result[1] = _isChangepoint;
    } else {
        result = new Object[1];
    }
    this._changepointScore = new DoubleWritable(0.d);
    result[0] = _changepointScore;
    this._result = result;

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 

@Override
public ObjectInspector initialize(@Nonnull ObjectInspector[] argOIs)
        throws UDFArgumentException {
    if (argOIs.length < 1 || argOIs.length > 2) {
        throw new UDFArgumentException(
            "_FUNC_(double|array<double> x [, const string options]) takes 1 or 2 arguments: "
                    + Arrays.toString(argOIs));
    }

    this._params = new Parameters();
    if (argOIs.length == 2) {
        String options = HiveUtils.getConstString(argOIs[1]);
        processOptions(options);
    }

    ObjectInspector argOI0 = argOIs[0];
    if (HiveUtils.isListOI(argOI0)) {
        ListObjectInspector listOI = HiveUtils.asListOI(argOI0);
        this._changeFinder = new ChangeFinder2D(_params, listOI);
    } else if (HiveUtils.isNumberOI(argOI0)) {
        PrimitiveObjectInspector xOI = HiveUtils.asDoubleCompatibleOI(argOI0);
        this._changeFinder = new ChangeFinder1D(_params, xOI);
    }

    this._scores = new double[2];

    final Object[] result;
    final ArrayList<String> fieldNames = new ArrayList<String>();
    final ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
    fieldNames.add("outlier_score");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
    fieldNames.add("changepoint_score");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
    if (_params.outlierThreshold != -1d) {
        fieldNames.add("is_outlier");
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableBooleanObjectInspector);
        this._isOutlier = new BooleanWritable(false);
        if (_params.changepointThreshold != -1d) {
            fieldNames.add("is_changepoint");
            fieldOIs.add(PrimitiveObjectInspectorFactory.writableBooleanObjectInspector);
            result = new Object[4];
            this._isChangepoint = new BooleanWritable(false);
            result[3] = _isChangepoint;
        } else {
            result = new Object[3];
        }
        result[2] = _isOutlier;
    } else {
        result = new Object[2];
    }
    this._outlierScore = new DoubleWritable(0d);
    result[0] = _outlierScore;
    this._changepointScore = new DoubleWritable(0d);
    result[1] = _changepointScore;
    this._result = result;

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 

@Test
public void testPartial2() throws HiveException {
    Funnel udaf = new Funnel();

    // Construct the object inspector for udaf evaluator
    ObjectInspector[] inputObjectInspectorList = new ObjectInspector[]{
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, // action_column
        PrimitiveObjectInspectorFactory.javaLongObjectInspector,   // timestamp_column
        ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector), // funnel_step_1
        ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector) // funnel_step_1
    };
    GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(inputObjectInspectorList, false, false);
    GenericUDAFEvaluator udafEvaluator = udaf.getEvaluator(paramInfo);

    // Construct the struct object inspector
    List<String> fieldNames = new ArrayList<>();
    fieldNames.add("action");
    fieldNames.add("timestamp");
    fieldNames.add("funnel");
    List<ObjectInspector> fieldInspectors = new ArrayList<>();
    fieldInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector));
    fieldInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector));
    fieldInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector));
    ObjectInspector structObjectInspector = ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldInspectors);
    ObjectInspector[] evaluatorInputObjectInspectorList = new ObjectInspector[]{structObjectInspector};

    ObjectInspector outputObjectInspector = udafEvaluator.init(Mode.PARTIAL2, evaluatorInputObjectInspectorList);

    // Create the two structs to merge
    List<Object> parameter1 = new ArrayList<>();
    parameter1.add(Arrays.asList("beta"));
    parameter1.add(Arrays.asList(300L));
    parameter1.add(Arrays.asList("alpha", null, "beta", null, "gamma", null, "epsilon", null));

    List<Object> parameter2 = new ArrayList<>();
    parameter2.add(Arrays.asList("gamma", "alpha"));
    parameter2.add(Arrays.asList(400L, 200L));
    parameter1.add(Arrays.asList("alpha", null, "beta", null, "gamma", null, "epsilon", null));

    // Process the data
    AggregationBuffer agg = udafEvaluator.getNewAggregationBuffer();
    udafEvaluator.reset(agg);
    udafEvaluator.merge(agg, parameter1);
    udafEvaluator.merge(agg, parameter2);
    Object result = udafEvaluator.terminatePartial(agg);

    // Expected
    List<Object> expected = new ArrayList<>();
    expected.add(Arrays.asList("beta", "gamma", "alpha"));
    expected.add(Arrays.asList(300L, 400L, 200L));
    expected.add(Arrays.asList("alpha", null, "beta", null, "gamma", null, "epsilon", null));

    Assert.assertEquals(expected, result);
}
 

@Override
public ObjectInspector init(Mode mode, ObjectInspector[] OIs) throws HiveException {
    super.init(mode, OIs);

    if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {
        weightOI = HiveUtils.asDoubleCompatibleOI(OIs[0]);

        // set const values
        nBins = HiveUtils.getConstInt(OIs[1]);
        if (OIs.length == 3) {
            autoShrink = HiveUtils.getConstBoolean(OIs[2]);
        }

        // check value of `num_of_bins`
        if (nBins < 2) {
            throw new UDFArgumentException(
                "Only greater than or equal to 2 is accepted but " + nBins
                        + " was passed as `num_of_bins`.");
        }

        quantiles = getQuantiles();
    } else {
        structOI = (StructObjectInspector) OIs[0];
        autoShrinkField = structOI.getStructFieldRef("autoShrink");
        histogramField = structOI.getStructFieldRef("histogram");
        quantilesField = structOI.getStructFieldRef("quantiles");
        autoShrinkOI =
                (WritableBooleanObjectInspector) autoShrinkField.getFieldObjectInspector();
        histogramOI =
                (StandardListObjectInspector) histogramField.getFieldObjectInspector();
        quantilesOI =
                (StandardListObjectInspector) quantilesField.getFieldObjectInspector();
        histogramElOI =
                (WritableDoubleObjectInspector) histogramOI.getListElementObjectInspector();
        quantileOI =
                (WritableDoubleObjectInspector) quantilesOI.getListElementObjectInspector();
    }

    if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {
        final ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableBooleanObjectInspector);
        fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector));
        fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector));

        return ObjectInspectorFactory.getStandardStructObjectInspector(
            Arrays.asList("autoShrink", "histogram", "quantiles"), fieldOIs);
    } else {
        return ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
    }
}