Java Code Examples for org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector#getMapValueObjectInspector()

private List<OneField> traverseMap(Object obj, MapObjectInspector moi)
        throws BadRecordException, IOException {
    List<OneField> complexRecord = new LinkedList<>();
    List<OneField> mapRecord = new LinkedList<>();
    ObjectInspector koi = moi.getMapKeyObjectInspector();
    ObjectInspector voi = moi.getMapValueObjectInspector();
    Map<?, ?> map = moi.getMap(obj);
    if (map == null) {
        throw new BadRecordException(
                "Illegal value NULL for Hive data type Map");
    } else if (map.isEmpty()) {
        traverseTuple(null, koi, complexRecord, true);
        traverseTuple(null, voi, complexRecord, true);
        addOneFieldToRecord(mapRecord, DataType.TEXT,
                HdfsUtilities.toString(complexRecord, mapkeyDelim));
    } else {
        for (Map.Entry<?, ?> entry : map.entrySet()) {
            traverseTuple(entry.getKey(), koi, complexRecord, true);
            traverseTuple(entry.getValue(), voi, complexRecord, true);
            addOneFieldToRecord(mapRecord, DataType.TEXT,
                    HdfsUtilities.toString(complexRecord, mapkeyDelim));
            complexRecord.clear();
        }
    }
    return mapRecord;
}
 

@Override
public Object convert(ObjectInspector objectInspector, Object o, TypeInfo odpsTypeInfo) {
  MapObjectInspector mapObjectInspector = (MapObjectInspector) objectInspector;
  ObjectInspector mapKeyObjectInspector = mapObjectInspector.getMapKeyObjectInspector();
  ObjectInspector mapValueObjectInspector = mapObjectInspector.getMapValueObjectInspector();
  TypeInfo mapKeyTypeInfo = ((MapTypeInfo) odpsTypeInfo).getKeyTypeInfo();
  TypeInfo mapValueTypeInfo = ((MapTypeInfo) odpsTypeInfo).getValueTypeInfo();

  Map map = mapObjectInspector.getMap(o);
  Map<Object, Object> newMap = new HashMap<>();
  for (Object k : map.keySet()) {
    Object v = map.get(k);
    newMap.put(HiveObjectConverter.convert(mapKeyObjectInspector, k, mapKeyTypeInfo),
        HiveObjectConverter.convert(mapValueObjectInspector, v, mapValueTypeInfo));
  }

  return newMap;
}
 

public static Object getMapObject(Map<String, AttributeValue> data, ObjectInspector objectInspector) {
  MapObjectInspector mapOI = (MapObjectInspector) objectInspector;
  ObjectInspector mapValueOI = mapOI.getMapValueObjectInspector();
  HiveDynamoDBType valueType = HiveDynamoDBTypeFactory.getTypeObjectFromHiveType(mapValueOI);

  Map<String, Object> values = new HashMap<>();
  for (Map.Entry<String, AttributeValue> entry : data.entrySet()) {
    values.put(entry.getKey(), valueType.getHiveData(entry.getValue(), mapValueOI));
  }

  return values;
}
 

private Object deparseMap(final Object obj, final MapObjectInspector mapOI) {
	final Map<Object, Object> map = new HashMap<Object, Object>();
	final ObjectInspector mapValOI = mapOI.getMapValueObjectInspector();
	final Map<?, ?> fields = mapOI.getMap(obj);
	for (final Map.Entry<?, ?> field : fields.entrySet()) {
		final Object fieldName = field.getKey();
		final Object fieldObj = field.getValue();
		map.put(fieldName, deparseObject(fieldObj, mapValOI));
	}
	return map;
}
 

private Writable createMap(final Object obj, final MapObjectInspector inspector)
    throws SerDeException {
  final Map<?, ?> sourceMap = inspector.getMap(obj);
  final ObjectInspector keyInspector = inspector.getMapKeyObjectInspector();
  final ObjectInspector valueInspector = inspector.getMapValueObjectInspector();
  final List<ArrayWritable> array = new ArrayList<ArrayWritable>();

  if (sourceMap != null) {
    for (final Entry<?, ?> keyValue : sourceMap.entrySet()) {
      final Writable key = createObject(keyValue.getKey(), keyInspector);
      final Writable value = createObject(keyValue.getValue(), valueInspector);
      if (key != null) {
        Writable[] arr = new Writable[2];
        arr[0] = key;
        arr[1] = value;
        array.add(new ArrayWritable(Writable.class, arr));
      }
    }
  }
  if (array.size() > 0) {
    final ArrayWritable subArray = new ArrayWritable(ArrayWritable.class,
        array.toArray(new ArrayWritable[array.size()]));
    return new ArrayWritable(Writable.class, new Writable[] {subArray});
  } else {
    return null;
  }
}
 

private Object deparseMap(Object obj, MapObjectInspector mapOI) {
	Map<Object, Object> map = new HashMap<Object, Object>();
	ObjectInspector mapValOI = mapOI.getMapValueObjectInspector();
	Map<?, ?> fields = mapOI.getMap(obj);
	for (Map.Entry<?, ?> field : fields.entrySet()) {
		Object fieldName = field.getKey();
		Object fieldObj = field.getValue();
		map.put(fieldName, deparseObject(fieldObj, mapValOI));
	}
	return map;
}
 

@Override
public ObjectInspector init(Mode mode, ObjectInspector[] argOIs) throws HiveException {
    super.init(mode, argOIs);

    // initialize input
    if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data
        this.inputKeyOI = HiveUtils.asPrimitiveObjectInspector(argOIs[0]);
        this.inputValueOI = argOIs[1];
        this.sizeOI = HiveUtils.asIntegerOI(argOIs[2]);
    } else {// from partial aggregation
        StructObjectInspector soi = (StructObjectInspector) argOIs[0];
        this.internalMergeOI = soi;

        this.partialMapField = soi.getStructFieldRef("partialMap");
        // re-extract input key/value OIs
        MapObjectInspector partialMapOI =
                (MapObjectInspector) partialMapField.getFieldObjectInspector();
        this.inputKeyOI = HiveUtils.asPrimitiveObjectInspector(
            partialMapOI.getMapKeyObjectInspector());
        this.inputValueOI = partialMapOI.getMapValueObjectInspector();

        this.partialMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(
            ObjectInspectorUtils.getStandardObjectInspector(inputKeyOI),
            ObjectInspectorUtils.getStandardObjectInspector(inputValueOI));

        this.sizeField = soi.getStructFieldRef("size");
        this.sizeOI = (PrimitiveObjectInspector) sizeField.getFieldObjectInspector();
    }

    // initialize output
    final ObjectInspector outputOI;
    if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {// terminatePartial
        outputOI = internalMergeOI(inputKeyOI, inputValueOI);
    } else {// terminate
        outputOI = ObjectInspectorFactory.getStandardMapObjectInspector(
            ObjectInspectorUtils.getStandardObjectInspector(inputKeyOI),
            ObjectInspectorUtils.getStandardObjectInspector(inputValueOI));
    }
    return outputOI;
}
 

private static void putIntoSet(@Nonnull final Map<?, ?> m,
        @Nonnull final Map<Object, Object> dst, @Nonnull final MapObjectInspector mapOI) {
    final ObjectInspector keyOI = mapOI.getMapKeyObjectInspector();
    final ObjectInspector valueOI = mapOI.getMapValueObjectInspector();

    for (Map.Entry<?, ?> e : m.entrySet()) {
        Object k = e.getKey();
        Object v = e.getValue();
        Object keyCopy = ObjectInspectorUtils.copyToStandardObject(k, keyOI);
        Object valCopy = ObjectInspectorUtils.copyToStandardObject(v, valueOI);
        dst.put(keyCopy, valCopy);
    }
}
 

private Object parseMap(JsonParser parser, MapObjectInspector oi)
        throws IOException, SerDeException {
    if (parser.getCurrentToken() == JsonToken.VALUE_NULL) {
        parser.nextToken();
        return null;
    }

    Map<Object, Object> ret = new LinkedHashMap<>();

    if (parser.getCurrentToken() != JsonToken.START_OBJECT) {
        throw new SerDeException("struct expected");
    }

    if (!(oi.getMapKeyObjectInspector() instanceof PrimitiveObjectInspector)) {
        throw new SerDeException("map key must be a primitive");
    }
    PrimitiveObjectInspector keyOI = (PrimitiveObjectInspector) oi.getMapKeyObjectInspector();
    ObjectInspector valOI = oi.getMapValueObjectInspector();

    JsonToken currentToken = parser.nextToken();
    while (currentToken != null && currentToken != JsonToken.END_OBJECT) {

        if (currentToken != JsonToken.FIELD_NAME) {
            throw new SerDeException("unexpected token: " + currentToken);
        }

        Object key = parseMapKey(parser, keyOI);
        Object val = parseDispatcher(parser, valOI);
        ret.put(key, val);

        currentToken = parser.getCurrentToken();
    }
    if (currentToken != null) {
        parser.nextToken();
    }
    return ret;
}
 

private static void serializeMap(@Nonnull final StringBuilder sb, @Nullable final Object obj,
        @Nonnull final MapObjectInspector moi) throws SerDeException {
    ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector();
    ObjectInspector mapValueObjectInspector = moi.getMapValueObjectInspector();
    Map<?, ?> omap = moi.getMap(obj);
    if (omap == null) {
        sb.append("null");
    } else {
        sb.append(SerDeUtils.LBRACE);
        boolean first = true;
        for (Object entry : omap.entrySet()) {
            if (first) {
                first = false;
            } else {
                sb.append(SerDeUtils.COMMA);
            }
            Map.Entry<?, ?> e = (Map.Entry<?, ?>) entry;
            StringBuilder keyBuilder = new StringBuilder();
            buildJSONString(keyBuilder, e.getKey(), mapKeyObjectInspector);
            String keyString = keyBuilder.toString().trim();
            if ((!keyString.isEmpty()) && (keyString.charAt(0) != SerDeUtils.QUOTE)) {
                appendWithQuotes(sb, keyString);
            } else {
                sb.append(keyString);
            }
            sb.append(SerDeUtils.COLON);
            buildJSONString(sb, e.getValue(), mapValueObjectInspector);
        }
        sb.append(SerDeUtils.RBRACE);
    }
}
 

private static Block serializeMap(Type type, BlockBuilder builder, Object object, MapObjectInspector inspector, boolean filterNullMapKeys)
{
    Map<?, ?> map = inspector.getMap(object);
    if (map == null) {
        requireNonNull(builder, "parent builder is null").appendNull();
        return null;
    }

    List<Type> typeParameters = type.getTypeParameters();
    checkArgument(typeParameters.size() == 2, "map must have exactly 2 type parameter");
    Type keyType = typeParameters.get(0);
    Type valueType = typeParameters.get(1);
    ObjectInspector keyInspector = inspector.getMapKeyObjectInspector();
    ObjectInspector valueInspector = inspector.getMapValueObjectInspector();
    BlockBuilder currentBuilder;

    boolean builderSynthesized = false;
    if (builder == null) {
        builderSynthesized = true;
        builder = type.createBlockBuilder(null, 1);
    }
    currentBuilder = builder.beginBlockEntry();

    for (Map.Entry<?, ?> entry : map.entrySet()) {
        // Hive skips map entries with null keys
        if (!filterNullMapKeys || entry.getKey() != null) {
            serializeObject(keyType, currentBuilder, entry.getKey(), keyInspector);
            serializeObject(valueType, currentBuilder, entry.getValue(), valueInspector);
        }
    }

    builder.closeEntry();
    if (builderSynthesized) {
        return (Block) type.getObject(builder, 0);
    }
    else {
        return null;
    }
}
 

@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    // Check if two arguments were passed
    if (arguments.length != ARG_COUNT) {
        throw new UDFArgumentLengthException(
                "The function map_element_at(map, key) takes exactly " + ARG_COUNT + " arguments.");
    }

    // Check if two argument is of category LIST
        if (!arguments[0].getCategory().equals(ObjectInspector.Category.MAP)) {
            throw new UDFArgumentTypeException(0,
                    "\"" + serdeConstants.MAP_TYPE_NAME + "\" "
                            + "expected at function map_element_at, but "
                            + "\"" + arguments[0].getTypeName() + "\" "
                            + "is found");
        }

    mapOI = (MapObjectInspector) arguments[0];
    keyOI = arguments[1];

    ObjectInspector mapKeyOI = mapOI.getMapKeyObjectInspector();
    ObjectInspector mapValueOI = mapOI.getMapValueObjectInspector();

    // Check if map value type are of same value type
    if (!ObjectInspectorUtils.compareTypes(mapKeyOI, keyOI)) {
        throw new UDFArgumentTypeException(1,
                "\"" + mapKeyOI.getTypeName() + "\""
                        + " expected at function map_element_at key, but "
                        + "\"" + keyOI.getTypeName() + "\""
                        + " is found");
    }

    return mapValueOI;
}
 

public HiveMap(Object object, ObjectInspector objectInspector, StdFactory stdFactory) {
  super(stdFactory);
  _object = object;
  _mapObjectInspector = (MapObjectInspector) objectInspector;
  _keyObjectInspector = _mapObjectInspector.getMapKeyObjectInspector();
  _valueObjectInspector = _mapObjectInspector.getMapValueObjectInspector();
}
 

/**
 * Converts a Hive column of type {@code Map<String,String>} into a DynamoDB item.
 *
 * It is expected that the Hive data is a map of type <String, String>. The key in Hive data
 * map is converted to a DynamoDB attribute name. The corresponding value in Hive data map is
 * converted into DynamoDB AttributeValue. This attribute value is expected to be a JSON
 * serialized AttributeValue.
 *
 * @param data                 Data from Hive
 * @param fieldObjectInspector The object inspector for the Hive data. Must have TypeName
 *                             Map<String,String>.
 *
 * @return DynamoDB item representation of provided data from Hive as a
 *         Map<String,AttributeValue>.
 *
 * @throws SerDeException
 */
public Map<String, AttributeValue> parseDynamoDBData(Object data, ObjectInspector
    fieldObjectInspector) throws SerDeException {

  if (!HiveDynamoDBTypeFactory.isHiveDynamoDBItemMapType(fieldObjectInspector)) {
    throw new SerDeException(getClass().toString() + " Expecting a MapObjectInspector of type "
        + "map<string,string> for a column which maps DynamoDB item. But we got: "
        + fieldObjectInspector.getTypeName());
  }

  Map<String, AttributeValue> item = new HashMap<>();

  /* map is of type <String, String> */
  MapObjectInspector mapOI = (MapObjectInspector) fieldObjectInspector;
  StringObjectInspector mapKeyObjectInspector = (StringObjectInspector) mapOI
      .getMapKeyObjectInspector();
  StringObjectInspector mapValueObjectInspector = (StringObjectInspector) mapOI
      .getMapValueObjectInspector();

  /*
   * Get the underlying map object. This is expected to be of type
   * <String,String>
   */
  Map<?, ?> map = mapOI.getMap(data);

  if (map == null || map.isEmpty()) {
    throw new SerDeException("Hive data cannot be null.");
  }

  /* Reconstruct the item */
  for (Entry<?, ?> entry : map.entrySet()) {

    /* Get the string key, value pair */
    String dynamoDBAttributeName = mapKeyObjectInspector.getPrimitiveJavaObject(entry.getKey());
    String dynamoDBAttributeValue = mapValueObjectInspector.getPrimitiveJavaObject(entry.getValue());

    /* Deserialize the AttributeValue string */
    AttributeValue deserializedAttributeValue = deserializeAttributeValue(dynamoDBAttributeValue);

    item.put(dynamoDBAttributeName, deserializedAttributeValue);
  }
  return item;
}
 

@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    // Check if two arguments were passed
    if (arguments.length != ARG_COUNT) {
        throw new UDFArgumentLengthException(
                "The function map_equals(map, map) takes exactly " + ARG_COUNT + " arguments.");
    }

    // Check if two argument is of category LIST
    for (int i = 0; i < 2; i++) {
        if (!arguments[i].getCategory().equals(ObjectInspector.Category.MAP)) {
            throw new UDFArgumentTypeException(i,
                    "\"" + serdeConstants.MAP_TYPE_NAME + "\" "
                            + "expected at function map_equals, but "
                            + "\"" + arguments[i].getTypeName() + "\" "
                            + "is found");
        }
    }

    leftMapOI = (MapObjectInspector) arguments[0];
    rightMapOI = (MapObjectInspector) arguments[1];

    ObjectInspector leftMapKeyOI = leftMapOI.getMapKeyObjectInspector();
    ObjectInspector leftMapValueOI = leftMapOI.getMapValueObjectInspector();
    ObjectInspector rightMapKeyOI = rightMapOI.getMapKeyObjectInspector();
    ObjectInspector rightMapValueOI = rightMapOI.getMapValueObjectInspector();

    // Check if two map are of same key and value type
    if (!ObjectInspectorUtils.compareTypes(leftMapKeyOI, rightMapKeyOI)) {
        throw new UDFArgumentTypeException(1,
                "\"" + leftMapKeyOI.getTypeName() + "\""
                        + " expected at function map_equals key, but "
                        + "\"" + rightMapKeyOI.getTypeName() + "\""
                        + " is found");
    }

    if (!ObjectInspectorUtils.compareTypes(leftMapValueOI, rightMapValueOI)) {
        throw new UDFArgumentTypeException(1,
                "\"" + leftMapValueOI.getTypeName() + "\""
                        + " expected at function map_equals value, but "
                        + "\"" + rightMapValueOI.getTypeName() + "\""
                        + " is found");
    }

    // Check if the comparison is supported for this type
    if (!ObjectInspectorUtils.compareSupported(leftMapValueOI)) {
        throw new UDFArgumentException("The function map_equals"
                + " does not support comparison for "
                + "\"" + leftMapValueOI.getTypeName() + "\""
                + " types");
    }

    result = new BooleanWritable(false);
    return PrimitiveObjectInspectorFactory.writableBooleanObjectInspector;
}
 

@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    // Check if two arguments were passed
    if (arguments.length != ARG_COUNT) {
        throw new UDFArgumentLengthException(
                "The function cosine_similarity(map, map) takes exactly " + ARG_COUNT + " arguments.");
    }

    // Check if two argument is of category LIST
    for (int i = 0; i < 2; i++) {
        if (!arguments[i].getCategory().equals(ObjectInspector.Category.MAP)) {
            throw new UDFArgumentTypeException(i,
                    "\"" + serdeConstants.MAP_TYPE_NAME + "\" "
                            + "expected at function cosine_similarity, but "
                            + "\"" + arguments[i].getTypeName() + "\" "
                            + "is found");
        }
    }

    leftMapOI = (MapObjectInspector) arguments[0];
    rightMapOI = (MapObjectInspector) arguments[1];

    ObjectInspector leftMapKeyOI = leftMapOI.getMapKeyObjectInspector();
    ObjectInspector leftMapValueOI = leftMapOI.getMapValueObjectInspector();
    ObjectInspector rightMapKeyOI = rightMapOI.getMapKeyObjectInspector();
    ObjectInspector rightMapValueOI = rightMapOI.getMapValueObjectInspector();

    // Check if two map are of same key and value type
    if (!ObjectInspectorUtils.compareTypes(leftMapKeyOI, rightMapKeyOI)) {
        throw new UDFArgumentTypeException(1,
                "\"" + leftMapKeyOI.getTypeName() + "\""
                        + " expected at function cosine_similarity key, but "
                        + "\"" + rightMapKeyOI.getTypeName() + "\""
                        + " is found");
    }

    if (!ObjectInspectorUtils.compareTypes(PrimitiveObjectInspectorFactory.javaStringObjectInspector, leftMapKeyOI)) {
        throw new UDFArgumentTypeException(1,
                "\"" + PrimitiveObjectInspectorFactory.javaStringObjectInspector.getTypeName() + "\""
                        + " expected at function cosine_similarity key, but "
                        + "\"" + leftMapKeyOI.getTypeName() + "\""
                        + " is found");
    }

    if (!ObjectInspectorUtils.compareTypes(leftMapValueOI, rightMapValueOI)) {
        throw new UDFArgumentTypeException(1,
                "\"" + leftMapValueOI.getTypeName() + "\""
                        + " expected at function cosine_similarity value, but "
                        + "\"" + rightMapValueOI.getTypeName() + "\""
                        + " is found");
    }

    if (!ObjectInspectorUtils.compareTypes(PrimitiveObjectInspectorFactory.javaDoubleObjectInspector, leftMapValueOI)) {
        throw new UDFArgumentTypeException(1,
                "\"" + PrimitiveObjectInspectorFactory.javaDoubleObjectInspector.getTypeName() + "\""
                        + " expected at function cosine_similarity value, but "
                        + "\"" + leftMapValueOI.getTypeName() + "\""
                        + " is found");
    }

    return ObjectInspectorFactory.getStandardMapObjectInspector(leftMapKeyOI, leftMapValueOI);
}
 

/**
 * It writes a map type and its key-pair values to the Parquet RecordConsumer.
 * This is called when the original type (MAP) is detected by writeValue().
 * This function assumes the following schema:
 * optional group mapCol (MAP) {
 * repeated group map (MAP_KEY_VALUE) {
 * required TYPE key;
 * optional TYPE value;
 * }
 * }
 *
 * @param value The object that contains the map key-values.
 * @param inspector The object inspector used to get the correct value type.
 * @param type Type that contains information about the group (MAP) schema.
 */
private void writeMap(Object value, MapObjectInspector inspector, GroupType type)
{
    // Get the internal map structure (MAP_KEY_VALUE)
    GroupType repeatedType = type.getType(0).asGroupType();

    recordConsumer.startGroup();
    Map<?, ?> mapValues = inspector.getMap(value);
    if (mapValues != null && mapValues.size() > 0) {
        recordConsumer.startField(repeatedType.getName(), 0);

        Type keyType = repeatedType.getType(0);
        String keyName = keyType.getName();
        ObjectInspector keyInspector = inspector.getMapKeyObjectInspector();

        Type valuetype = repeatedType.getType(1);
        String valueName = valuetype.getName();
        ObjectInspector valueInspector = inspector.getMapValueObjectInspector();

        for (Map.Entry<?, ?> keyValue : mapValues.entrySet()) {
            recordConsumer.startGroup();
            if (keyValue != null) {
                // write key element
                Object keyElement = keyValue.getKey();
                recordConsumer.startField(keyName, 0);
                writeValue(keyElement, keyInspector, keyType);
                recordConsumer.endField(keyName, 0);

                // write value element
                Object valueElement = keyValue.getValue();
                if (valueElement != null) {
                    recordConsumer.startField(valueName, 1);
                    writeValue(valueElement, valueInspector, valuetype);
                    recordConsumer.endField(valueName, 1);
                }
            }
            recordConsumer.endGroup();
        }

        recordConsumer.endField(repeatedType.getName(), 0);
    }
    recordConsumer.endGroup();
}