Java Code Examples for org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector#getList()

@Nullable
public static String[] asStringArray(@Nonnull final DeferredObject arg,
        @Nonnull final ListObjectInspector listOI) throws HiveException {
    Object argObj = arg.get();
    if (argObj == null) {
        return null;
    }
    List<?> data = listOI.getList(argObj);
    final int size = data.size();
    final String[] arr = new String[size];
    for (int i = 0; i < size; i++) {
        Object o = data.get(i);
        if (o != null) {
            arr[i] = o.toString();
        }
    }
    return arr;
}
 

public static Object deparseObject(Object field, ObjectInspector fieldOI) throws SerDeException {
    switch(fieldOI.getCategory()) {
        case PRIMITIVE: {
            PrimitiveObjectInspector oi = (PrimitiveObjectInspector) fieldOI;
            return oi.getPrimitiveJavaObject(field);
        }
        case LIST: {
            ListObjectInspector listOI = (ListObjectInspector) fieldOI;
            List<?> elements = listOI.getList(field);
            List<Object> list = new ArrayList<Object>(elements.size());
            ObjectInspector elemOI = listOI.getListElementObjectInspector();
            for(Object elem : elements) {
                Object o = deparseObject(elem, elemOI);
                list.add(o);
            }
            return list;
        }
        default:
            throw new SerDeException("Unexpected fieldOI: " + fieldOI);
    }
}
 

private ArrayWritable createArray(final Object obj, final ListObjectInspector inspector)
    throws SerDeException {
  final List<?> sourceArray = inspector.getList(obj);
  final ObjectInspector subInspector = inspector.getListElementObjectInspector();
  final List<Writable> array = new ArrayList<Writable>();
  if (sourceArray != null) {
    for (final Object curObj : sourceArray) {
      final Writable newObj = createObject(curObj, subInspector);
      if (newObj != null) {
        array.add(newObj);
      }
    }
  }
  if (array.size() > 0) {
    final ArrayWritable subArray = new ArrayWritable(array.get(0).getClass(),
        array.toArray(new Writable[array.size()]));
    return new ArrayWritable(Writable.class, new Writable[] {subArray});
  } else {
    return null;
  }
}
 

@Test
public void testPA1TrainWithParameter() throws UDFArgumentException {
    PassiveAggressiveUDTF udtf = new PassiveAggressiveUDTF.PA1();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ListObjectInspector intListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(intOI);

    ObjectInspector param = ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-c 0.1");
    /* define aggressive parameter */
    udtf.initialize(new ObjectInspector[] {intListOI, intOI, param});

    /* train weights */
    List<?> features = (List<?>) intListOI.getList(new Object[] {1, 2, 3});
    udtf.train(features, 1);

    /* check weights */
    assertEquals(0.1000000f, udtf.model.get(1).get(), 1e-5f);
    assertEquals(0.1000000f, udtf.model.get(2).get(), 1e-5f);
    assertEquals(0.1000000f, udtf.model.get(3).get(), 1e-5f);
}
 

@Test
public void testPA1TrainWithoutParameter() throws UDFArgumentException {
    PassiveAggressiveUDTF udtf = new PassiveAggressiveUDTF.PA1();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ListObjectInspector intListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(intOI);

    /* define aggressive parameter */
    udtf.initialize(new ObjectInspector[] {intListOI, intOI});

    /* train weights */
    List<?> features = (List<?>) intListOI.getList(new Object[] {1, 2, 3});
    udtf.train(features, 1);

    /* check weights */
    assertEquals(0.3333333f, udtf.model.get(1).get(), 1e-5f);
    assertEquals(0.3333333f, udtf.model.get(2).get(), 1e-5f);
    assertEquals(0.3333333f, udtf.model.get(3).get(), 1e-5f);
}
 

@Nullable
public static List<String> asStringList(@Nonnull final DeferredObject arg,
        @Nonnull final ListObjectInspector listOI) throws HiveException {
    Object argObj = arg.get();
    if (argObj == null) {
        return null;
    }
    List<?> data = listOI.getList(argObj);
    int size = data.size();
    if (size == 0) {
        return Collections.emptyList();
    }
    final String[] ary = new String[size];
    for (int i = 0; i < size; i++) {
        Object o = data.get(i);
        if (o != null) {
            ary[i] = o.toString();
        }
    }
    return Arrays.asList(ary);
}
 

@Nonnull
private static void serializeList(@Nonnull final StringBuilder sb, @Nullable final Object obj,
        @Nullable final ListObjectInspector loi) throws SerDeException {
    ObjectInspector listElementObjectInspector = loi.getListElementObjectInspector();
    List<?> olist = loi.getList(obj);

    if (olist == null) {
        sb.append("null");
    } else {
        sb.append(SerDeUtils.LBRACKET);
        for (int i = 0; i < olist.size(); i++) {
            if (i > 0) {
                sb.append(SerDeUtils.COMMA);
            }
            buildJSONString(sb, olist.get(i), listElementObjectInspector);
        }
        sb.append(SerDeUtils.RBRACKET);
    }
}
 

/**
 * This method currently supports BinarySet data type of DynamoDB
 */
public static List<ByteBuffer> getByteBuffers(Object data, ObjectInspector objectInspector, String
        ddType) {
  ListObjectInspector listObjectInspector = (ListObjectInspector) objectInspector;
  List<?> dataList = listObjectInspector.getList(data);

  if (dataList == null) {
    return null;
  }

  ObjectInspector itemObjectInspector = listObjectInspector.getListElementObjectInspector();
  List<ByteBuffer> itemList = new ArrayList<>();
  for (Object dataItem : dataList) {
    if (dataItem == null) {
      throw new NullPointerException("Null element found in list: " + dataList);
    }

    if (ddType.equals(DynamoDBTypeConstants.BINARY_SET)) {
      itemList.add(getByteBuffer(dataItem, itemObjectInspector));
    } else {
      throw new IllegalArgumentException("Expecting BinarySet type: " + ddType);
    }
  }

  return itemList;
}
 

public static List<AttributeValue> getListAttribute(Object data, ObjectInspector objectInspector,
                                                    boolean nullSerialization) {
  ListObjectInspector listObjectInspector = (ListObjectInspector) objectInspector;
  List<?> dataList = listObjectInspector.getList(data);

  if (dataList == null) {
    return null;
  }

  ObjectInspector itemObjectInspector = listObjectInspector.getListElementObjectInspector();
  HiveDynamoDBType itemType = HiveDynamoDBTypeFactory.getTypeObjectFromHiveType(itemObjectInspector);
  List<AttributeValue> itemList = new ArrayList<>();
  for (Object dataItem : dataList) {
    AttributeValue item = dataItem == null ?
        getNullAttribute(nullSerialization) :
        itemType.getDynamoDBData(dataItem, itemObjectInspector, nullSerialization);

    if (item == null) {
      throw new NullPointerException("Null element found in list: " + dataList);
    }

    itemList.add(item);
  }

  return itemList;
}
 

private void writeSingleLevelArray(Object value, ListObjectInspector inspector, GroupType type)
{
    // Get the internal array structure
    Type elementType = type.getType(0);

    recordConsumer.startGroup();

    List<?> arrayValues = inspector.getList(value);
    if (!arrayValues.isEmpty()) {
        recordConsumer.startField(elementType.getName(), 0);
        ObjectInspector elementInspector = inspector.getListElementObjectInspector();

        for (Object element : arrayValues) {
            if (element == null) {
                throw new IllegalArgumentException("Array elements are requires in given schema definition");
            }
            writeValue(element, elementInspector, elementType);
        }

        recordConsumer.endField(elementType.getName(), 0);
    }
    recordConsumer.endGroup();
}
 

/**
 * This method currently supports StringSet and NumberSet data type of DynamoDB
 */
public static List<String> getSetAttribute(Object data, ObjectInspector objectInspector, String
        ddType) {
  ListObjectInspector listObjectInspector = (ListObjectInspector) objectInspector;
  List<?> dataList = listObjectInspector.getList(data);

  if (dataList == null) {
    return null;
  }

  ObjectInspector itemObjectInspector = listObjectInspector.getListElementObjectInspector();
  List<String> itemList = new ArrayList<>();
  for (Object dataItem : dataList) {
    if (dataItem == null) {
      throw new NullPointerException("Null element found in list: " + dataList);
    }

    if (ddType.equals(DynamoDBTypeConstants.STRING_SET)) {
      itemList.add(getString(dataItem, itemObjectInspector));
    } else if (ddType.equals(DynamoDBTypeConstants.NUMBER_SET)) {
      itemList.add(getNumber(dataItem, itemObjectInspector));
    } else {
      throw new IllegalArgumentException("Expecting NumberSet or StringSet type: " + ddType);
    }
  }

  return itemList;
}
 

private static Block serializeList(Type type, BlockBuilder builder, Object object, ListObjectInspector inspector)
{
    List<?> list = inspector.getList(object);
    if (list == null) {
        requireNonNull(builder, "parent builder is null").appendNull();
        return null;
    }

    List<Type> typeParameters = type.getTypeParameters();
    checkArgument(typeParameters.size() == 1, "list must have exactly 1 type parameter");
    Type elementType = typeParameters.get(0);
    ObjectInspector elementInspector = inspector.getListElementObjectInspector();
    BlockBuilder currentBuilder;
    if (builder != null) {
        currentBuilder = builder.beginBlockEntry();
    }
    else {
        currentBuilder = elementType.createBlockBuilder(null, list.size());
    }

    for (Object element : list) {
        serializeObject(elementType, currentBuilder, element, elementInspector);
    }

    if (builder != null) {
        builder.closeEntry();
        return null;
    }
    else {
        Block resultBlock = currentBuilder.build();
        return resultBlock;
    }
}
 

@Test
public void testTrain() throws HiveException {
    PassiveAggressiveUDTF udtf = new PassiveAggressiveUDTF();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ListObjectInspector intListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(intOI);
    udtf.initialize(new ObjectInspector[] {intListOI, intOI});

    /* train weights by List<Object> */
    List<Integer> features1 = new ArrayList<Integer>();
    features1.add(1);
    features1.add(2);
    features1.add(3);
    udtf.train(features1, 1);

    /* check weights */
    assertEquals(0.3333333f, udtf.model.get(1).get(), 1e-5f);
    assertEquals(0.3333333f, udtf.model.get(2).get(), 1e-5f);
    assertEquals(0.3333333f, udtf.model.get(3).get(), 1e-5f);

    /* train weights by Object[] */
    List<?> features2 = (List<?>) intListOI.getList(new Object[] {3, 4, 5});
    udtf.train(features2, 1);

    /* check weights */
    assertEquals(0.3333333f, udtf.model.get(1).get(), 1e-5f);
    assertEquals(0.3333333f, udtf.model.get(2).get(), 1e-5f);
    assertEquals(0.5555555f, udtf.model.get(3).get(), 1e-5f);
    assertEquals(0.2222222f, udtf.model.get(4).get(), 1e-5f);
    assertEquals(0.2222222f, udtf.model.get(5).get(), 1e-5f);
}
 

@Override
public Object convert(ObjectInspector objectInspector, Object o, TypeInfo odpsTypeInfo) {
  ListObjectInspector listObjectInspector = (ListObjectInspector) objectInspector;
  ObjectInspector elementInspector = listObjectInspector.getListElementObjectInspector();
  TypeInfo elementTypeInfo = ((ArrayTypeInfo) odpsTypeInfo).getElementTypeInfo();
  List list = listObjectInspector.getList(o);
  List<Object> newList = new ArrayList<>();
  for (Object element : list) {
    newList.add(HiveObjectConverter.convert(elementInspector, element, elementTypeInfo));
  }

  return newList;
}
 

private List<OneField> traverseList(Object obj, ListObjectInspector loi)
        throws BadRecordException, IOException {
    List<OneField> listRecord = new LinkedList<>();
    List<?> list = loi.getList(obj);
    ObjectInspector eoi = loi.getListElementObjectInspector();
    if (list == null) {
        throw new BadRecordException(
                "Illegal value NULL for Hive data type List");
    }
    for (Object object : list) {
        traverseTuple(object, eoi, listRecord, true);
    }
    return listRecord;
}
 

/**
 * Deparses a list and its elements.
 *
 * @param obj - Hive object to deparse
 * @param listOI  - ObjectInspector for the object
 * @return - A deparsed object
 */
private Object deparseList(Object obj, ListObjectInspector listOI) {
	List<Object> list = new ArrayList<Object>();
	List<?> field = listOI.getList(obj);
	ObjectInspector elemOI = listOI.getListElementObjectInspector();
	for (Object elem : field) {
		list.add(deparseObject(elem, elemOI));
	}
	return list;
}
 

private void add(BlurRecord blurRecord, String columnName, ObjectInspector objectInspector, Object data)
    throws SerDeException {
  if (data == null) {
    return;
  }
  if (objectInspector instanceof PrimitiveObjectInspector) {
    PrimitiveObjectInspector primitiveObjectInspector = (PrimitiveObjectInspector) objectInspector;
    String strValue = toString(columnName, data, primitiveObjectInspector);
    if (columnName.equals(BlurObjectInspectorGenerator.ROWID)) {
      blurRecord.setRowId(strValue);
    } else if (columnName.equals(BlurObjectInspectorGenerator.RECORDID)) {
      blurRecord.setRecordId(strValue);
    } else {
      blurRecord.addColumn(columnName, strValue);
    }
  } else if (objectInspector instanceof StructObjectInspector) {
    StructObjectInspector structObjectInspector = (StructObjectInspector) objectInspector;
    Map<String, StructField> allStructFieldRefs = toMap(structObjectInspector.getAllStructFieldRefs());
    String latitude = getFieldData(columnName, data, structObjectInspector, allStructFieldRefs,
        BlurObjectInspectorGenerator.LATITUDE);
    String longitude = getFieldData(columnName, data, structObjectInspector, allStructFieldRefs,
        BlurObjectInspectorGenerator.LONGITUDE);
    blurRecord.addColumn(columnName, toLatLong(latitude, longitude));
  } else if (objectInspector instanceof ListObjectInspector) {
    ListObjectInspector listObjectInspector = (ListObjectInspector) objectInspector;
    List<?> list = listObjectInspector.getList(data);
    ObjectInspector listElementObjectInspector = listObjectInspector.getListElementObjectInspector();
    for (Object obj : list) {
      add(blurRecord, columnName, listElementObjectInspector, obj);
    }
  } else {
    throw new SerDeException("ObjectInspector [" + objectInspector + "] of type ["
        + (objectInspector != null ? objectInspector.getClass() : null) + "] not supported.");
  }
}
 

/**
 * Deparses a list and its elements.
 *
 * @param obj
 *            - Hive object to deparse
 * @param oi
 *            - ObjectInspector for the object
 * @return - A deparsed object
 */
private Object deparseList(final Object obj,
		final ListObjectInspector listOI) {
	final List<Object> list = new ArrayList<Object>();
	final List<?> field = listOI.getList(obj);
	final ObjectInspector elemOI = listOI.getListElementObjectInspector();
	for (final Object elem : field) {
		list.add(deparseObject(elem, elemOI));
	}
	return list;
}
 

/**
 * It writes a list type and its array elements to the Parquet RecordConsumer.
 * This is called when the original type (LIST) is detected by writeValue()/
 * This function assumes the following schema:
 * optional group arrayCol (LIST) {
 * repeated group array {
 * optional TYPE array_element;
 * }
 * }
 *
 * @param value The object that contains the array values.
 * @param inspector The object inspector used to get the correct value type.
 * @param type Type that contains information about the group (LIST) schema.
 */
private void writeArray(Object value, ListObjectInspector inspector, GroupType type)
{
    // Get the internal array structure
    GroupType repeatedType = type.getType(0).asGroupType();
    recordConsumer.startGroup();

    List<?> arrayValues = inspector.getList(value);
    if (!arrayValues.isEmpty()) {
        recordConsumer.startField(repeatedType.getName(), 0);
        ObjectInspector elementInspector = inspector.getListElementObjectInspector();

        Type elementType = repeatedType.getType(0);
        String elementName = elementType.getName();

        for (Object element : arrayValues) {
            recordConsumer.startGroup();
            if (element != null) {
                recordConsumer.startField(elementName, 0);
                writeValue(element, elementInspector, elementType);
                recordConsumer.endField(elementName, 0);
            }
            recordConsumer.endGroup();
        }

        recordConsumer.endField(repeatedType.getName(), 0);
    }
    recordConsumer.endGroup();
}
 

private static void removeAll(@Nonnull final List<?> values, @Nonnull final Object target,
        @Nonnull final Converter converter, @Nonnull final ListObjectInspector valueListOI) {
    Object converted = converter.convert(target);
    List<?> convertedList = valueListOI.getList(converted);
    values.removeAll(convertedList);
}