Java Code Examples for org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector#getAllStructFieldRefs()

StructTreeWriter(int columnId,
                 ObjectInspector inspector,
                 StreamFactory writer,
                 boolean nullable, Configuration conf,
                 boolean useVInts, boolean lowMemoryMode,
                 MemoryEstimate memoryEstimate) throws IOException {
  super(columnId, inspector, writer, nullable, conf, useVInts, memoryEstimate);
  StructObjectInspector structObjectInspector =
    (StructObjectInspector) inspector;
  fields = structObjectInspector.getAllStructFieldRefs();
  childrenWriters = new TreeWriter[fields.size()];
  for(int i=0; i < childrenWriters.length; ++i) {
    childrenWriters[i] = createTreeWriter(
      fields.get(i).getFieldObjectInspector(), writer, true, conf, useVInts,
      lowMemoryMode, memoryEstimate);
  }
  recordPosition(rowIndexPosition);
}
 

private static void checkIntermediateResultInspector(ObjectInspector resultInspector) {
  Assert.assertNotNull(resultInspector);
  Assert.assertEquals(resultInspector.getCategory(), ObjectInspector.Category.STRUCT);
  StructObjectInspector structResultInspector = (StructObjectInspector) resultInspector;
  List<?> fields = structResultInspector.getAllStructFieldRefs();
  Assert.assertEquals(fields.size(), 2);

  ObjectInspector inspector1 = ((StructField) fields.get(0)).getFieldObjectInspector();
  Assert.assertEquals(inspector1.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector1 = (PrimitiveObjectInspector) inspector1;
  Assert.assertEquals(primitiveInspector1.getPrimitiveCategory(), PrimitiveCategory.LONG);

  ObjectInspector inspector2 = ((StructField) fields.get(1)).getFieldObjectInspector();
  Assert.assertEquals(inspector2.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector2 = (PrimitiveObjectInspector) inspector2;
  Assert.assertEquals(primitiveInspector2.getPrimitiveCategory(), PrimitiveCategory.BINARY);
}
 

static void checkIntermediateResultInspector(ObjectInspector resultInspector) {
    Assert.assertNotNull(resultInspector);
    Assert.assertEquals(resultInspector.getCategory(), ObjectInspector.Category.STRUCT);
    StructObjectInspector structResultInspector = (StructObjectInspector) resultInspector;
    List<?> fields = structResultInspector.getAllStructFieldRefs();
    Assert.assertEquals(fields.size(), 3);

    ObjectInspector inspector1 = ((StructField) fields.get(0)).getFieldObjectInspector();
    Assert.assertEquals(inspector1.getCategory(), ObjectInspector.Category.PRIMITIVE);
    PrimitiveObjectInspector primitiveInspector1 = (PrimitiveObjectInspector) inspector1;
    Assert.assertEquals(primitiveInspector1.getPrimitiveCategory(), PrimitiveCategory.INT);

    ObjectInspector inspector2 = ((StructField) fields.get(1)).getFieldObjectInspector();
    Assert.assertEquals(inspector2.getCategory(), ObjectInspector.Category.PRIMITIVE);
    PrimitiveObjectInspector primitiveInspector2 = (PrimitiveObjectInspector) inspector2;
    Assert.assertEquals(primitiveInspector2.getPrimitiveCategory(), PrimitiveCategory.LONG);

    ObjectInspector inspector3 = ((StructField) fields.get(2)).getFieldObjectInspector();
    Assert.assertEquals(inspector3.getCategory(), ObjectInspector.Category.PRIMITIVE);
    PrimitiveObjectInspector primitiveInspector3 = (PrimitiveObjectInspector) inspector3;
    Assert.assertEquals(primitiveInspector3.getPrimitiveCategory(), PrimitiveCategory.BINARY);
}
 

/**
 * Deparses struct data into a serializable JSON object.
 *
 * @param obj      - Hive struct data
 * @param structOI - ObjectInspector for the struct
 * @param isRow    - Whether or not this struct represents a top-level row
 * @return - A deparsed struct
 */
private Object deparseStruct(Object obj,
							 StructObjectInspector structOI,
							 boolean isRow) {
	Map<Object, Object> struct = new HashMap<Object, Object>();
	List<? extends StructField> fields = structOI.getAllStructFieldRefs();
	for (int i = 0; i < fields.size(); i++) {
		StructField field = fields.get(i);
		// The top-level row object is treated slightly differently from other
		// structs, because the field names for the row do not correctly reflect
		// the Hive column names. For lower-level structs, we can get the field
		// name from the associated StructField object.
		String fieldName = isRow ? colNames.get(i) : field.getFieldName();
		ObjectInspector fieldOI = field.getFieldObjectInspector();
		Object fieldObj = structOI.getStructFieldData(obj, field);
		struct.put(fieldName, deparseObject(fieldObj, fieldOI));
	}
	return struct;
}
 

/**
 * This method takes an object representing a row of data from Hive, and uses
 * the ObjectInspector to get the data for each column and serialize.
 */
@Override
public DbRecordWritable serialize(Object row, ObjectInspector inspector) throws SerDeException {
    final StructObjectInspector structInspector = (StructObjectInspector) inspector;
    final List<? extends StructField> fields = structInspector.getAllStructFieldRefs();
    if(fields.size() != fieldCount) {
        throw new SerDeException(String.format("Required %d columns, received %d.", fieldCount, fields.size()));
    }

    cachedWritable.clear();

    for(int i = 0; i < fieldCount; i++) {
        StructField structField = fields.get(i);
        if(structField != null) {
            Object field = structInspector.getStructFieldData(row, structField);
            ObjectInspector fieldOI = structField.getFieldObjectInspector();
            Object javaObject = HiveJdbcBridgeUtils.deparseObject(field, fieldOI);
            cachedWritable.set(i, javaObject);
        }
    }

    return cachedWritable;
}
 

static void checkIntermediateResultInspector(ObjectInspector resultInspector) {
  Assert.assertNotNull(resultInspector);
  Assert.assertEquals(resultInspector.getCategory(), ObjectInspector.Category.STRUCT);
  StructObjectInspector structResultInspector = (StructObjectInspector) resultInspector;
  List<?> fields = structResultInspector.getAllStructFieldRefs();
  Assert.assertEquals(fields.size(), 2);

  ObjectInspector inspector1 = ((StructField) fields.get(0)).getFieldObjectInspector();
  Assert.assertEquals(inspector1.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector1 = (PrimitiveObjectInspector) inspector1;
  Assert.assertEquals(primitiveInspector1.getPrimitiveCategory(), PrimitiveCategory.INT);

  ObjectInspector inspector2 = ((StructField) fields.get(1)).getFieldObjectInspector();
  Assert.assertEquals(inspector2.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector2 = (PrimitiveObjectInspector) inspector2;
  Assert.assertEquals(primitiveInspector2.getPrimitiveCategory(), PrimitiveCategory.BINARY);
}
 

/**
 * Deparses struct data into a serializable JSON object.
 *
 * @param obj
 *            - Hive struct data
 * @param structOI
 *            - ObjectInspector for the struct
 * @param isRow
 *            - Whether or not this struct represents a top-level row
 * @return - A deparsed struct
 */
private Object deparseStruct(final Object obj,
		final StructObjectInspector structOI, final boolean isRow) {
	final Map<Object, Object> struct = new HashMap<Object, Object>();
	final List<? extends StructField> fields = structOI
			.getAllStructFieldRefs();
	for (int i = 0; i < fields.size(); i++) {
		final StructField field = fields.get(i);
		// The top-level row object is treated slightly differently from
		// other
		// structs, because the field names for the row do not correctly
		// reflect
		// the Hive column names. For lower-level structs, we can get the
		// field
		// name from the associated StructField object.
		final String fieldName = isRow ? colNames.get(i) : field
				.getFieldName();
		final ObjectInspector fieldOI = field.getFieldObjectInspector();
		final Object fieldObj = structOI.getStructFieldData(obj, field);
		struct.put(fieldName, deparseObject(fieldObj, fieldOI));
	}
	return struct;
}
 

/**
 * Helper method that creates {@link VectorizedRowBatch}. For each selected column an input vector is created in the
 * batch. For unselected columns the vector entry is going to be null. The order of input vectors in batch should
 * match the order the columns in ORC file.
 *
 * @param rowOI Used to find the ordinal of the selected column.
 * @return
 */
private VectorizedRowBatch createVectorizedRowBatch(StructObjectInspector rowOI, boolean isOriginal) {
  final List<? extends StructField> fieldRefs = rowOI.getAllStructFieldRefs();
  final List<ColumnVector> vectors = getVectors(rowOI);

  final VectorizedRowBatch result = new VectorizedRowBatch(fieldRefs.size());

  ColumnVector[] vectorArray =  vectors.toArray(new ColumnVector[0]);

  if (!isOriginal) {
    vectorArray = createTransactionalVectors(vectorArray);
  }

  result.cols = vectorArray;
  result.numCols = fieldRefs.size();
  result.reset();
  return result;
}
 

private ArrayWritable createStruct(final Object obj, final StructObjectInspector inspector)
    throws SerDeException {
  final List<? extends StructField> fields = inspector.getAllStructFieldRefs();
  final Writable[] arr = new Writable[fields.size()];
  for (int i = 0; i < fields.size(); i++) {
    final StructField field = fields.get(i);
    final Object subObj = inspector.getStructFieldData(obj, field);
    final ObjectInspector subInspector = field.getFieldObjectInspector();
    arr[i] = createObject(subObj, subInspector);
  }
  return new ArrayWritable(Writable.class, arr);
}
 

private static int getStructFieldIndex(@Nonnull StructObjectInspector oi,
        @Nonnull StructField field) {
    final List<? extends StructField> fields = oi.getAllStructFieldRefs();
    for (int i = 0, size = fields.size(); i < size; i++) {
        StructField f = fields.get(i);
        if (field.equals(f)) {
            return i;
        }
    }
    return -1;
}
 

public static Object getStructObject(Map<String, AttributeValue> data, ObjectInspector objectInspector) {
  StructObjectInspector structOI = (StructObjectInspector) objectInspector;
  List<? extends StructField> structFields = structOI.getAllStructFieldRefs();

  List<Object> values = new ArrayList<>();
  for (StructField field : structFields) {
    values.add(getStructFieldObject(data, field));
  }

  return values;
}
 

private static Block serializeStruct(Type type, BlockBuilder builder, Object object, StructObjectInspector inspector)
{
    if (object == null) {
        requireNonNull(builder, "parent builder is null").appendNull();
        return null;
    }

    List<Type> typeParameters = type.getTypeParameters();
    List<? extends StructField> allStructFieldRefs = inspector.getAllStructFieldRefs();
    checkArgument(typeParameters.size() == allStructFieldRefs.size());
    BlockBuilder currentBuilder;

    boolean builderSynthesized = false;
    if (builder == null) {
        builderSynthesized = true;
        builder = type.createBlockBuilder(null, 1);
    }
    currentBuilder = builder.beginBlockEntry();

    for (int i = 0; i < typeParameters.size(); i++) {
        StructField field = allStructFieldRefs.get(i);
        serializeObject(typeParameters.get(i), currentBuilder, inspector.getStructFieldData(object, field), field.getFieldObjectInspector());
    }

    builder.closeEntry();
    if (builderSynthesized) {
        return (Block) type.getObject(builder, 0);
    }
    else {
        return null;
    }
}
 

private ColumnVector getStructColumnVector(StructObjectInspector soi) {
  ArrayList<ColumnVector> vectors = new ArrayList<>();
  List<? extends StructField> members = soi.getAllStructFieldRefs();
  for (StructField structField: members) {
    vectors.add(getColumnVector(structField.getFieldObjectInspector()));
  }
  ColumnVector[] columnVectors = vectors.toArray(new ColumnVector[0]);
  return new StructColumnVector(VectorizedRowBatch.DEFAULT_SIZE, columnVectors);
}
 

StructTreeWriter(int columnId,
                 ObjectInspector inspector,
                 StreamFactory writer,
                 boolean nullable) throws IOException {
    super(columnId, inspector, writer, nullable);
    StructObjectInspector structObjectInspector =
            (StructObjectInspector) inspector;
    fields = structObjectInspector.getAllStructFieldRefs();
    childrenWriters = new TreeWriter[fields.size()];
    for (int i = 0; i < childrenWriters.length; ++i) {
        childrenWriters[i] = createTreeWriter(
                fields.get(i).getFieldObjectInspector(), writer, true);
    }
    recordPosition(rowIndexPosition);
}
 

private String getColumnNamesFromInspector(ObjectInspector inspector) {
    List<String> fieldNames = Lists.newArrayList();
    Joiner joiner = Joiner.on(",");
    if (inspector instanceof StructObjectInspector) {
        StructObjectInspector soi = (StructObjectInspector) inspector;
        List<? extends StructField> fields = soi.getAllStructFieldRefs();
        fieldNames.addAll(fields.stream().map((Function<StructField, String>) StructField::getFieldName).collect(Collectors.toList()));
    }
    return joiner.join(fieldNames);
}
 

/**
 * Returns the canonical type.
 *
 * @param fieldInspector inspector
 * @return type
 */
Type getCanonicalType(final ObjectInspector fieldInspector) {
    switch (fieldInspector.getCategory()) {
        case PRIMITIVE:
            return getPrimitiveType(fieldInspector);
        case MAP:
            final MapObjectInspector mapObjectInspector =
                TypeUtils.checkType(fieldInspector, MapObjectInspector.class,
                    "fieldInspector");
            final Type keyType = getCanonicalType(mapObjectInspector.getMapKeyObjectInspector());
            final Type valueType = getCanonicalType(mapObjectInspector.getMapValueObjectInspector());
            if (keyType == null || valueType == null) {
                return null;
            }
            return TypeRegistry.getTypeRegistry().getParameterizedType(TypeEnum.MAP,
                ImmutableList.of(keyType.getTypeSignature(), valueType.getTypeSignature()), ImmutableList.of());
        case LIST:
            final ListObjectInspector listObjectInspector =
                TypeUtils.checkType(fieldInspector, ListObjectInspector.class,
                    "fieldInspector");
            final Type elementType =
                getCanonicalType(listObjectInspector.getListElementObjectInspector());
            if (elementType == null) {
                return null;
            }
            return TypeRegistry.getTypeRegistry().getParameterizedType(TypeEnum.ARRAY,
                ImmutableList.of(elementType.getTypeSignature()), ImmutableList.of());
        case STRUCT:
            final StructObjectInspector structObjectInspector =
                TypeUtils.checkType(fieldInspector, StructObjectInspector.class, "fieldInspector");
            final List<TypeSignature> fieldTypes = new ArrayList<>();
            final List<Object> fieldNames = new ArrayList<>();
            for (StructField field : structObjectInspector.getAllStructFieldRefs()) {
                fieldNames.add(field.getFieldName());
                final Type fieldType = getCanonicalType(field.getFieldObjectInspector());
                if (fieldType == null) {
                    return null;
                }
                fieldTypes.add(fieldType.getTypeSignature());
            }
            return TypeRegistry.getTypeRegistry()
                .getParameterizedType(TypeEnum.ROW, fieldTypes, fieldNames);
        default:
            log.info("Currently unsupported type {}, returning Unknown type", fieldInspector.getTypeName());
            return BaseType.UNKNOWN;
    }
}
 

private static boolean searchAllFields(final ObjectInspector rootOI,
                                       final String name,
                                       final int[] childCounts,
                                       SearchResult position
                                       ) {
  Category category = rootOI.getCategory();
  if (category == Category.STRUCT) {
    position.index++; // first child is immediately next to parent
    StructObjectInspector sOi = (StructObjectInspector) rootOI;
    for (StructField sf : sOi.getAllStructFieldRefs()) {
      // We depend on the fact that caller takes care of calling current method
      // once for each segment in the selected column path. So, we should always get
      // searched field as immediate child
      if (position.index >= childCounts.length) {
        // input schema has more columns than what reader can read
        return false;
      }
      if (sf.getFieldName().equalsIgnoreCase(name)) {
        position.oI = sf.getFieldObjectInspector();
        return true;
      } else {
        position.index += childCounts[position.index];
      }
    }
  } else if (category == Category.MAP) {
    position.index++; // first child is immediately next to parent
    if (position.index >= childCounts.length) {
      // input schema has more columns than what reader can read
      return false;
    }
    if (name.equalsIgnoreCase(HiveUtilities.MAP_KEY_FIELD_NAME)) {
      ObjectInspector kOi = ((MapObjectInspector) rootOI).getMapKeyObjectInspector();
      position.oI = kOi;
      return true;
    }
    position.index += childCounts[position.index];
    if (position.index >= childCounts.length) {
      // input schema has more columns than what reader can read
      return false;
    }
    if (name.equalsIgnoreCase(HiveUtilities.MAP_VALUE_FIELD_NAME)) {
      ObjectInspector vOi = ((MapObjectInspector) rootOI).getMapValueObjectInspector();
      position.oI = vOi;
      return true;
    }
  }
  return false;
}
 

@Override
public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException {
  // Prepare the field ObjectInspectors
  StructObjectInspector soi = (StructObjectInspector) objInspector;
  List<? extends StructField> fields = soi.getAllStructFieldRefs();
  List<Object> rowData = soi.getStructFieldsDataAsList(obj);
  Map<String, AttributeValue> item = Maps.newHashMap();

  validateData(fields, rowData);

  for (int i = 0; i < fields.size(); i++) {
    StructField field = fields.get(i);
    Object data = rowData.get(i);
    String columnName = columnNames.get(i);
    ObjectInspector fieldOI = field.getFieldObjectInspector();

    // Get the Hive to DynamoDB mapper
    HiveDynamoDBType ddType = typeMappings.get(columnName);

    // Check if this column maps a DynamoDB item.
    if (HiveDynamoDBTypeFactory.isHiveDynamoDBItemMapType(ddType)) {
      HiveDynamoDBItemType ddItemType = (HiveDynamoDBItemType) ddType;
      Map<String, AttributeValue> backupItem = ddItemType.parseDynamoDBData(data, fieldOI);

      // We give higher priority to attributes directly mapped to
      // columns. So we do not update the value of an attribute if
      // it already exists. This can happen in case of partial schemas
      // when there is a full backup column and attribute mapped
      // columns.
      for (Map.Entry<String, AttributeValue> entry : backupItem.entrySet()) {
        if (!columnMappings.containsValue(entry.getKey())) {
          item.put(entry.getKey(), entry.getValue());
        }
      }
    } else {
      // User has mapped individual attribute in DynamoDB to
      // corresponding Hive columns.
      AttributeValue attributeValue = data == null ?
          DynamoDBDataParser.getNullAttribute(nullSerialization) :
          ddType.getDynamoDBData(data, fieldOI, nullSerialization);

      if (attributeValue != null) {
        item.put(columnMappings.get(columnName), attributeValue);
      }
    }
  }

  return new DynamoDBItemWritable(item);
}
 

private String[] parseHeader(StructObjectInspector inspector) {
    List<? extends StructField> fields = inspector.getAllStructFieldRefs();
    return fields.stream().map(StructField::getFieldName)
                 .collect(Collectors.toList())
                 .toArray(new String[]{});
}
 

/**
   * This method takes an object representing a row of data from Hive, and
   * uses the ObjectInspector to get the data for each column and serialize
   * it.
   */
  //@Override
  public Writable serialize(Object obj, ObjectInspector oi)
          throws SerDeException {
  	ExecRow row = null;
  	int[] execRowFormatIds = null;
      try {
	List<NameType> nameTypes = sqlUtil.getTableStructure(tableName);
	execRowFormatIds = sqlUtil.getExecRowFormatIds(colNames, nameTypes);
	row = sqlUtil.getExecRow(execRowFormatIds);
	if (row == null)
		throw new SerDeException("ExecRow Cannot be Null");
} catch (SQLException | StandardException | IOException e1) {
	throw new SerDeException(e1);
}    	
  	if (Log.isTraceEnabled())
  		SpliceLogUtils.trace(Log, "serialize with obj=%s, oi=%s",obj,oi);
      if (oi.getCategory() != ObjectInspector.Category.STRUCT) {
          throw new SerDeException(getClass().toString()
                  + " can only serialize struct types, but we got: "
                  + oi.getTypeName());
      }

      StructObjectInspector soi = (StructObjectInspector) oi;
      List<? extends StructField> fields = soi.getAllStructFieldRefs();

      try {

      	DataValueDescriptor dvd;
          for (int i = 0; i < fields.size(); i++) {
              StructField field = fields.get(i);
              dvd = row.getColumn(i+1);
              ObjectInspector fieldOI = field.getFieldObjectInspector();
              Object fieldObj = soi.getStructFieldData(obj, field);
              PrimitiveObjectInspector primOI = (PrimitiveObjectInspector) fieldOI;
              Object data = primOI.getPrimitiveJavaObject(fieldObj);

              PrimitiveCategory primitiveCategory = primOI.getPrimitiveCategory();
              switch (primitiveCategory) {
                  case BYTE:
                      dvd.setValue(((Byte) data).byteValue());
                      break;
                  case INT:
                      dvd.setValue(((Integer) data).intValue());
                      break;
                  case VARCHAR:
                      dvd.setValue(((HiveVarchar)data).getValue());
                      break;
                  case CHAR:
                      dvd.setValue(((HiveChar)data).getValue());
                      break;
                  case STRING:
                      dvd.setValue((String) data);
                      break;
                  case BINARY:
                      dvd.setValue((SerializationUtils.serialize((Serializable) data))); // is this right?  Should just be a byte[]
                      break;
                  case BOOLEAN:
                      dvd.setValue(((Boolean) data).booleanValue());
                      break;
                  case DECIMAL:
                      dvd.setValue(((HiveDecimal) data).doubleValue());
                      break;
                  case DOUBLE:
                      dvd.setValue(((Double) data).doubleValue());
                      break;
                  case FLOAT:
                      dvd.setValue(((Float) data).floatValue());
                      break;
                  case LONG:
                      dvd.setValue(((Long) data).longValue());
                      break;
                  case SHORT:
                      dvd.setValue(((Short) data).shortValue());
                      break;
                  case TIMESTAMP:
                      dvd.setValue((Timestamp) data);
                      break;
                  case DATE:
                      dvd.setValue((java.sql.Date) data);
                      break;
                  default:
                      throw new SerDeException(String.format("Hive Type %s Not Supported Yet",primOI.getPrimitiveCategory()));
              }
          }

      } catch (StandardException e) {
          // TODO Auto-generated catch block
          throw new RuntimeException("Serialized Object To Java Type Error");
      }
      ExecRowWritable rowWritable = new ExecRowWritable(WriteReadUtils.getExecRowFromTypeFormatIds(execRowFormatIds));
      rowWritable.set(row);
      return rowWritable;
  }