Java Code Examples for org.apache.arrow.vector.types.pojo.Field#getName()

public StructArrowValueProjector(FieldReader structReader)
{
    this.structReader = requireNonNull(structReader, "structReader is null");

    ImmutableMap.Builder<String, Projection> projectionMapBuilder = ImmutableMap.builder();

    List<Field> children = structReader.getField().getChildren();

    for (Field child : children) {
        String childName = child.getName();
        Types.MinorType minorType = Types.getMinorTypeForArrowType(child.getType());
        Projection projection = createValueProjection(minorType);
        projectionMapBuilder.put(childName, projection);
    }

    this.projectionsMap = projectionMapBuilder.build();
}
 

/**
 * Addes the specified Apache Arrow field to the Scan to satisfy the requested projection.
 *
 * @param scan The scan object that will be used to read data from HBase.
 * @param field The field to be added to the scan.
 */
private void addToProjection(Scan scan, Field field)
{
    //ignore the special 'row' column since we get that by default.
    if (HbaseSchemaUtils.ROW_COLUMN_NAME.equalsIgnoreCase(field.getName())) {
        return;
    }

    Types.MinorType columnType = Types.getMinorTypeForArrowType(field.getType());
    switch (columnType) {
        case STRUCT:
            for (Field child : field.getChildren()) {
                scan.addColumn(field.getName().getBytes(UTF_8), child.getName().getBytes(UTF_8));
            }
            return;
        default:
            String[] nameParts = HbaseSchemaUtils.extractColumnParts(field.getName());
            if (nameParts.length != 2) {
                throw new RuntimeException("Column name " + field.getName() + " does not meet family:column hbase convention.");
            }
            scan.addColumn(nameParts[0].getBytes(UTF_8), nameParts[1].getBytes(UTF_8));
    }
}
 

public static ColumnMetaData metaDataFromField(Field field) {
    ArrowType arrowType = field.getFieldType().getType();
    if (arrowType instanceof ArrowType.Int) {
        ArrowType.Int intType = (ArrowType.Int) arrowType;
        return intType.getBitWidth() == 32 ? new IntegerMetaData(field.getName()) : new LongMetaData(field.getName());
    } else if (arrowType instanceof ArrowType.Bool) {
        return new BooleanMetaData(field.getName());
    } else if (arrowType instanceof ArrowType.FloatingPoint) {
        ArrowType.FloatingPoint floatingPointType = (ArrowType.FloatingPoint) arrowType;
        return floatingPointType.getPrecision() == FloatingPointPrecision.DOUBLE ? new DoubleMetaData(field.getName()) : new FloatMetaData(field.getName());
    } else if (arrowType instanceof ArrowType.Binary) {
        return new BinaryMetaData(field.getName());
    } else if (arrowType instanceof ArrowType.Utf8) {
        return new StringMetaData(field.getName());
    } else if (arrowType instanceof ArrowType.Date) {
        return new TimeMetaData(field.getName());
    } else {
        throw new IllegalStateException("Illegal type " + field.getFieldType().getType());
    }
}
 

public <T extends ValueVector> T addField(Field field, Class<T> clazz) throws SchemaChangeException {
  ValueVector v = fieldVectorMap.get(field.getName().toLowerCase());
  if (v == null || v.getClass() != clazz) {
    // Field does not exist--add it to the map and the output container.
    v = TypeHelper.getNewVector(field, allocator, callBack);
    if (!clazz.isAssignableFrom(v.getClass())) {
      throw new SchemaChangeException(
              String.format(
                      "The class that was provided, %s, does not correspond to the "
                              + "expected vector type of %s.",
                      clazz.getSimpleName(), v.getClass().getSimpleName()));
    }

    String fieldName = field.getName();
    addVectorForFieldName(v, fieldName);
    // Added new vectors to the container--mark that the schema has changed.
  }

  return clazz.cast(v);
}
 

private static ColumnMetaData metaDataFromField(Field field) {
    ArrowType arrowType = field.getFieldType().getType();
    if(arrowType instanceof ArrowType.Int) {
        val intType = (ArrowType.Int) arrowType;
        if(intType.getBitWidth() == 32)
            return new IntegerMetaData(field.getName());
        else {
            return new LongMetaData(field.getName());
        }
    }
    else if(arrowType instanceof ArrowType.Bool) {
        return new BooleanMetaData(field.getName());
    }
    else if(arrowType  instanceof ArrowType.FloatingPoint) {
        val floatingPointType = (ArrowType.FloatingPoint) arrowType;
        if(floatingPointType.getPrecision() == FloatingPointPrecision.DOUBLE)
            return new DoubleMetaData(field.getName());
        else {
            return new FloatMetaData(field.getName());
        }
    }
    else if(arrowType instanceof  ArrowType.Binary) {
        return new BinaryMetaData(field.getName());
    }
    else if(arrowType instanceof ArrowType.Utf8) {
        return new StringMetaData(field.getName());

    }
    else if(arrowType instanceof ArrowType.Date) {
        return new TimeMetaData(field.getName());
    }
    else {
        throw new IllegalStateException("Illegal type " + field.getFieldType().getType());
    }

}
 

private static DataType fromArrowField(Field field)
{
  if (field.getType().getTypeID() == ArrowTypeID.List)
  {
    Field elementField = field.getChildren().get(0);
    DataType elementType = fromArrowField(elementField);

    return new ArrayType(elementType, elementField.isNullable());
  }

  if (field.getType().getTypeID() == ArrowTypeID.Struct)
  {
    java.util.List<Field> fieldChildren = field.getChildren();
    StructField[] structFields = new StructField[fieldChildren.size()];

    int ind = 0;

    for (Field childField : field.getChildren())
    {
      DataType childType = fromArrowField(childField);
      structFields[ind++] = new StructField(childField.getName(), childType, childField.isNullable(), Metadata.empty());
    }

    return new StructType(structFields);
  }

  return fromArrowType(field.getType());
}
 

private static ColumnMetaData metaDataFromField(Field field) {
    ArrowType arrowType = field.getFieldType().getType();
    if(arrowType instanceof ArrowType.Int) {
        val intType = (ArrowType.Int) arrowType;
        if(intType.getBitWidth() == 32)
            return new IntegerMetaData(field.getName());
        else {
            return new LongMetaData(field.getName());
        }
    }
    else if(arrowType instanceof ArrowType.Bool) {
        return new BooleanMetaData(field.getName());
    }
    else if(arrowType  instanceof ArrowType.FloatingPoint) {
        val floatingPointType = (ArrowType.FloatingPoint) arrowType;
        if(floatingPointType.getPrecision() == FloatingPointPrecision.DOUBLE)
            return new DoubleMetaData(field.getName());
        else {
            return new FloatMetaData(field.getName());
        }
    }
    else if(arrowType instanceof  ArrowType.Binary) {
        return new BinaryMetaData(field.getName());
    }
    else if(arrowType instanceof ArrowType.Utf8) {
        return new StringMetaData(field.getName());

    }
    else if(arrowType instanceof ArrowType.Date) {
        return new TimeMetaData(field.getName());
    }
    else {
        throw new IllegalStateException("Illegal type " + field.getFieldType().getType());
    }

}
 

public static ImmutableMap<String, Column> getColumnsFromSchema(BatchSchema schema){
  ImmutableMap.Builder<String, Column> columns = ImmutableMap.builder();
  for (int i = 0; i < schema.getFieldCount(); ++i) {
    final Field column = schema.getColumn(i);
    final String name = column.getName();
    final MajorType type = getMajorTypeForField(column);
    DataType dataType = DataTypeUtil.getDataType(type);

    columns.put(name, new Column(name, dataType, i));
  }

  return columns.build();
}
 

private static Map<String, Integer> getColumnIndicesFromSchema(BatchSchema schema){
  ImmutableMap.Builder<String, Integer> columns = ImmutableMap.builder();

  for (int i = 0; i < schema.getFieldCount(); ++i) {
    final Field column = schema.getColumn(i);
    final String name = column.getName();

    columns.put(name, i);
  }

  return columns.build();
}
 

private PrimitiveType getPrimitiveType(Field field) {
  MajorType majorType = getMajorTypeForField(field);
  MinorType minorType = majorType.getMinorType();
  String name = field.getName();
  PrimitiveTypeName primitiveTypeName = ParquetTypeHelper.getPrimitiveTypeNameForMinorType(minorType);
  if (primitiveTypeName == null) {
    return null;
  }
  OriginalType originalType = ParquetTypeHelper.getOriginalTypeForMinorType(minorType);
  int length = ParquetTypeHelper.getLengthForMinorType(minorType);
  DecimalMetadata decimalMetadata  = ParquetTypeHelper.getDecimalMetadataForField(majorType);
  return new PrimitiveType(OPTIONAL, primitiveTypeName, length, name, originalType, decimalMetadata, null);
}
 

public TwosComplementValuePair(BufferAllocator allocator, Field field, byte[] value) {
  super(field.getName(), value != null ? DecimalTools.signExtend16(value) : null);
  CompleteType type = CompleteType.fromField(field);
  scale = type.getScale();
  precision = type.getPrecision();
  if (value != null) {
    buf = allocator.buffer(16);
    /* set the bytes in LE format in the buffer of decimal vector. since we
     * are populating the decimal vector multiple times with the same buffer, it
     * is fine to swap the bytes once here as opposed to swapping while writing.
     */
    byte [] decimalBytesInLE = DecimalUtils.convertDecimalBytesToArrowByteArray(value);
    buf.setBytes(0, decimalBytesInLE);
  }
}
 

/**
 * Allows for coercing types in the event that schema has evolved or there were other data issues.
 * @param field The Apache Arrow field that the value belongs to.
 * @param origVal The original value from Redis (before any conversion or coercion).
 * @return The coerced value.
 */
public static Object convert(Field field, String origVal)
{
    if (origVal == null) {
        return origVal;
    }

    ArrowType arrowType = field.getType();
    Types.MinorType minorType = Types.getMinorTypeForArrowType(arrowType);

    switch (minorType) {
        case VARCHAR:
            return origVal;
        case INT:
        case SMALLINT:
        case TINYINT:
            return Integer.valueOf(origVal);
        case BIGINT:
            return Long.valueOf(origVal);
        case FLOAT8:
            return Double.valueOf(origVal);
        case FLOAT4:
            return Float.valueOf(origVal);
        case BIT:
            return Boolean.valueOf(origVal);
        case VARBINARY:
            try {
                return origVal.getBytes("UTF-8");
            }
            catch (UnsupportedEncodingException ex) {
                throw new RuntimeException(ex);
            }
        default:
            throw new RuntimeException("Unsupported type conversation " + minorType + " field: " + field.getName());
    }
}
 

protected Map<String, Object> doProject()
{
    List<Field> fields = structReader.getField().getChildren();
    Map<String, Object> nameToValues = new HashMap<>();
    for (Field child : fields) {
        String childName = child.getName();
        FieldReader subReader = structReader.reader(childName);
        Projection childProjection = projectionsMap.get(childName);
        nameToValues.put(child.getName(), childProjection.doProjection(subReader));
    }
    return nameToValues;
}
 

public static BatchSchemaField fromField(Field field) {
  List<Field> children = field.getChildren().stream().map(
    BatchSchemaField::fromField).collect(Collectors.toList());

  return new BatchSchemaField(field.getName(), field.isNullable(), field.getType(), children);
}
 

public static FieldVector getNewVector(Field field, BufferAllocator allocator, CallBack callBack) {
  if (field.getType() instanceof ObjectType) {
    return new ObjectVector(field.getName(), allocator);
  }

  MinorType type = org.apache.arrow.vector.types.Types.getMinorTypeForArrowType(field.getType());

  List<Field> children = field.getChildren();

  switch (type) {

  case UNION:
    UnionVector unionVector = new UnionVector(field.getName(), allocator, callBack);
    if (!children.isEmpty()) {
      unionVector.initializeChildrenFromFields(children);
    }
    return unionVector;
  case LIST:
    ListVector listVector = new ListVector(field.getName(), allocator, callBack);
    if (!children.isEmpty()) {
      listVector.initializeChildrenFromFields(children);
    }
    return listVector;
  case STRUCT:
    StructVector structVector = new StructVector(field.getName(), allocator, callBack);
    if (!children.isEmpty()) {
      structVector.initializeChildrenFromFields(children);
    }
    return structVector;

  case NULL:
    return new ZeroVector();
  case TINYINT:
    return new TinyIntVector(field, allocator);
  case UINT1:
    return new UInt1Vector(field, allocator);
  case UINT2:
    return new UInt2Vector(field, allocator);
  case SMALLINT:
    return new SmallIntVector(field, allocator);
  case INT:
    return new IntVector(field, allocator);
  case UINT4:
    return new UInt4Vector(field, allocator);
  case FLOAT4:
    return new Float4Vector(field, allocator);
  case INTERVALYEAR:
    return new IntervalYearVector(field, allocator);
  case TIMEMILLI:
    return new TimeMilliVector(field, allocator);
  case BIGINT:
    return new BigIntVector(field, allocator);
  case UINT8:
    return new UInt8Vector(field, allocator);
  case FLOAT8:
    return new Float8Vector(field, allocator);
  case DATEMILLI:
    return new DateMilliVector(field, allocator);
  case TIMESTAMPMILLI:
    return new TimeStampMilliVector(field, allocator);
  case INTERVALDAY:
    return new IntervalDayVector(field, allocator);
  case DECIMAL:
    return new DecimalVector(field, allocator);
  case FIXEDSIZEBINARY:
    return new FixedSizeBinaryVector(field.getName(), allocator, WIDTH_ESTIMATE);
  case VARBINARY:
    return new VarBinaryVector(field, allocator);
  case VARCHAR:
    return new VarCharVector(field, allocator);
  case BIT:
    return new BitVector(field, allocator);
  default:
    break;
  }
  // All ValueVector types have been handled.
  throw new UnsupportedOperationException(buildErrorMessage("get new vector", type));
}
 

public void updateColumnMetaData(String catalogName, String schemaName,
                                 String tableName, BatchSchema schema,
                                 List<Class<?>> getObjectClasses ) {
  final List<ColumnMetaData> newColumns =
      new ArrayList<>(schema.getFieldCount());
  for (int colOffset = 0; colOffset < schema.getFieldCount(); colOffset++) {
    final Field field = schema.getColumn(colOffset);
    Class<?> objectClass = getObjectClasses.get( colOffset );

    final String columnName = field.getName();

    final MajorType rpcDataType = getMajorTypeForField(field);
    final AvaticaType bundledSqlDataType = getAvaticaType(rpcDataType);
    final String columnClassName = objectClass.getName();

    final int nullability;
    switch ( rpcDataType.getMode() ) {
      case OPTIONAL: nullability = ResultSetMetaData.columnNullable; break;
      case REQUIRED: nullability = ResultSetMetaData.columnNoNulls;  break;
      // Should REPEATED still map to columnNoNulls? or to columnNullable?
      case REPEATED: nullability = ResultSetMetaData.columnNoNulls;  break;
      default:
        throw new AssertionError( "Unexpected new DataMode value '"
                                  + rpcDataType.getMode() + "'" );
    }
    final boolean isSigned = Types.isJdbcSignedType( rpcDataType );

    // TODO(DRILL-3355):  TODO(DRILL-3356):  When string lengths, precisions,
    // interval kinds, etc., are available from RPC-level data, implement:
    // - precision for ResultSetMetadata.getPrecision(...) (like
    //   getColumns()'s COLUMN_SIZE)
    // - scale for getScale(...), and
    // - and displaySize for getColumnDisplaySize(...).
    final int precision = Types.getPrecision(rpcDataType);
    final int scale = Types.getScale(rpcDataType);
    final int displaySize = Types.getJdbcDisplaySize(rpcDataType);

    ColumnMetaData col = new ColumnMetaData(
        colOffset,    // (zero-based ordinal (for Java arrays/lists).)
        false,        /* autoIncrement */
        false,        /* caseSensitive */
        true,         /* searchable */
        false,        /* currency */
        nullability,
        isSigned,
        displaySize,
        columnName,   /* label */
        columnName,   /* columnName */
        schemaName,
        precision,
        scale,
        tableName,
        catalogName,
        bundledSqlDataType,
        true,         /* readOnly */
        false,        /* writable */
        false,        /* definitelyWritable */
        columnClassName
       );
    newColumns.add(col);
  }
  columns = newColumns;
}
 

/**
 * Return the field value from a complex structure or list.
 * @param field is the field that we would like to extract from the provided value.
 * @param originalValue is the original value object.
 * @return the field's value as a List for a LIST field type, a Map for a STRUCT field type, or the actual
 * value if neither of the above.
 * @throws IllegalArgumentException if originalValue is not an instance of Map.
 * @throws RuntimeException if the fieldName does not exist in originalValue, if the fieldType is a STRUCT and
 * the fieldValue is not instance of Map, or if the fieldType is neither a LIST or a STRUCT but the fieldValue
 * is instance of Map (STRUCT).
 */
@Override
public Object getFieldValue(Field field, Object originalValue)
        throws RuntimeException
{
    Types.MinorType fieldType = Types.getMinorTypeForArrowType(field.getType());
    String fieldName = field.getName();
    Object fieldValue;

    if (originalValue instanceof Map) {
        if (((Map) originalValue).containsKey(fieldName)) {
            fieldValue = ((Map) originalValue).get(fieldName);
        }
        else {
            throw new RuntimeException("Field not found in Document: " + fieldName);
        }
    }
    else {
        throw new IllegalArgumentException("Invalid argument type. Expecting a Map, but got: " +
                originalValue.getClass().getTypeName());
    }

    switch (fieldType) {
        case LIST:
            return coerceListField(field, fieldValue);
        case STRUCT:
            if (fieldValue instanceof Map) {
                // Both fieldType and fieldValue are nested structures => return as map.
                return fieldValue;
            }
            break;
        default:
            if (!(fieldValue instanceof Map)) {
                return coerceField(field, fieldValue);
            }
            break;
    }

    throw new RuntimeException("Invalid field value encountered in Document for field: " + field +
            ",value: " + fieldValue);
}
 

private FieldWriter makeFieldWriter(FieldVector vector)
{
    Field field = vector.getField();
    String fieldName = field.getName();
    Types.MinorType fieldType = Types.getMinorTypeForArrowType(field.getType());
    Extractor extractor = extractors.get(fieldName);
    ConstraintProjector constraint = constraints.get(fieldName);
    FieldWriterFactory factory = fieldWriterFactories.get(fieldName);

    if (factory != null) {
        return factory.create(vector, extractor, constraint);
    }

    if (extractor == null) {
        throw new IllegalStateException("Missing extractor for field[" + fieldName + "]");
    }

    switch (fieldType) {
        case INT:
            return new IntFieldWriter((IntExtractor) extractor, (IntVector) vector, constraint);
        case BIGINT:
            return new BigIntFieldWriter((BigIntExtractor) extractor, (BigIntVector) vector, constraint);
        case DATEMILLI:
            return new DateMilliFieldWriter((DateMilliExtractor) extractor, (DateMilliVector) vector, constraint);
        case DATEDAY:
            return new DateDayFieldWriter((DateDayExtractor) extractor, (DateDayVector) vector, constraint);
        case TINYINT:
            return new TinyIntFieldWriter((TinyIntExtractor) extractor, (TinyIntVector) vector, constraint);
        case SMALLINT:
            return new SmallIntFieldWriter((SmallIntExtractor) extractor, (SmallIntVector) vector, constraint);
        case FLOAT4:
            return new Float4FieldWriter((Float4Extractor) extractor, (Float4Vector) vector, constraint);
        case FLOAT8:
            return new Float8FieldWriter((Float8Extractor) extractor, (Float8Vector) vector, constraint);
        case DECIMAL:
            return new DecimalFieldWriter((DecimalExtractor) extractor, (DecimalVector) vector, constraint);
        case BIT:
            return new BitFieldWriter((BitExtractor) extractor, (BitVector) vector, constraint);
        case VARCHAR:
            return new VarCharFieldWriter((VarCharExtractor) extractor, (VarCharVector) vector, constraint);
        case VARBINARY:
            return new VarBinaryFieldWriter((VarBinaryExtractor) extractor, (VarBinaryVector) vector, constraint);
        default:
            throw new RuntimeException(fieldType + " is not supported");
    }
}