Java Code Examples for org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo#setAllStructFieldTypeInfos()

The following examples show how to use org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo#setAllStructFieldTypeInfos() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MDSSerde.java    From multiple-dimension-spread with Apache License 2.0 6 votes vote down vote up 
private StructTypeInfo getAllReadTypeInfo( final String columnNameProperty , final String columnTypeProperty ){
  ArrayList<TypeInfo> fieldTypes = TypeInfoUtils.getTypeInfosFromTypeString( columnTypeProperty );
  ArrayList<String> columnNames = new ArrayList<String>();
  if ( columnNameProperty != null && 0 < columnNameProperty.length() ) {
    String[] columnNameArray = columnNameProperty.split(",");
    for( int i = 0 ; i < columnNameArray.length ; i++ ){
      columnNames.add( columnNameArray[i] );
      filedIndexMap.put( columnNameArray[i] , i );
    }
  }
  StructTypeInfo rootType = new StructTypeInfo();

  rootType.setAllStructFieldNames( columnNames );
  rootType.setAllStructFieldTypeInfos( fieldTypes );

  return rootType;
}
Example 2
Source File: MDSSerde.java    From multiple-dimension-spread with Apache License 2.0 6 votes vote down vote up 
private StructTypeInfo getColumnProjectionTypeInfo( final String columnNameProperty , final String columnTypeProperty , final String projectionColumnNames ){
  Set<String> columnNameSet = new HashSet<String>();
  for( String columnName : projectionColumnNames.split(",") ){
    columnNameSet.add( columnName );
  }

  ArrayList<TypeInfo> fieldTypes = TypeInfoUtils.getTypeInfosFromTypeString( columnTypeProperty );
  String[] splitNames = columnNameProperty.split(",");

  ArrayList<String> projectionColumnNameList = new ArrayList<String>();
  ArrayList<TypeInfo> projectionFieldTypeList = new ArrayList<TypeInfo>();
  for( int i = 0 ; i < fieldTypes.size() ; i++ ){
    if( columnNameSet.contains( splitNames[i] ) ){
      projectionColumnNameList.add( splitNames[i] );
      projectionFieldTypeList.add( fieldTypes.get(i) );
    }
    filedIndexMap.put( splitNames[i] , i );
  }
  StructTypeInfo rootType = new StructTypeInfo();

  rootType.setAllStructFieldNames( projectionColumnNameList );
  rootType.setAllStructFieldTypeInfos( projectionFieldTypeList );

  return rootType;
}
Example 3
Source File: HiveSchemaConverter.java    From kite with Apache License 2.0 6 votes vote down vote up 
public static Schema convertTable(String table, Collection<FieldSchema> columns,
                                  @Nullable PartitionStrategy strategy) {
  ArrayList<String> fieldNames = Lists.newArrayList();
  ArrayList<TypeInfo> fieldTypes = Lists.newArrayList();
  LinkedList<String> start = Lists.newLinkedList();
  Collection<String[]> requiredFields = requiredFields(strategy);

  List<Schema.Field> fields = Lists.newArrayList();
  for (FieldSchema column : columns) {
    // pass null for the initial path to exclude the table name
    TypeInfo type = parseTypeInfo(column.getType());
    fieldNames.add(column.getName());
    fieldTypes.add(type);
    fields.add(convertField(start, column.getName(), type, requiredFields));
  }

  StructTypeInfo struct = new StructTypeInfo();
  struct.setAllStructFieldNames(fieldNames);
  struct.setAllStructFieldTypeInfos(fieldTypes);

  Schema recordSchema = Schema.createRecord(table, doc(struct), null, false);
  recordSchema.setFields(fields);

  return recordSchema;
}
Example 4
Source File: OrcSerde.java    From hive-dwrf with Apache License 2.0 5 votes vote down vote up 
@Override
public void initialize(Configuration conf, Properties table) {
  // Read the configuration parameters
  String columnNameProperty = table.getProperty("columns");
  // NOTE: if "columns.types" is missing, all columns will be of String type
  String columnTypeProperty = table.getProperty("columns.types");

  // Parse the configuration parameters
  ArrayList<String> columnNames = EMPTY_STRING_ARRAYLIST;
  if (columnNameProperty != null && columnNameProperty.length() > 0) {
    String[] splits = columnNameProperty.split(",");
    columnNames = new ArrayList<String>(splits.length);

    for(String name: splits) {
      columnNames.add(name);
    }
  }
  if (columnTypeProperty == null) {
    // Default type: all string
    StringBuilder sb = new StringBuilder();
    for (int i = 0; i < columnNames.size(); i++) {
      if (i > 0) {
        sb.append(":");
      }
      sb.append("string");
    }
    columnTypeProperty = sb.toString();
  }

  ArrayList<TypeInfo> fieldTypes =
    TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty);
  StructTypeInfo rootType = new StructTypeInfo();
  rootType.setAllStructFieldNames(columnNames);
  rootType.setAllStructFieldTypeInfos(fieldTypes);
  inspector = new OrcLazyRowObjectInspector(rootType);
}