Java Code Examples for org.apache.pig.data.DataType#ERROR

The following examples show how to use org.apache.pig.data.DataType#ERROR . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: HiveRCSchemaUtil.java    From spork with Apache License 2.0 6 votes vote down vote up 
/**
    * Returns the pig DataType for the hive type
    * 
    * @param hiveType
    * @return byte from DataType
    */
   public static byte findPigDataType(String hiveType) {
hiveType = hiveType.toLowerCase();

if (hiveType.equals("string"))
    return DataType.CHARARRAY;
else if (hiveType.equals("int"))
    return DataType.INTEGER;
else if (hiveType.equals("bigint") || hiveType.equals("long"))
    return DataType.LONG;
else if (hiveType.equals("float"))
    return DataType.FLOAT;
else if (hiveType.equals("double"))
    return DataType.DOUBLE;
else if (hiveType.equals("boolean"))
    return DataType.BOOLEAN;
else if (hiveType.equals("byte"))
    return DataType.INTEGER;
else if (hiveType.contains("array"))
    return DataType.TUPLE;
else if (hiveType.contains("map"))
    return DataType.MAP;
else
    return DataType.ERROR;
   }
Example 2
Source File: TOBAG.java    From spork with Apache License 2.0 5 votes vote down vote up 
@Override
public Schema outputSchema(Schema inputSch) {
    byte type = DataType.ERROR;
    Schema innerSchema = null;
    if(inputSch != null){
        for(FieldSchema fs : inputSch.getFields()){
            if(type == DataType.ERROR){
                type = fs.type;
                innerSchema = fs.schema;
            }else{
                if( type != fs.type || !nullEquals(innerSchema, fs.schema)){
                    // invalidate the type
                    type = DataType.ERROR;
                    break;
                }
            }
        }
    }
    try {
        if(type == DataType.ERROR){
            return Schema.generateNestedSchema(DataType.BAG, DataType.NULL);
        }
        FieldSchema innerFs = new Schema.FieldSchema(null, innerSchema, type);
        Schema innerSch = new Schema(innerFs);
        Schema bagSchema = new Schema(new FieldSchema(null, innerSch, DataType.BAG));
        return bagSchema;
    } catch (FrontendException e) {
        //This should not happen
        throw new RuntimeException("Bug : exception thrown while " +
                "creating output schema for TOBAG udf", e);
    }

}
Example 3
Source File: HBaseStorage.java    From spork with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
private byte[] objToBytes(Object o, byte type) throws IOException {
    LoadStoreCaster caster = (LoadStoreCaster) caster_;
    if (o == null) return null;
    switch (type) {
    case DataType.BYTEARRAY: return ((DataByteArray) o).get();
    case DataType.BAG: return caster.toBytes((DataBag) o);
    case DataType.CHARARRAY: return caster.toBytes((String) o);
    case DataType.DOUBLE: return caster.toBytes((Double) o);
    case DataType.FLOAT: return caster.toBytes((Float) o);
    case DataType.INTEGER: return caster.toBytes((Integer) o);
    case DataType.LONG: return caster.toBytes((Long) o);
    case DataType.BIGINTEGER: return caster.toBytes((BigInteger) o);
    case DataType.BIGDECIMAL: return caster.toBytes((BigDecimal) o);
    case DataType.BOOLEAN: return caster.toBytes((Boolean) o);
    case DataType.DATETIME: return caster.toBytes((DateTime) o);

    // The type conversion here is unchecked.
    // Relying on DataType.findType to do the right thing.
    case DataType.MAP: return caster.toBytes((Map<String, Object>) o);

    case DataType.NULL: return null;
    case DataType.TUPLE: return caster.toBytes((Tuple) o);
    case DataType.ERROR: throw new IOException("Unable to determine type of " + o.getClass());
    default: throw new IOException("Unable to find a converter for tuple field " + o);
    }
}
Example 4
Source File: TOBAG2.java    From spork with Apache License 2.0 5 votes vote down vote up 
@Override
public Schema outputSchema(Schema inputSch) {
    byte type = DataType.ERROR;
    Schema innerSchema = null;
    if(inputSch != null){
        for(FieldSchema fs : inputSch.getFields()){
            if(type == DataType.ERROR){
                type = fs.type;
                innerSchema = fs.schema;
            }else{
                if( type != fs.type || !nullEquals(innerSchema, fs.schema)){
                    // invalidate the type
                    type = DataType.ERROR;
                    break;
                }
            }
        }
    }
    try {
        if(type == DataType.ERROR){
            return Schema.generateNestedSchema(DataType.BAG, DataType.NULL);
        }
        FieldSchema innerFs = new Schema.FieldSchema(null, innerSchema, type);
        Schema innerSch = new Schema(innerFs);
        Schema bagSchema = new Schema(new FieldSchema(null, innerSch, DataType.BAG));
        return bagSchema;
    } catch (FrontendException e) {
        //This should not happen
        throw new RuntimeException("Bug : exception thrown while " +
                "creating output schema for TOBAG udf", e);
    }

}
Example 5
Source File: SequenceFileLoader.java    From spork with Apache License 2.0 5 votes vote down vote up 
protected void setKeyType(Class<?> keyClass) throws BackendException {
  this.keyType |= inferPigDataType(keyClass);
  if (keyType == DataType.ERROR) {
    LOG.warn("Unable to translate key "+key.getClass()+" to a Pig datatype");
    throw new BackendException("Unable to translate "+key.getClass()+" to a Pig datatype");
  }
}
Example 6
Source File: SequenceFileLoader.java    From spork with Apache License 2.0 5 votes vote down vote up 
protected void setValueType(Class<?> valueClass) throws BackendException {
  this.valType |= inferPigDataType(valueClass);
  if (keyType == DataType.ERROR) {
    LOG.warn("Unable to translate key "+key.getClass()+" to a Pig datatype");
    throw new BackendException("Unable to translate "+key.getClass()+" to a Pig datatype");
  }
}
Example 7
Source File: SequenceFileLoader.java    From spork with Apache License 2.0 5 votes vote down vote up 
protected byte inferPigDataType(Type t) {
  if (t == BytesWritable.class) return DataType.BYTEARRAY;
  else if (t == Text.class) return DataType.CHARARRAY;
  else if (t == IntWritable.class) return DataType.INTEGER;
  else if (t == LongWritable.class) return DataType.LONG;
  else if (t == FloatWritable.class) return DataType.FLOAT;
  else if (t == DoubleWritable.class) return DataType.DOUBLE;
  else if (t == BooleanWritable.class) return DataType.BOOLEAN;
  else if (t == ByteWritable.class) return DataType.BYTE;
  else if (t == DateTimeWritable.class) return DataType.DATETIME;
  // not doing maps or other complex types for now
  else return DataType.ERROR;
}
Example 8
Source File: AbstractAccumuloStorage.java    From spork with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
protected byte[] objToBytes(Object o, byte type) throws IOException {
    if (o == null)
        return null;
    switch (type) {
    case DataType.BYTEARRAY:
        return ((DataByteArray) o).get();
    case DataType.BAG:
        return caster.toBytes((DataBag) o);
    case DataType.CHARARRAY:
        return caster.toBytes((String) o);
    case DataType.DOUBLE:
        return caster.toBytes((Double) o);
    case DataType.FLOAT:
        return caster.toBytes((Float) o);
    case DataType.INTEGER:
        return caster.toBytes((Integer) o);
    case DataType.LONG:
        return caster.toBytes((Long) o);
    case DataType.BIGINTEGER:
        return caster.toBytes((BigInteger) o);
    case DataType.BIGDECIMAL:
        return caster.toBytes((BigDecimal) o);
    case DataType.BOOLEAN:
        return caster.toBytes((Boolean) o);
    case DataType.DATETIME:
        return caster.toBytes((DateTime) o);

        // The type conversion here is unchecked.
        // Relying on DataType.findType to do the right thing.
    case DataType.MAP:
        return caster.toBytes((Map<String, Object>) o);

    case DataType.NULL:
        return null;
    case DataType.TUPLE:
        return caster.toBytes((Tuple) o);
    case DataType.ERROR:
        throw new IOException("Unable to determine type of " + o.getClass());
    default:
        throw new IOException("Unable to find a converter for tuple field "
                + o);
    }
}
Example 9
Source File: Coalesce.java    From datafu with Apache License 2.0 4 votes vote down vote up 
@Override
public Schema getOutputSchema(Schema input)
{
  if (input.getFields().size() == 0)
  {
    throw new RuntimeException("Expected at least one parameter");
  }
      
  Byte outputType = null;
  int pos = 0;
  for (FieldSchema field : input.getFields())
  {
    if (DataType.isSchemaType(field.type))
    {
      throw new RuntimeException(String.format("Not supported on schema types.  Found %s in position %d.",DataType.findTypeName(field.type),pos));
    }
    
    if (DataType.isComplex(field.type))
    {
      throw new RuntimeException(String.format("Not supported on complex types.  Found %s in position %d.",DataType.findTypeName(field.type),pos));
    }
    
    if (!DataType.isUsableType(field.type))
    {
      throw new RuntimeException(String.format("Not a usable type.  Found %s in position %d.",DataType.findTypeName(field.type),pos));
    }
    
    if (outputType == null)
    {
      outputType = field.type;
    }
    else if (!outputType.equals(field.type))
    {        
      if (strict)
      {
        throw new RuntimeException(String.format("Expected all types to be equal, but found '%s' in position %d.  First element has type '%s'.  "
                                                 + "If you'd like to attempt merging types, use the '%s' option, as '%s' is the default.",
                                                 DataType.findTypeName(field.type),pos,DataType.findTypeName((byte)outputType),LAZY_OPTION,STRICT_OPTION));
      }
      else
      {
        byte merged = DataType.mergeType(outputType, field.type);
        if (merged == DataType.ERROR)
        {
          throw new RuntimeException(String.format("Expected all types to be equal, but found '%s' in position %d, where output type is '%s', and types could not be merged.",
                                                   DataType.findTypeName(field.type),pos,DataType.findTypeName((byte)outputType)));
        }
        outputType = merged;
      }
    }
    
    pos++;
  }
  
  getInstanceProperties().put("type", outputType);
      
  return new Schema(new Schema.FieldSchema("item",outputType));
}