org.apache.pig.data.DataType#DOUBLE

Source File: CastUtils.java From spork with Apache License 2.0

6 votes

/**
 *
 * @param caster LoadCaster to be used to convert the bytes into a field.
 * @param bytes
 * @param fieldSchema schema of Bag or Tuple; pass in null if a simple type.
 * @param dataType type from DataType
 * @return converted object.
 * @throws IOException
 */
public static Object convertToType(LoadCaster caster, byte[] bytes,
        ResourceFieldSchema fieldSchema, byte dataType) throws IOException {
    switch (dataType) {
    case (DataType.BAG): return caster.bytesToBag(bytes, fieldSchema);
    case (DataType.BYTEARRAY): return new DataByteArray(bytes);
    case (DataType.CHARARRAY): return caster.bytesToCharArray(bytes);
    case (DataType.DOUBLE): return caster.bytesToDouble(bytes);
    case (DataType.FLOAT): return caster.bytesToFloat(bytes);
    case (DataType.INTEGER): return caster.bytesToInteger(bytes);
    case (DataType.BIGINTEGER): return caster.bytesToBigInteger(bytes);
    case (DataType.BIGDECIMAL): return caster.bytesToBigDecimal(bytes);
    case (DataType.LONG): return caster.bytesToLong(bytes);
    case (DataType.BOOLEAN): return caster.bytesToBoolean(bytes);
    case (DataType.DATETIME): return caster.bytesToDateTime(bytes);
    case (DataType.MAP): return caster.bytesToMap(bytes, fieldSchema);
    case (DataType.TUPLE): return caster.bytesToTuple(bytes, fieldSchema);
    default: throw new IOException("Unknown type " + dataType);
    }
}

Source File: HiveRCSchemaUtil.java From spork with Apache License 2.0

6 votes

/**
    * Returns the pig DataType for the hive type
    * 
    * @param hiveType
    * @return byte from DataType
    */
   public static byte findPigDataType(String hiveType) {
hiveType = hiveType.toLowerCase();

if (hiveType.equals("string"))
    return DataType.CHARARRAY;
else if (hiveType.equals("int"))
    return DataType.INTEGER;
else if (hiveType.equals("bigint") || hiveType.equals("long"))
    return DataType.LONG;
else if (hiveType.equals("float"))
    return DataType.FLOAT;
else if (hiveType.equals("double"))
    return DataType.DOUBLE;
else if (hiveType.equals("boolean"))
    return DataType.BOOLEAN;
else if (hiveType.equals("byte"))
    return DataType.INTEGER;
else if (hiveType.contains("array"))
    return DataType.TUPLE;
else if (hiveType.contains("map"))
    return DataType.MAP;
else
    return DataType.ERROR;
   }

Source File: SchemaTupleClassGenerator.java From spork with Apache License 2.0

6 votes

public String typeName(byte type) {
    switch(type) {
        case (DataType.INTEGER): return "int";
        case (DataType.LONG): return "long";
        case (DataType.FLOAT): return "float";
        case (DataType.DOUBLE): return "double";
        case (DataType.BYTEARRAY): return "byte[]";
        case (DataType.CHARARRAY): return "String";
        case (DataType.BOOLEAN): return "boolean";
        case (DataType.DATETIME): return "DateTime";
        case (DataType.BIGDECIMAL): return "BigDecimal";
        case (DataType.BIGINTEGER): return "BigInteger";
        case (DataType.TUPLE): return "Tuple";
        case (DataType.BAG): return "DataBag";
        case (DataType.MAP): return "Map";
        default: throw new RuntimeException("Can't return String for given type " + DataType.findTypeName(type));
    }
}

Source File: SchemaTupleClassGenerator.java From spork with Apache License 2.0

6 votes

@Override
public void process(int fieldPos, Schema.FieldSchema fs) {
    add("public "+typeName()+" getDummy_"+fieldPos+"() {");
    switch (fs.type) {
    case (DataType.INTEGER): add("    return 0;"); break;
    case (DataType.LONG): add("    return 0L;"); break;
    case (DataType.FLOAT): add("    return 0.0f;"); break;
    case (DataType.DOUBLE): add("    return 0.0;"); break;
    case (DataType.BOOLEAN): add("    return true;"); break;
    case (DataType.DATETIME): add("    return new DateTime();"); break;
    case (DataType.BIGDECIMAL): add("    return (BigDecimal)null;"); break;
    case (DataType.BIGINTEGER): add("    return (BigInteger)null;"); break;
    case (DataType.BYTEARRAY): add("    return (byte[])null;"); break;
    case (DataType.CHARARRAY): add("    return (String)null;"); break;
    case (DataType.TUPLE): add("    return (Tuple)null;"); break;
    case (DataType.BAG): add("    return (DataBag)null;"); break;
    case (DataType.MAP): add("    return (Map<String,Object>)null;"); break;
    default: throw new RuntimeException("Unsupported type");
    }
    add("}");
    addBreak();
}

Source File: ParquetLoader.java From parquet-mr with Apache License 2.0

6 votes

@Override
public List<String> getPredicateFields(String s, Job job) throws IOException {
  if(!job.getConfiguration().getBoolean(ENABLE_PREDICATE_FILTER_PUSHDOWN, DEFAULT_PREDICATE_PUSHDOWN_ENABLED)) {
    return null;
  }

  List<String> fields = new ArrayList<String>();

  for(FieldSchema field : schema.getFields()) {
    switch(field.type) {
      case DataType.BOOLEAN:
      case DataType.INTEGER:
      case DataType.LONG:
      case DataType.FLOAT:
      case DataType.DOUBLE:
      case DataType.CHARARRAY:
        fields.add(field.alias);
        break;
      default:
        // Skip BYTEARRAY, TUPLE, MAP, BAG, DATETIME, BIGINTEGER, BIGDECIMAL
        break;
    }
  }

  return fields;
}

Source File: TestPOBinCond.java From spork with Apache License 2.0

5 votes

private DataBag getBag(byte type) {
    DataBag bag = DefaultBagFactory.getInstance().newDefaultBag();
    for(int i = 0; i < 10; i ++) {
        Tuple t = TupleFactory.getInstance().newTuple();
        switch(type) {
            case DataType.BOOLEAN:
                t.append(r.nextBoolean());
                break;
            case DataType.INTEGER:
                t.append(r.nextInt(2));
                break;
            case DataType.LONG:
                t.append(r.nextLong() % 2L);
                break;
            case DataType.FLOAT:
                t.append((i % 2 == 0 ? 1.0f : 0.0f));
                break;
            case DataType.DOUBLE:
                t.append((i % 2 == 0 ? 1.0 : 0.0));
                break;
            case DataType.DATETIME:
                t.append(new DateTime(r.nextLong() % 2L));
                break;
        }
        t.append(1);
        t.append(0);
        bag.add(t);
    }
    return bag;
}

Source File: Sty.java From validatar with Apache License 2.0

5 votes

private TypedObject getTypedObject(Object data, FieldDetail detail) throws ExecException {
    if (data == null) {
        return null;
    }
    byte type = detail.type;
    switch (type) {
        case DataType.BOOLEAN:
            return TypeSystem.asTypedObject(DataType.toBoolean(data, type));
        case DataType.INTEGER:
        case DataType.LONG:
            return TypeSystem.asTypedObject(DataType.toLong(data, type));
        case DataType.FLOAT:
        case DataType.DOUBLE:
            return TypeSystem.asTypedObject(DataType.toDouble(data, type));
        case DataType.DATETIME:
            return TypeSystem.asTypedObject(new Timestamp(DataType.toDateTime(data, type).getMillis()));
        case DataType.BYTE:
        case DataType.BYTEARRAY:
        case DataType.CHARARRAY:
            return TypeSystem.asTypedObject(DataType.toString(data, type));
        case DataType.BIGINTEGER:
        case DataType.BIGDECIMAL:
            return TypeSystem.asTypedObject(DataType.toBigDecimal(data, type));
        default:
            //TUPLE, BAG, MAP, INTERNALMAP, GENERIC_WRITABLECOMPARABLE, ERROR, UNKNOWN, NULL and anything else
            return null;
    }
}

Source File: VARBAG.java From spork with Apache License 2.0

5 votes

public VARBAG(String bagColName, String tupleColName, String fieldType) {

        if ( bagColName== null || tupleColName == null  || fieldType == null)  {
	    throw new RuntimeException("The bagColName  and fieldType cannot be null");
        }

    	this.bagColName   = bagColName;	
    	this.tupleColName = tupleColName;	
        if ( fieldType.equalsIgnoreCase( "CHARARRAY" )){ 
             this.fieldType = DataType.CHARARRAY;

        } else if ( fieldType.equalsIgnoreCase( "DOUBLE" )){ 
            this.fieldType = DataType.DOUBLE;

        } else if ( fieldType.equalsIgnoreCase( "FLOAT" )){ 
            this.fieldType = DataType.FLOAT; 

        } else if ( fieldType.equalsIgnoreCase( "BOOLEAN" )) {
            this.fieldType = DataType.BOOLEAN;
            
        } else if ( fieldType.equalsIgnoreCase( "INTEGER" )){ 
            this.fieldType = DataType.INTEGER;

        } else if ( fieldType.equalsIgnoreCase( "LONG" )){ 
            this.fieldType = DataType.LONG; 

        } else if ( fieldType.equalsIgnoreCase( "MAP" )){ 
            this.fieldType = DataType.MAP; 
        } else {
	    throw new RuntimeException("This type"+ fieldType +"is not supported in " + this.getClass().getSimpleName());
        }

    }

Source File: TOBAG.java From spork with Apache License 2.0

5 votes

public TOBAG(String bagColName, String tupleColName, String fieldType) {

        if ( bagColName== null || tupleColName == null  || fieldType == null)  {
	    throw new RuntimeException("The bagColName  and fieldType cannot be null");
        }

    	this.bagColName   = bagColName;	
    	this.tupleColName = tupleColName;	

        if ( fieldType.equalsIgnoreCase( "CHARARRAY" )){ 
             this.fieldType = DataType.CHARARRAY;

        } else if ( fieldType.equalsIgnoreCase( "DOUBLE" )){ 
            this.fieldType = DataType.DOUBLE;

        } else if ( fieldType.equalsIgnoreCase( "FLOAT" )){ 
            this.fieldType = DataType.FLOAT; 

        } else if ( fieldType.equalsIgnoreCase( "BOOLEAN" )){ 
            this.fieldType = DataType.BOOLEAN; 

        } else if ( fieldType.equalsIgnoreCase( "INTEGER" )){ 
            this.fieldType = DataType.INTEGER;

        } else if ( fieldType.equalsIgnoreCase( "LONG" )){ 
            this.fieldType = DataType.LONG; 

        } else if ( fieldType.equalsIgnoreCase( "MAP" )){ 
            this.fieldType = DataType.MAP; 
        } else {
	    throw new RuntimeException("This type"+ fieldType +"is not supported in TOBAG");
        }

    }

Source File: TestSchema.java From spork with Apache License 2.0

5 votes

@Test
public void testCharArray2Numeric(){
	byte[] numbericTypes=new byte[]{DataType.DOUBLE,DataType.FLOAT,DataType.LONG,DataType.INTEGER};
	Schema.FieldSchema inputFieldSchema=new Schema.FieldSchema("",DataType.CHARARRAY);
	for (byte type:numbericTypes){
		Schema.FieldSchema castFieldSchema=new Schema.FieldSchema("",type);
		assertTrue(Schema.FieldSchema.castable(castFieldSchema, inputFieldSchema));
	}
}

Source File: LongVAR.java From datafu with Apache License 2.0

4 votes

@Override
public Schema outputSchema(Schema input) {
    return new Schema(new Schema.FieldSchema(null, DataType.DOUBLE));
}

Source File: NEXTUP.java From spork with Apache License 2.0

4 votes

@Override
public Schema outputSchema(Schema input) {
    return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), input), DataType.DOUBLE));
}

Source File: VAR.java From datafu with Apache License 2.0

4 votes

@Override
public Schema outputSchema(Schema input) {
    return new Schema(new Schema.FieldSchema(null, DataType.DOUBLE));
}

Source File: TestPOUserFunc.java From spork with Apache License 2.0

4 votes

@Override
public Schema outputSchema(Schema input) {
          return new Schema(new Schema.FieldSchema(null, DataType.DOUBLE));
}

Source File: IntVAR.java From datafu with Apache License 2.0

4 votes

@Override
public Schema outputSchema(Schema input) {
    return new Schema(new Schema.FieldSchema(null, DataType.DOUBLE));
}

Source File: IntAvg.java From spork with Apache License 2.0

4 votes

@Override
public Schema outputSchema(Schema input) {
    return new Schema(new Schema.FieldSchema(null, DataType.DOUBLE)); 
}

Source File: TestTypeCheckingValidatorNewLP.java From spork with Apache License 2.0

4 votes

@Test
public void testExpressionTypeChecking9() throws Throwable {
    LogicalExpressionPlan plan = new LogicalExpressionPlan();

    TupleFactory tupleFactory = TupleFactory.getInstance();

    ArrayList<Object> innerObjList = new ArrayList<Object>();
    ArrayList<Object> objList = new ArrayList<Object>();

    innerObjList.add("10");
    innerObjList.add("3");
    innerObjList.add(7);
    innerObjList.add("17");

    Tuple innerTuple = tupleFactory.newTuple(innerObjList);

    objList.add("World");
    objList.add(42);
    objList.add(innerTuple);

    Tuple tuple = tupleFactory.newTuple(objList);

    ArrayList<Schema.FieldSchema> innerFss = new ArrayList<Schema.FieldSchema>();
    ArrayList<Schema.FieldSchema> fss = new ArrayList<Schema.FieldSchema>();
    ArrayList<Schema.FieldSchema> castFss = new ArrayList<Schema.FieldSchema>();

    Schema.FieldSchema stringFs = new Schema.FieldSchema(null, DataType.CHARARRAY);
    Schema.FieldSchema intFs = new Schema.FieldSchema(null, DataType.INTEGER);
    Schema.FieldSchema doubleFs = new Schema.FieldSchema(null, DataType.DOUBLE);

    innerFss.add(stringFs);
    innerFss.add(stringFs);
    innerFss.add(intFs);
    innerFss.add(stringFs);

    Schema innerTupleSchema = new Schema(innerFss);

    fss.add(stringFs);
    fss.add(intFs);
    fss.add(new Schema.FieldSchema(null, innerTupleSchema, DataType.TUPLE));

    Schema tupleSchema = new Schema(fss);

    castFss.add(stringFs);
    castFss.add(stringFs);
    castFss.add(doubleFs);
    castFss.add(intFs);

    Schema castSchema = new Schema(castFss);


    ConstantExpression constant1 = new ConstantExpression(plan, innerTuple);
    ConstantExpression constant2 =  new ConstantExpression(plan, tuple);
    CastExpression cast1 = new CastExpression(plan, constant1,
            org.apache.pig.newplan.logical.Util.translateFieldSchema(new FieldSchema(null, castSchema, DataType.TUPLE)));

    EqualExpression equal1 = new EqualExpression(plan, cast1, constant2);
    CompilationMessageCollector collector = new CompilationMessageCollector();
    LogicalRelationalOperator dummyRelOp = createDummyRelOpWithAlias();
    TypeCheckingExpVisitor expTypeChecker = new TypeCheckingExpVisitor(plan, collector, dummyRelOp);
    expTypeChecker.visit();

    printMessageCollector(collector);
    //printTypeGraph(plan);

    if (collector.hasError()) {
        throw new Exception("Error expected");
    }
}

Source File: TypeUtil.java From phoenix with Apache License 2.0

4 votes

/**
 * This method returns the most appropriate PDataType associated with 
 * the incoming Pig type. Note for Pig DataType DATETIME, returns DATE as 
 * inferredSqlType. 
 * 
 * This is later used to make a cast to targetPhoenixType accordingly. See
 * {@link #castPigTypeToPhoenix(Object, byte, PDataType)}
 * 
 * @param obj
 * @return PDataType
 */
public static PDataType getType(Object obj, byte type) {
	if (obj == null) {
		return null;
	}
	PDataType sqlType;

	switch (type) {
	case DataType.BYTEARRAY:
		sqlType = PVarbinary.INSTANCE;
		break;
	case DataType.CHARARRAY:
		sqlType = PVarchar.INSTANCE;
		break;
	case DataType.DOUBLE:
	case DataType.BIGDECIMAL:
		sqlType = PDouble.INSTANCE;
		break;
	case DataType.FLOAT:
		sqlType = PFloat.INSTANCE;
		break;
	case DataType.INTEGER:
		sqlType = PInteger.INSTANCE;
		break;
	case DataType.LONG:
	case DataType.BIGINTEGER:
		sqlType = PLong.INSTANCE;
		break;
	case DataType.BOOLEAN:
		sqlType = PBoolean.INSTANCE;
		break;
	case DataType.DATETIME:
		sqlType = PDate.INSTANCE;
		break;
	case DataType.BYTE:
		sqlType = PTinyint.INSTANCE;
		break;
	default:
		throw new RuntimeException("Unknown type " + obj.getClass().getName()
				+ " passed to PhoenixHBaseStorage");
	}

	return sqlType;

}

Source File: VespaQuerySchema.java From vespa with Apache License 2.0

4 votes

public Tuple buildTuple(int rank, JsonNode hit) {
    Tuple tuple = TupleFactory.getInstance().newTuple();

    for (VespaQuerySchema.AliasTypePair tupleElement : tupleSchema) {
        String alias = tupleElement.getAlias();
        Byte type = DataType.findTypeByName(tupleElement.getType());

        // reserved word
        if ("rank".equals(alias)) {
            tuple.append(rank);
        } else {
            JsonNode field = hit;
            String[] path = alias.split("/"); // move outside
            for (String p : path) {
                field = field.get(p);
                if (field == null) {
                    type = DataType.NULL; // effectively skip field as it is not found
                    break;
                }
            }
            switch (type) {
                case DataType.BOOLEAN:
                    tuple.append(field.asBoolean());
                    break;
                case DataType.INTEGER:
                    tuple.append(field.asInt());
                    break;
                case DataType.LONG:
                    tuple.append(field.asLong());
                    break;
                case DataType.FLOAT:
                case DataType.DOUBLE:
                    tuple.append(field.asDouble());
                    break;
                case DataType.DATETIME:
                    tuple.append(field.asText());
                    break;
                case DataType.CHARARRAY:
                    tuple.append(field.asText());
                    break;
                default:
                    // the rest of the data types are currently not supported
            }
        }
    }
    return tuple;
}

Source File: TezDagBuilder.java From spork with Apache License 2.0

4 votes

private static Class<? extends WritableComparator> getGroupingComparatorForKeyType(byte keyType)
        throws JobCreationException {

    switch (keyType) {
    case DataType.BOOLEAN:
        return PigGroupingBooleanWritableComparator.class;

    case DataType.INTEGER:
        return PigGroupingIntWritableComparator.class;

    case DataType.BIGINTEGER:
        return PigGroupingBigIntegerWritableComparator.class;

    case DataType.BIGDECIMAL:
        return PigGroupingBigDecimalWritableComparator.class;

    case DataType.LONG:
        return PigGroupingLongWritableComparator.class;

    case DataType.FLOAT:
        return PigGroupingFloatWritableComparator.class;

    case DataType.DOUBLE:
        return PigGroupingDoubleWritableComparator.class;

    case DataType.DATETIME:
        return PigGroupingDateTimeWritableComparator.class;

    case DataType.CHARARRAY:
        return PigGroupingCharArrayWritableComparator.class;

    case DataType.BYTEARRAY:
        return PigGroupingDBAWritableComparator.class;

    case DataType.MAP:
        int errCode = 1068;
        String msg = "Using Map as key not supported.";
        throw new JobCreationException(msg, errCode, PigException.INPUT);

    case DataType.TUPLE:
        return PigGroupingTupleWritableComparator.class;

    case DataType.BAG:
        errCode = 1068;
        msg = "Using Bag as key not supported.";
        throw new JobCreationException(msg, errCode, PigException.INPUT);

    default:
        errCode = 2036;
        msg = "Unhandled key type " + DataType.findTypeName(keyType);
        throw new JobCreationException(msg, errCode, PigException.BUG);
    }
}

Java Code Examples for org.apache.pig.data.DataType#DOUBLE