org.apache.pig.data.DataType Java Examples

Lag(Object[] args) throws IOException {
    rowsBehind = 1;
    deflt = null;
    if (args != null) {
        if (args.length >= 1) {
            try {
                rowsBehind = (Integer)args[0];
            } catch (ClassCastException cce) {
                int errCode = 2107; // TODO not sure this is the right one
                String msg = "Lag expected an integer for arg 2 " +
                    " but received " + DataType.findTypeName(args[0]);
                throw new ExecException(msg, errCode, PigException.INPUT);
            }
        }
        if (args.length >= 2) {
            deflt = args[1];
        }
    }
    reset();
}
 

@Test
public void testMapNe() throws ExecException {
    Map map_1 = new HashMap();
    map_1.put("key_1", "value_1");
    Map map_2 = new HashMap();
    map_2.put("key_1", "value_2");

    Tuple tuple_2 = TupleFactory.getInstance().newTuple("item_2");
    ConstantExpression lt = GenPhyOp.exprConst();
    lt.setValue(map_1);
    ConstantExpression rt = GenPhyOp.exprConst();
    rt.setValue(map_2);
    EqualToExpr g = GenPhyOp.compEqualToExpr();
    g.setLhs(lt);
    g.setRhs(rt);
    g.setOperandType(DataType.MAP);
    Result r = g.getNextBoolean();
    assertEquals(POStatus.STATUS_OK, r.returnStatus);
    assertFalse((Boolean)r.result);
}
 

/**
 * {@link RegexExpression} expects CharArray as input
 * Itself always returns Boolean
 * @param rg
 * @throws FrontendException
 */
@Override
public void visit(RegexExpression rg) throws FrontendException {
    // We allow BYTEARRAY to be converted to CHARARRAY
    if (rg.getLhs().getType() == DataType.BYTEARRAY){
        insertCast(rg, DataType.CHARARRAY, rg.getLhs());
    }
    if (rg.getRhs().getType() == DataType.BYTEARRAY){
        insertCast(rg, DataType.CHARARRAY, rg.getRhs());
    }

    // Other than that if it's not CharArray just say goodbye
    if (rg.getLhs().getType() != DataType.CHARARRAY ||
            rg.getRhs().getType() != DataType.CHARARRAY)
    {
        int errCode = 1037;
        String msg = "Operands of Regex can be CharArray only :" + rg;
        msgCollector.collect(msg, MessageType.Error);
        throw new TypeCheckerException(rg, msg, errCode, PigException.INPUT) ;
    }
}
 

@Test
public void testPOBinCondLongWithNull() throws  ExecException, PlanException {
   bag= getBagWithNulls(DataType.LONG);
   TestPoBinCondHelper testHelper= new TestPoBinCondHelper(DataType.LONG, new Long(1L) );

   for (Tuple t : bag) {
       testHelper.getPlan().attachInput(t);

       Long value=null;
       if ( t.get(0)!=null){
           value = (Long) t.get(0);
       }
       Integer dummy = new Integer(0);
       Integer result=(Integer)testHelper.getOperator().getNextInteger().result;
       int expected;
       int actual;
       if ( value!=null ) {
           expected=(value.intValue() == 1)? 1:0 ;
           actual  = result.intValue();
           assertEquals( expected, actual );
       } else {
           assertNull(result);
       }
   }
}
 

/**
 * java level API
 * @param input expects a numeric value to round, a number of digits to keep, and an optional rounding mode.
 * @return output returns a single numeric value, the number with only those digits retained
 */
@Override
public Double exec(Tuple input) throws IOException {
    if (input == null || input.size() < 2)
        return null;

    try {
        Double       num    = (Double)input.get(0);
        Integer      digits = (Integer)input.get(1);
        RoundingMode mode   = (input.size() >= 3) ?
            RoundingMode.valueOf(DataType.toInteger(input.get(2))) : RoundingMode.HALF_EVEN;
        if (num == null) return null;

        BigDecimal bdnum  = BigDecimal.valueOf(num);
        bdnum = bdnum.setScale(digits, mode);
        return bdnum.doubleValue();
    } catch (Exception e){
        throw new IOException("Caught exception processing input row ", e);
    }
}
 

protected Ntile(Object[] args) throws IOException {

            if (args == null || args.length != 1) {
                throw new ExecException(
                    "Ntile args must contain arg describing how to split data, "
                    + "e.g. ntile(4)", 2107, PigException.INPUT);
            }

            try {
                numBuckets = (Integer)args[0];
            } catch (ClassCastException cce) {
                throw new ExecException(
                    "Ntile expected integer argument but received " +
                    DataType.findTypeName(args[0]), 2107, PigException.INPUT);
            }
            reset();

        }
 

public static BlockSchema getWiderSchema(BlockSchema schema1, BlockSchema schema2)
{
    /// require that the schemas are "consistent"
    // schema1.equalsIgnoreNumeric(schema2)) must be satisfied

    ColumnType[] ctypes = new ColumnType[schema1.getNumColumns()];
    for (int i = 0; i < ctypes.length; i++)
    {
        ColumnType type1 = schema1.getColumnType(i);
        ColumnType type2 = schema2.getColumnType(i);

        if (type1.getType().isNumerical() && type2.getType().isNumerical())
        {
            ctypes[i] = new ColumnType(type1.getName(),
                                       com.linkedin.cubert.block.DataType.getWiderType(type1.getType(), type2.getType()));
        }
        else
        {
            ctypes[i] = type1;
        }
    }

    return new BlockSchema(ctypes);

}
 

static public Tuple buildBinTuple(final Object... args) throws IOException {
    return TupleFactory.getInstance().newTuple(Lists.transform(
            Lists.newArrayList(args), new Function<Object, DataByteArray>() {
                @Override
                public DataByteArray apply(Object o) {
                    if (o == null) {
                        return null;
                    }
                    try {
                        return new DataByteArray(DataType.toBytes(o));
                    } catch (ExecException e) {
                        return null;
                    }
                }
            }));
}
 

@Test
public void testTupleEq() throws ExecException {
    Tuple tuple_1 = TupleFactory.getInstance().newTuple("item_1");
    Tuple tuple_2 = TupleFactory.getInstance().newTuple("item_1");
    ConstantExpression lt = GenPhyOp.exprConst();
    lt.setValue(tuple_1);
    ConstantExpression rt = GenPhyOp.exprConst();
    rt.setValue(tuple_2);
    EqualToExpr g = GenPhyOp.compEqualToExpr();
    g.setLhs(lt);
    g.setRhs(rt);
    g.setOperandType(DataType.TUPLE);
    Result r = g.getNextBoolean();
    assertEquals(POStatus.STATUS_OK, r.returnStatus);
    assertTrue((Boolean)r.result);
}
 

/**
 * java level API
 * @param input expects a tuple containing two numeric DataAtom value
 * @param output returns a single numeric DataAtom value, which is 
 * fistArgument pow(2,secondArgument) rounded as if performed by a 
 * single correctly rounded floating-point multiply to a member of 
 * the double value set.
 */
public Double exec(Tuple input) throws IOException {
       if (input == null || input.size() < 2)
           return null;
       if (input.get(0) == null || input.get(1) == null) {
           return null;
       }
	try{
		Double first = DataType.toDouble(input.get(0));
		Integer second = DataType.toInteger(input.get(1));

           return Math.scalb(first, second);
	} catch (NumberFormatException nfe){
           System.err.println("Failed to process input; error - " + nfe.getMessage());
           return null;
       } catch(Exception e){
           throw new IOException("Caught exception in MAX.Initial", e);
       }
}
 

/**
 * Test one-level Pig Schema: bag without tuple field
 */
@Test
public void testResourceSchemaWithInvalidPigSchema2() 
throws FrontendException {
    String [] aliases ={"f1"};
    byte[] types = {DataType.INTEGER};
    Schema level0 = TypeCheckingTestUtil.genFlatSchema(
            aliases,types);
    Schema.FieldSchema fld0 = 
        new Schema.FieldSchema("f0", level0, DataType.BAG);
    Schema level1 = new Schema(fld0);
    try {
        Schema.getPigSchema(new ResourceSchema(level1));
        Assert.fail();
    } catch (FrontendException e) {
        assertTrue(e.getErrorCode()==2218);
    }
}
 

@Override
public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
    List<FuncSpec> funcList = new ArrayList<FuncSpec>();

    Schema s_dbl_2 = new Schema();
    s_dbl_2.add(new Schema.FieldSchema(null, DataType.DOUBLE));
    s_dbl_2.add(new Schema.FieldSchema(null, DataType.INTEGER));

    Schema s_dbl_3 = new Schema();
    s_dbl_3.add(new Schema.FieldSchema(null, DataType.DOUBLE));
    s_dbl_3.add(new Schema.FieldSchema(null, DataType.INTEGER));
    s_dbl_3.add(new Schema.FieldSchema(null, DataType.INTEGER));

    funcList.add(new FuncSpec(this.getClass().getName(), s_dbl_2));
    funcList.add(new FuncSpec(this.getClass().getName(), s_dbl_3));

    return funcList;
}
 

@Test
public void testIntegerLt() throws Exception {
    ConstantExpression lt = GenPhyOp.exprConst();
    lt.setValue(new Integer(0));
    ConstantExpression rt = GenPhyOp.exprConst();
    rt.setValue(new Integer(1));
    GTOrEqualToExpr g = GenPhyOp.compGTOrEqualToExpr();
    g.setLhs(lt);
    g.setRhs(rt);
    g.setOperandType(DataType.INTEGER);
    Result r = g.getNextBoolean();
    assertEquals(POStatus.STATUS_OK, r.returnStatus);
    assertFalse((Boolean)r.result);
}
 

private Type convertWithName(FieldSchema fieldSchema, String name) {
  try {
    switch (fieldSchema.type) {
    case DataType.BAG:
      return convertBag(name, fieldSchema);
    case DataType.TUPLE:
      return convertTuple(name, fieldSchema, Repetition.OPTIONAL);
    case DataType.MAP:
      return convertMap(name, fieldSchema);
    case DataType.BOOLEAN:
      return primitive(name, PrimitiveTypeName.BOOLEAN);
    case DataType.CHARARRAY:
      return primitive(name, PrimitiveTypeName.BINARY, stringType());
    case DataType.INTEGER:
      return primitive(name, PrimitiveTypeName.INT32);
    case DataType.LONG:
      return primitive(name, PrimitiveTypeName.INT64);
    case DataType.FLOAT:
      return primitive(name, PrimitiveTypeName.FLOAT);
    case DataType.DOUBLE:
      return primitive(name, PrimitiveTypeName.DOUBLE);
    case DataType.DATETIME:
      throw new UnsupportedOperationException();
    case DataType.BYTEARRAY:
      return primitive(name, PrimitiveTypeName.BINARY);
    default:
      throw new SchemaConversionException("Unknown type " + fieldSchema.type + " " + DataType.findTypeName(fieldSchema.type));
    }
  } catch (FrontendException e) {
    throw new SchemaConversionException("can't convert "+fieldSchema, e);
  }
}
 

@Override
public LogicalSchema getSchema() throws FrontendException {
    if (schema!=null)
        return schema;
    
    if (prj.findReferent().getSchema()!=null && prj.getFieldSchema()!=null) {
        if (prj.getFieldSchema().type==DataType.BAG) {
            sourceIsBag = true;
            alias = prj.getFieldSchema().alias;
            if (prj.getFieldSchema().schema!=null) {
                LogicalFieldSchema tupleSchema = prj.getFieldSchema().schema.getField(0);
                if (tupleSchema!=null && tupleSchema.schema!=null) {
                    schema = new LogicalSchema();
                    for (int i=0;i<tupleSchema.schema.size();i++)
                        schema.addField(tupleSchema.schema.getField(i));
                }
            }
        }
        else {
            schema = new LogicalSchema();
            schema.addField(prj.getFieldSchema());
        }
    } else if (!prj.isRangeOrStarProject()) {
        schema = new LogicalSchema();
        schema.addField(new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
    }
    return schema;
}
 

public DateTime exec(Tuple input) throws IOException {
    if (input == null || input.size() < 1 || input.get(0) == null) {
        return null;
    }
    String dtStr = DataType.toString(input.get(0));
    return ToDate.extractDateTime(dtStr);
}
 

@Test
public void testLongEq() throws Exception {
    ConstantExpression lt = GenPhyOp.exprConst();
    lt.setValue(new Long(1L));
    ConstantExpression rt = GenPhyOp.exprConst();
    rt.setValue(new Long(1L));
    EqualToExpr g = GenPhyOp.compEqualToExpr();
    g.setLhs(lt);
    g.setRhs(rt);
    g.setOperandType(DataType.LONG);
    Result r = g.getNextBoolean();
    assertEquals(POStatus.STATUS_OK, r.returnStatus);
    assertTrue((Boolean)r.result);
}
 

/** check whether it is just a wrapped tuple */
public static boolean isTupleWrapper(ResourceFieldSchema pigSchema) {
    Boolean status = false;
    if(pigSchema.getType() == DataType.TUPLE)
        if(pigSchema.getName() != null)
            if(pigSchema.getName().equals(AvroStorageUtils.PIG_TUPLE_WRAPPER))
                status = true;
    return status;
}
 

@Test
public void testExpressionTypeCheckingFail4() throws Throwable {
    LogicalExpressionPlan plan = new LogicalExpressionPlan();
    ConstantExpression constant1 = new ConstantExpression(plan, 10);
    ConstantExpression constant2 =  new ConstantExpression(plan, 20D);
    ConstantExpression constant3 =  new ConstantExpression(plan, "123");

    DivideExpression div1 = new DivideExpression(plan, constant1, constant2);
    CastExpression cast1 = new CastExpression(plan,  constant3, createFS(DataType.BYTEARRAY));
    NotEqualExpression notequal1 = new NotEqualExpression(plan, div1, cast1);

    CompilationMessageCollector collector = new CompilationMessageCollector();
    TypeCheckingExpVisitor expTypeChecker = new TypeCheckingExpVisitor(plan, collector, null);

    try {
        expTypeChecker.visit();
        fail("Exception expected");
    } catch (TypeCheckerException pve) {
        // good
    }
    printMessageCollector(collector);
    //printTypeGraph(plan);

    if (!collector.hasError()) {
        fail("Error during type checking");
    }
}
 

/** wrap a pig schema as tuple */
public static ResourceFieldSchema wrapAsTuple(ResourceFieldSchema subFieldSchema) throws IOException {
    ResourceSchema listSchema = new ResourceSchema();
    listSchema.setFields(new ResourceFieldSchema[] { subFieldSchema });

    ResourceFieldSchema tupleWrapper = new ResourceFieldSchema();
    tupleWrapper.setType(DataType.TUPLE);
    tupleWrapper.setName(PIG_TUPLE_WRAPPER);
    tupleWrapper.setSchema(listSchema);

    return tupleWrapper;
}
 

private boolean atLeastOneCastNeeded(LogicalSchema determinedSchema, LogicalSchema s) {
    for (int i = 0; i < s.size(); i++) {
        LogicalSchema.LogicalFieldSchema fs = s.getField(i);
        if (fs.type != DataType.BYTEARRAY && (determinedSchema == null || (!fs.isEqual(determinedSchema.getField(i))))) {
            // we have to cast this field from the default BYTEARRAY type to
            // whatever the user specified in the 'AS' clause of the LOAD
            // statement (the fs.type).
            return true;
        }
    }
    return false;
}
 

@Test
public void testFloatGt() throws Exception {
    ConstantExpression lt = GenPhyOp.exprConst();
    lt.setValue(new Float(1.0f));
    ConstantExpression rt = GenPhyOp.exprConst();
    rt.setValue(new Float(0.0f));
    GreaterThanExpr g = GenPhyOp.compGreaterThanExpr();
    g.setLhs(lt);
    g.setRhs(rt);
    g.setOperandType(DataType.FLOAT);
    Result r = g.getNextBoolean();
    assertEquals(POStatus.STATUS_OK, r.returnStatus);
    assertTrue((Boolean)r.result);
}
 

@Test(expected = ExecException.class)
public void testIntegerConversionErr1() throws Exception {
    DataByteArray ba = new DataByteArray("hello world");
    try {
        DataType.toInteger(ba);
    } catch (ExecException ee) {
        assertEquals(1074, ee.getErrorCode());
        throw ee;
    }
}
 

private PhysicalPlan constructPlan(POCast op) throws IOException {
       LoadFunc load = new TestLoader();
       op.setFuncSpec(new FuncSpec(load.getClass().getName()));
       POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
       PhysicalPlan plan = new PhysicalPlan();
       plan.add(prj);
       plan.add(op);
       plan.connect(prj, op);
       prj.setResultType(DataType.BYTEARRAY);
       return plan;
}
 

@Test
public void testDataByteArrayNe() throws Exception {
    ConstantExpression lt = GenPhyOp.exprConst();
    lt.setValue(new DataByteArray("a"));
    ConstantExpression rt = GenPhyOp.exprConst();
    rt.setValue(new DataByteArray("b"));
    NotEqualToExpr g = GenPhyOp.compNotEqualToExpr();
    g.setLhs(lt);
    g.setRhs(rt);
    g.setOperandType(DataType.BYTEARRAY);
    Result r = g.getNextBoolean();
    assertEquals(POStatus.STATUS_OK, r.returnStatus);
    assertTrue((Boolean)r.result);
}
 

public TOBAG(String bagColName, String tupleColName, String fieldType) {

        if ( bagColName== null || tupleColName == null  || fieldType == null)  {
	    throw new RuntimeException("The bagColName  and fieldType cannot be null");
        }

    	this.bagColName   = bagColName;	
    	this.tupleColName = tupleColName;	

        if ( fieldType.equalsIgnoreCase( "CHARARRAY" )){ 
             this.fieldType = DataType.CHARARRAY;

        } else if ( fieldType.equalsIgnoreCase( "DOUBLE" )){ 
            this.fieldType = DataType.DOUBLE;

        } else if ( fieldType.equalsIgnoreCase( "FLOAT" )){ 
            this.fieldType = DataType.FLOAT; 

        } else if ( fieldType.equalsIgnoreCase( "BOOLEAN" )){ 
            this.fieldType = DataType.BOOLEAN; 

        } else if ( fieldType.equalsIgnoreCase( "INTEGER" )){ 
            this.fieldType = DataType.INTEGER;

        } else if ( fieldType.equalsIgnoreCase( "LONG" )){ 
            this.fieldType = DataType.LONG; 

        } else if ( fieldType.equalsIgnoreCase( "MAP" )){ 
            this.fieldType = DataType.MAP; 
        } else {
	    throw new RuntimeException("This type"+ fieldType +"is not supported in TOBAG");
        }

    }
 

@Override
public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
    List<FuncSpec> funcList = new ArrayList<FuncSpec>();
    funcList.add(new FuncSpec(this.getClass().getName(),
        new Schema(new FieldSchema(null, DataType.CHARARRAY))));
    funcList.add(new FuncSpec(IntTest.class.getName(),
        new Schema(new FieldSchema(null, DataType.INTEGER))));
    return funcList;
}
 

@Override
public Schema outputSchema(Schema input)
{
  try {
    return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), input),
            input.getField(0).schema, DataType.BAG));
  }
  catch (FrontendException e) {
    return null;
  }
}
 

@Test
public void testFloatGt() throws Exception {
    ConstantExpression lt = GenPhyOp.exprConst();
    lt.setValue(new Float(1.0f));
    ConstantExpression rt = GenPhyOp.exprConst();
    rt.setValue(new Float(0.0f));
    LessThanExpr g = GenPhyOp.compLessThanExpr();
    g.setLhs(lt);
    g.setRhs(rt);
    g.setOperandType(DataType.FLOAT);
    Result r = g.getNextBoolean();
    assertEquals(POStatus.STATUS_OK, r.returnStatus);
    assertFalse((Boolean)r.result);
}
 

/**
 * This method encodes a value with Phoenix data type. It begins
 * with checking whether an object is BINARY and makes a call to
 * {@link #castBytes(Object, PDataType)} to convery bytes to
 * targetPhoenixType
 * 
 * @param o
 * @param targetPhoenixType
 * @return Object
 */
public static Object castPigTypeToPhoenix(Object o, byte objectType, PDataType targetPhoenixType) {
	PDataType inferredPType = getType(o, objectType);
	
	if(inferredPType == null) {
		return null;
	}

	if(inferredPType == PVarbinary.INSTANCE) {
		try {
			o = castBytes(o, targetPhoenixType);
			if(targetPhoenixType != PVarbinary.INSTANCE && targetPhoenixType != PBinary.INSTANCE) {
				inferredPType = getType(o, DataType.findType(o));	
			}
		} catch (IOException e) {
			throw new RuntimeException("Error while casting bytes for object " +o);
		}
	}
	if(inferredPType == PDate.INSTANCE) {
		int inferredSqlType = targetPhoenixType.getSqlType();

		if(inferredSqlType == Types.DATE) {
			return new Date(((DateTime)o).getMillis());
		} 
		if(inferredSqlType == Types.TIME) {
			return new Time(((DateTime)o).getMillis());
		}
		if(inferredSqlType == Types.TIMESTAMP) {
			return new Timestamp(((DateTime)o).getMillis());
		}
	}
	
	if (targetPhoenixType == inferredPType || inferredPType.isCoercibleTo(targetPhoenixType)) {
		return inferredPType.toObject(o, targetPhoenixType);
	}
	
	throw new RuntimeException(o.getClass().getName()
			+ " cannot be coerced to "+targetPhoenixType.toString());
}