Java Code Examples for org.apache.pig.PigException#BUG

/**
 * Compiles a split operator. The logic is to
 * close the split job by replacing the split oper by
 * a store and creating a new Map MRoper and return
 * that as the current MROper to which other operators
 * would be compiled into. The new MROper would be connected
 * to the split job by load-store. Also add the split oper
 * to the splitsSeen map.
 * @param op - The split operator
 * @throws VisitorException
 */
@Override
public void visitSplit(POSplit op) throws VisitorException{
    try{
        FileSpec fSpec = op.getSplitStore();
        MapReduceOper mro = endSingleInputPlanWithStr(fSpec);
        mro.setSplitter(true);
        splitsSeen.put(op.getOperatorKey(), mro);
        curMROp = startNew(fSpec, mro);
        phyToMROpMap.put(op, curMROp);
    }catch(Exception e){
        int errCode = 2034;
        String msg = "Error compiling operator " + op.getClass().getSimpleName();
        throw new MRCompilerException(msg, errCode, PigException.BUG, e);
    }
}
 

@Override
public void accumulate(Tuple b) throws IOException {
    try {
        DataBag bag = (DataBag)b.get(0);
        Iterator it = bag.iterator();
        while (it.hasNext()){
            Tuple t = (Tuple)it.next();
            if (t != null && t.size() > 0 && t.get(0) != null) {
                intermediateCount += 1;
            }
        }
    } catch (ExecException ee) {
        throw ee;
    } catch (Exception e) {
        int errCode = 2106;
        String msg = "Error while computing min in " + this.getClass().getSimpleName();
        throw new ExecException(msg, errCode, PigException.BUG, e);           
    }
}
 

@Override
public void accumulate(Tuple b) throws IOException {
    try {
        String curMax = max(b);
        if (curMax == null) {
            return;
        }
        // check if it lexicographically follows curMax
        if (intermediateMax == null || intermediateMax.compareTo(curMax) > 0) {
            intermediateMax = curMax;
        }            

    } catch (ExecException ee) {
        throw ee;
    } catch (Exception e) {
        int errCode = 2106;
        String msg = "Error while computing max in " + this.getClass().getSimpleName();
        throw new ExecException(msg, errCode, PigException.BUG, e);           
    }
}
 

protected static BigInteger doTupleWork(Tuple input, KnownOpProvider opProvider) throws ExecException {
    DataBag values = (DataBag)input.get(0);
    // if we were handed an empty bag, return NULL
    // this is in compliance with SQL standard
    if(values.size() == 0) {
        return null;
    }
    BigInteger sofar = AlgebraicBigIntegerMathBase.getSeed(opProvider.getOp());
    boolean sawNonNull = false;
    for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
        Tuple t = it.next();
        try {
            Number n = (Number)(t.get(0));
            if (n == null) continue;
            BigInteger d = (BigInteger) n;
            sawNonNull = true;
            sofar = doWork(sofar, d, opProvider.getOp());
        } catch(RuntimeException exp) {
            int errCode = 2103;
            throw new ExecException("Problem doing work on BigInteger", errCode, PigException.BUG, exp);
        }
    }
    return sawNonNull ? sofar : null;
}
 

@Override
public void accumulate(Tuple b) throws IOException {
    try {
        Double sum = sum(b);
        if(sum == null) {
            return;
        }
        
        Double sumSquare = sumSquare(b);
        if(sumSquare == null) {
            return;
        }
        
        // set default values
        if (intermediateSum == null || intermediateCount == null) {
            intermediateSumSquare = 0.0;
            intermediateSum = 0.0;
            intermediateCount = (long) 0;
        }
        
        long count = (Long)count(b);

        if (count > 0) {
            intermediateCount += count;
            intermediateSum += sum;
            intermediateSumSquare += sumSquare;
        }
    } catch (ExecException ee) {
        throw ee;
    } catch (Exception e) {
        int errCode = 2106;
        String msg = "Error while computing variance in " + this.getClass().getSimpleName();
        throw new ExecException(msg, errCode, PigException.BUG, e);
    }
}
 

@Override
public Tuple exec(Tuple input) throws IOException {
    try {
        DataBag b = (DataBag)input.get(0);
        return combine(b);
    } catch (ExecException ee) {
        throw ee;
    } catch (Exception e) {
        int errCode = 2106;
        String msg = "Error while computing average in " + this.getClass().getSimpleName();
        throw new ExecException(msg, errCode, PigException.BUG, e);           
    }
}
 

/**
 * Compares a tuple with two fields. Emits any differences.
 * @param input a tuple with exactly two fields.
 * @throws IOException if there are not exactly two fields in a tuple
 */
@Override
public DataBag exec(Tuple input) throws IOException {
    if (input.size() != 2) {
        int errCode = 2107;
        String msg = "DIFF expected two inputs but received " + input.size() + " inputs.";
        throw new ExecException(msg, errCode, PigException.BUG);
    }
    try {
        DataBag output = mBagFactory.newDefaultBag();
        Object o1 = input.get(0);
        if (o1 instanceof DataBag) {
            DataBag bag1 = (DataBag)o1;
            DataBag bag2 = (DataBag)input.get(1);
            computeDiff(bag1, bag2, output);
        } else {
            Object d1 = input.get(0);
            Object d2 = input.get(1);
            if (!d1.equals(d2)) {
                output.add(mTupleFactory.newTuple(d1));
                output.add(mTupleFactory.newTuple(d2));
            }
        }
        return output;
    } catch (ExecException ee) {
        throw ee;
    }
}
 

public boolean processOnePackageOutput(Context oc)
        throws IOException, InterruptedException {

    Result res = pack.getNextTuple();
    if(res.returnStatus==POStatus.STATUS_OK){
        Tuple packRes = (Tuple)res.result;

        if(rp.isEmpty()){
            oc.write(null, packRes);
            return false;
        }
        for (int i = 0; i < roots.length; i++) {
            roots[i].attachInput(packRes);
        }
        runPipeline(leaf);

    }

    if(res.returnStatus==POStatus.STATUS_NULL) {
        return false;
    }

    if(res.returnStatus==POStatus.STATUS_ERR){
        int errCode = 2093;
        String msg = "Encountered error in package operator while processing group.";
        throw new ExecException(msg, errCode, PigException.BUG);
    }

    if(res.returnStatus==POStatus.STATUS_EOP) {
        return true;
    }

    return false;

}
 

private Result getResult(PhysicalPlan plan, byte resultType) throws ExecException {
	ExpressionOperator Op = (ExpressionOperator) plan.getLeaves().get(0);
	Result res = null;

	switch (resultType) {
          case DataType.BYTEARRAY:
          case DataType.CHARARRAY:
          case DataType.DOUBLE:
          case DataType.FLOAT:
          case DataType.BOOLEAN:
          case DataType.INTEGER:
          case DataType.LONG:
          case DataType.BIGINTEGER:
          case DataType.BIGDECIMAL:
          case DataType.DATETIME:
          case DataType.TUPLE:
              res = Op.getNext(resultType);
              break;

          default: {
              int errCode = 2082;
              String msg = "Did not expect result of type: " +
                      DataType.findTypeName(resultType);
                  throw new ExecException(msg, errCode, PigException.BUG);
          }

          }
	return res;
}
 

@Override
public Tuple exec(Tuple input) throws IOException {
    try {
        Tuple t = mTupleFactory.newTuple(2);
        // input is a bag with one tuple containing
        // the column we are trying to avg on
        DataBag bg = (DataBag) input.get(0);
        Long l = null;
        if(bg.iterator().hasNext()) {
            Tuple tp = bg.iterator().next();
            l = (Long)(tp.get(0));
        }
        t.set(0, l);
        if (l != null)
            t.set(1, 1L);
        else
            t.set(1, 0L);
        return t;
    } catch (ExecException ee) {
        throw ee;
    } catch (Exception e) {
        int errCode = 2106;
        String msg = "Error while computing average in " + this.getClass().getSimpleName();
        throw new ExecException(msg, errCode, PigException.BUG, e);           
    }
        
}
 

public TezCompiler(PhysicalPlan plan, PigContext pigContext)
        throws TezCompilerException {
    super(plan, new DepthFirstWalker<PhysicalOperator, PhysicalPlan>(plan));
    this.plan = plan;
    this.pigContext = pigContext;

    pigProperties = pigContext.getProperties();
    splitsSeen = Maps.newHashMap();
    tezPlan = new TezOperPlan();
    nig = NodeIdGenerator.getGenerator();
    udfFinder = new UDFFinder();
    List<PhysicalOperator> roots = plan.getRoots();
    if((roots == null) || (roots.size() <= 0)) {
        int errCode = 2053;
        String msg = "Internal error. Did not find roots in the physical plan.";
        throw new TezCompilerException(msg, errCode, PigException.BUG);
    }
    scope = roots.get(0).getOperatorKey().getScope();
    localRearrangeFactory = new POLocalRearrangeTezFactory(scope, nig);
    phyToTezOpMap = Maps.newHashMap();

    fileConcatenationThreshold = Integer.parseInt(pigProperties
            .getProperty(FILE_CONCATENATION_THRESHOLD, "100"));
    optimisticFileConcatenation = pigProperties.getProperty(
            OPTIMISTIC_FILE_CONCATENATION, "false").equals("true");
    LOG.info("File concatenation threshold: " + fileConcatenationThreshold
            + " optimistic? " + optimisticFileConcatenation);
}
 

@Override
public void visitFilter(POFilter op) throws VisitorException{
    try{
        nonBlocking(op);
        processUDFs(op.getPlan());
        phyToMROpMap.put(op, curMROp);
    }catch(Exception e){
        int errCode = 2034;
        String msg = "Error compiling operator " + op.getClass().getSimpleName();
        throw new MRCompilerException(msg, errCode, PigException.BUG, e);
    }
}
 

private void runPipeline(POStore posStore) throws IOException {
    while (true) {
        Result res = posStore.getNextTuple();
        if (res.returnStatus == POStatus.STATUS_OK)
            continue;

        if (res.returnStatus == POStatus.STATUS_EOP) {
            posStore.tearDown();
            return;
        }

        if (res.returnStatus == POStatus.STATUS_NULL)
            continue;

        if(res.returnStatus==POStatus.STATUS_ERR){
            String errMsg;
            if(res.result != null) {
                errMsg = "Fetch failed. Couldn't retrieve result: " + res.result;
            } else {
                errMsg = "Fetch failed. Couldn't retrieve result";
            }
            int errCode = 2088;
            ExecException ee = new ExecException(errMsg, errCode, PigException.BUG);
            throw ee;
        }
    }
}
 

@Override
public void accumulate(Tuple b) throws IOException {
    try {
        Long sum = sum(b);
        if(sum == null) {
            return;
        }
        
        Long sumSquare = sumSquare(b);
        if(sumSquare == null) {
            return;
        }
        
        // set default values
        if (intermediateSum == null || intermediateCount == null) {
            intermediateSumSquare = (long) 0;
            intermediateSum = (long) 0;
            intermediateCount = (long) 0;
        }
        
        long count = (Long)count(b);

        if (count > 0) {
            intermediateCount += count;
            intermediateSum += sum;
            intermediateSumSquare += sumSquare;
        }
    } catch (ExecException ee) {
        throw ee;
    } catch (Exception e) {
        int errCode = 2106;
        String msg = "Error while computing variance in " + this.getClass().getSimpleName();
        throw new ExecException(msg, errCode, PigException.BUG, e);
    }
}
 

@Override
public void visitCounter(POCounter op) throws VisitorException {
    // Refer visitRank(PORank) for more details
    try{
        POCounterTez counterTez = new POCounterTez(op);
        nonBlocking(counterTez);
        phyToTezOpMap.put(op, curTezOp);
    } catch (Exception e) {
        int errCode = 2034;
        String msg = "Error compiling operator " + op.getClass().getSimpleName();
        throw new TezCompilerException(msg, errCode, PigException.BUG, e);
    }
}
 

private Object extractKeysFromTuple(Result inp, int lrIdx) throws ExecException{

        //Separate Key & Value of input using corresponding LR operator
        POLocalRearrange lr = LRs[lrIdx];
        lr.attachInput((Tuple)inp.result);
        Result lrOut = lr.getNextTuple();
        lr.detachInput();
        if(lrOut.returnStatus!=POStatus.STATUS_OK){
            int errCode = 2167;
            String errMsg = "LocalRearrange used to extract keys from tuple isn't configured correctly";
            throw new ExecException(errMsg,errCode,PigException.BUG);
        } 
          
        return ((Tuple) lrOut.result).get(1);
    }
 

@Override
public Long exec(Tuple input) throws IOException {
    try {
       return input.get(0) == null ? null : Long.valueOf(1);
    } catch (ExecException exp) {
        throw exp;
    } catch (Exception e) {
        int errCode = 2106;
        String msg = "Error while computing size in " + this.getClass().getSimpleName();
        throw new ExecException(msg, errCode, PigException.BUG, e);           
    }
}
 

public byte[] toBytes(Double d) throws IOException {
    int errCode = 2109;
    String msg = "TextLoader does not support conversion from Double.";
    throw new ExecException(msg, errCode, PigException.BUG);
}
 

@Override
public void visit(LOForEach foreach) throws FrontendException{
    LogicalPlan innerPlan = foreach.getInnerPlan();

    //visit the inner plan first
    PlanWalker newWalker = currentWalker.spawnChildWalker(innerPlan);
    pushWalker(newWalker);
    currentWalker.walk(this);
    popWalker();

    //get the LOGenerate
    List<Operator> feOutputs = innerPlan.getSinks();
    LOGenerate gen = null;
    for( Operator op  : feOutputs){
        if(op instanceof LOGenerate){
            if(gen != null){
                String msg = "Expected single LOGenerate output in innerplan of foreach";
                throw new VisitorException(foreach,
                        msg,
                        2266,
                        PigException.BUG
                );
            }
            gen = (LOGenerate) op;
        }
    }

    List<Operator> loGenPreds = innerPlan.getPredecessors(gen);

    if(loGenPreds == null){
        // there are no LOInnerLoads , must be working on just constants
        // no project-star expansion to be done
        return;
    }

    //get mapping of LOGenerate predecessor current position to object
    Map<Integer, LogicalRelationalOperator> oldPos2Rel =
        new HashMap<Integer, LogicalRelationalOperator>();
    
    for(int i=0; i<loGenPreds.size(); i++){
        oldPos2Rel.put(i, (LogicalRelationalOperator) loGenPreds.get(i));
    }

    //store mapping between the projection in inner plans of
    // of LOGenerate to the input relation object
    Map<ProjectExpression, LogicalRelationalOperator> proj2InpRel =
        new HashMap<ProjectExpression, LogicalRelationalOperator>();
    
    List<LOInnerLoad> expandedInLoads = new ArrayList<LOInnerLoad>();
    
    //visit each expression plan, and expand the projects in the udf
    for( OperatorPlan plan : gen.getOutputPlans()){
        ProjExpanderForForeach projExpander = new ProjExpanderForForeach(
                plan,
                gen,
                oldPos2Rel,
                proj2InpRel,
                foreach,
                expandedInLoads
        );
        projExpander.visit();
    }
    
    //remove the LOInnerLoads that have been expanded
    for(LOInnerLoad inLoad : expandedInLoads){
        innerPlan.disconnect(inLoad, gen);
        innerPlan.remove(inLoad);
    }
    
    //reset the input relation position in the projects
    //get mapping of LoGenerate input relation to current position
    Map<LogicalRelationalOperator, Integer> rel2pos = new HashMap<LogicalRelationalOperator, Integer>();
    List<Operator> newGenPreds = innerPlan.getPredecessors(gen);
    int numNewGenPreds = 0;
    if(newGenPreds != null)
        numNewGenPreds = newGenPreds.size();
        
    for(int i=0; i<numNewGenPreds; i++){
        rel2pos.put((LogicalRelationalOperator) newGenPreds.get(i),i);
    }
    
    //correct the input num for projects
    for(Entry<ProjectExpression, LogicalRelationalOperator> projAndInp : proj2InpRel.entrySet()){
       ProjectExpression proj = projAndInp.getKey();
       LogicalRelationalOperator rel = projAndInp.getValue();
       proj.setInputNum(rel2pos.get(rel));
    }
}
 

private POForEach compileFE4Flattening(boolean[] innerFlags,String scope,
        int parallel, String alias, SourceLocation location, List<Operator> inputs)
            throws FrontendException {

    List<PhysicalPlan> fePlans = new ArrayList<PhysicalPlan>();
    List<Boolean> flattenLst = new ArrayList<Boolean>();
    POForEach fe;
    try{
        for(int i=0;i< inputs.size();i++){
            PhysicalPlan fep1 = new PhysicalPlan();
            POProject feproj1 = new POProject(new OperatorKey(scope, nodeGen.getNextNodeId(scope)),
                    parallel, i+1); //i+1 since the first column is the "group" field
            feproj1.addOriginalLocation(alias, location);
            feproj1.setResultType(DataType.BAG);
            feproj1.setOverloaded(false);
            fep1.add(feproj1);
            fePlans.add(fep1);
            // the parser would have marked the side
            // where we need to keep empty bags on
            // non matched as outer (innerFlags[i] would be
            // false)
            if(!(innerFlags[i])) {
                Operator joinInput = inputs.get(i);
                // for outer join add a bincond
                // which will project nulls when bag is
                // empty
                updateWithEmptyBagCheck(fep1, joinInput);
            }
            flattenLst.add(true);
        }

        fe = new POForEach(new OperatorKey(scope, nodeGen.getNextNodeId(scope)),
                parallel, fePlans, flattenLst );
        fe.addOriginalLocation(alias, location);

    }catch (PlanException e1) {
        int errCode = 2015;
        String msg = "Invalid physical operators in the physical plan" ;
        throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e1);
    }
    return fe;
}