Java Code Examples for org.apache.pig.newplan.logical.relational.LogicalSchema#getFields()

/**
 * This function translates the new LogicalSchema into old Schema format required
 * by PhysicalOperators
 * @param schema LogicalSchema to be converted to Schema
 * @return Schema that is converted from LogicalSchema
 * @throws FrontendException
 */
public static Schema translateSchema(LogicalSchema schema) {
    if (schema == null) {
        return null;
    }

    Schema s2 = new Schema();
    List<LogicalSchema.LogicalFieldSchema> ll = schema.getFields();
    for (LogicalSchema.LogicalFieldSchema f: ll) {
        Schema.FieldSchema f2 = null;
        try {
            f2 = new Schema.FieldSchema(f.alias, translateSchema(f.schema), f.type);
            f2.canonicalName = ((Long)f.uid).toString();
            s2.add(f2);
        } catch (FrontendException e) {
        }
    }

    return s2;
}
 

private Set<Long> getAllUids( LogicalSchema schema ) {
    Set<Long> uids = new HashSet<Long>();

    if( schema == null ) {
        return uids;
    }

    for( LogicalFieldSchema field : schema.getFields() ) {
        if( ( field.type == DataType.TUPLE || field.type == DataType.BAG )
                && field.schema != null ) {
           uids.addAll( getAllUids( field.schema ) );
        }
        uids.add( field.uid );
    }
    return uids;
}
 

private void collectUids(LogicalRelationalOperator currentOp, LogicalExpressionPlan exp, Set<Long> uids) throws FrontendException {
    List<Operator> ll = exp.getSinks();
    for(Operator op: ll) {
        if (op instanceof ProjectExpression) {
            if (!((ProjectExpression)op).isRangeOrStarProject()) {
                long uid = ((ProjectExpression)op).getFieldSchema().uid;
                uids.add(uid);
            } else {
                LogicalRelationalOperator ref = ((ProjectExpression)op).findReferent();
                LogicalSchema s = ref.getSchema();
                if (s == null) {
                    throw new SchemaNotDefinedException("Schema not defined for " + ref.getAlias());
                }
                for(LogicalFieldSchema f: s.getFields()) {
                    uids.add(f.uid);
                }
            }
        }
    }
}
 

/**
 * checks if a relational operator contains all of the specified uids
 * @param op LogicalRelational operator that should contain the uid
 * @param uids Uids to check for
 * @return true if given LogicalRelationalOperator has all the given uids
 */
private boolean hasAll(LogicalRelationalOperator op, Pair<List<Long>,
        List<Byte>> uidWithTypes) throws FrontendException {
    LogicalSchema schema = op.getSchema();

    if (schema==null)
        return false;

    List<Long> uids = uidWithTypes.first;
    List<Byte> types = uidWithTypes.second;

    for (int i=0;i<uids.size();i++) {
        boolean found = false;
        for (LogicalSchema.LogicalFieldSchema fs : schema.getFields()) {
            if (fs.uid==uids.get(i) && fs.type==types.get(i))
                found = true;
        }
        if (!found)
            return false;
    }
    return true;
}
 

/**
 * This function checks if the schema has a map.
 * We dont check for a nested structure.
 * @param schema Schema to be checked
 * @return true if it has a map, else false
 * @throws NullPointerException incase Schema is null
 */
private boolean hasMap(LogicalSchema schema ) {
    for( LogicalFieldSchema field : schema.getFields() ) {
        if( field.type == DataType.MAP ) {
            return true;
        }
    }
    return false;
}
 

/**
 * This function returns a set of Uids corresponding to
 * map datatype in the first level of this schema
 * @param schema Schema having fields
 * @return
 */
private static Set<Long> getMapUids(LogicalSchema schema ) {
    Set<Long> uids = new HashSet<Long>();
    if( schema != null ) {
        for( LogicalFieldSchema field : schema.getFields() ) {
            uids.add( field.uid );
        }
    }
    return uids;
}
 

/**
 * map all uids in schema to funcSpec 
 * @param schema
 * @param funcSpec
 * @throws VisitorException
 */
private void setLoadFuncForUids(LogicalSchema schema, FuncSpec funcSpec)
throws VisitorException {
    if(schema == null){
        return;
    }
    for(LogicalFieldSchema fs : schema.getFields()){
        addUidLoadFuncToMap((Long) fs.uid, funcSpec);
        setLoadFuncForUids(fs.schema, funcSpec);
    }
    
}
 

@Override
public void visit(LOGenerate gen) throws FrontendException{
    mapToPredLoadFunc(gen);
    List<LogicalExpressionPlan> expPlans = gen.getOutputPlans();
    for(LogicalExpressionPlan expPlan : expPlans){
       visitExpression(expPlan);
    }
    
    //associate flatten output to the load func associated with input
    //expression of flatten
    boolean[] flattens = gen.getFlattenFlags();
    for(int i=0; i<flattens.length; i++){
        if(flattens[i] == true){
            //get the output schema corresponding to this exp plan
            gen.getSchema();
            if(gen.getOutputPlanSchemas() == null || gen.getOutputPlanSchemas().size() <= i){
                return;
            }
            LogicalSchema sch = gen.getOutputPlanSchemas().get(i);
            if(sch == null){
                continue;
            }
            
            //get the only output exp of the ith plan
            LogicalExpression exp =
                (LogicalExpression) gen.getOutputPlans().get(i).getSources().get(0);

            //get its funcspec and associate it with uid of all fields in the schema
            FuncSpec funcSpec = uid2LoadFuncMap.get(exp.getFieldSchema().uid);
            for(LogicalFieldSchema fs : sch.getFields()){
                addUidLoadFuncToMap(fs.uid, funcSpec);
            }
        }
    }
    
}
 

@Override
public void visit(ProjectExpression p) throws FrontendException {
    // if project is a project-star or range, ie it could not be expanded
    // then its not possible to determine the matching input columns
    // before runtime
    if (p.isRangeOrStarProject()) {
        return;
    }
    
    // Get the uid for this projection.  It must match the uid of the 
    // value it is projecting.
    long myUid = p.getFieldSchema().uid;
    
    // Find the operator this projection references
    LogicalRelationalOperator pred = p.findReferent();
    
    if (p.getAttachedRelationalOp() instanceof LOGenerate && p.getPlan().getSuccessors(p)==null) {
        // No need to adjust
        return;
    }
    else {
        // Get the schema for this operator and search it for the matching uid
        int match = -1;
        LogicalSchema schema = pred.getSchema();
        if (schema==null)
            return;
        List<LogicalSchema.LogicalFieldSchema> fields = schema.getFields();
        for (int i = 0; i < fields.size(); i++) {
            if (fields.get(i).uid == myUid) {
                match = i;
                break;
            }
        }
        if (match == -1) {
            throw new FrontendException("Couldn't find matching uid " + match + " for project "+p, 2229);
        }
        p.setColNum(match);
    }
}
 

/**
 * For any UNKNOWN type in the schema fields, set the type to BYTEARRAY
 * @param sch
 */
static void setBytearrayForNULLType(LogicalSchema sch){
    for(LogicalFieldSchema fs : sch.getFields()){
        if(fs.type == DataType.NULL){
            fs.type = DataType.BYTEARRAY;
        }
        if(fs.schema != null){
            setBytearrayForNULLType(fs.schema);
        }
    }
}
 

/**
 * Construct a ResourceSchema from a {@link LogicalSchema}
 * @param pigSchema Schema to use
 */
public ResourceSchema(LogicalSchema pigSchema) {
    List<LogicalFieldSchema> pigSchemaFields = pigSchema.getFields();
    fields = new ResourceFieldSchema[pigSchemaFields.size()];
    for (int i=0; i<fields.length; i++) {
        fields[i] = new ResourceFieldSchema(pigSchemaFields.get(i));
    }        
}
 

@Override
public void visit(LOUnion relOp) throws FrontendException{
    mapToPredLoadFunc(relOp);
    
    // Since the uid changes for Union, add mappings for new uids to funcspec
    LogicalSchema schema = relOp.getSchema();
    if(schema != null){
        // For each output field, checking all the fields being
        // union-ed(bundled) together and only set the funcspec when ALL
        // of them come from the same caster
        //
        // A = (i,j)
        // B = (i,j)
        // C = UNION A, B;
        // Checking if A.i and B.i have the same caster.
        // Same for A.j and B.j
        // A.i and A.j may come from the different casters
        for (LogicalFieldSchema logicalFieldSchema : schema.getFields()) {
            Set<Long> inputs = relOp.getInputUids(logicalFieldSchema.uid);
            if( inputs.size() == 0 ) {
                // uid was not changed.
                // funcspec should be already set. skipping
                continue;
            }
            FuncSpec prevLoadFuncSpec = null, curLoadFuncSpec = null;
            boolean allSameLoader = true;
            for(Long inputUid: inputs) {
                curLoadFuncSpec = uid2LoadFuncMap.get(inputUid) ;
                if( prevLoadFuncSpec != null
                  && !haveIdenticalCasters(prevLoadFuncSpec,
                                           curLoadFuncSpec) ) {
                    allSameLoader = false;
                    break;
                }
              prevLoadFuncSpec  = curLoadFuncSpec;
            }
            if( allSameLoader ) {
                addUidLoadFuncToMap(logicalFieldSchema.uid,curLoadFuncSpec);
            }
        }
    }
}