org.apache.pig.newplan.logical.relational.LogicalSchema Java Examples

The following examples show how to use org.apache.pig.newplan.logical.relational.LogicalSchema. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestSchema.java    From spork with Apache License 2.0 6 votes vote down vote up
@Test
public void testNewMergeNullSchemas() throws Throwable {
    LogicalSchema a = Utils.parseSchema( "a1:bytearray, b1:(b11:int, b12:float), c1:long" );
    LogicalSchema b = Utils.parseSchema( "a2:bytearray, b2:(), c2:int" );

    LogicalSchema mergedSchema = LogicalSchema.merge(a, b, LogicalSchema.MergeMode.Union);
    LogicalSchema expected = Utils.parseSchema( "a1:bytearray, b1:(), c1:long" );
    assertTrue(LogicalSchema.equals(mergedSchema, expected, false, false));

    mergedSchema = LogicalSchema.merge(a, b, LogicalSchema.MergeMode.LoadForEach);
    expected = Utils.parseSchema( "a1:bytearray, b1:(b11:int, b12:float), c1:long" );
    assertTrue(LogicalSchema.equals(mergedSchema, expected, false, false));

    mergedSchema = LogicalSchema.merge(b, a, LogicalSchema.MergeMode.LoadForEach);
    expected = Utils.parseSchema( "a2:bytearray, b2:(b11:int,b12:float), c2:int" );
    assertTrue(LogicalSchema.equals(mergedSchema, expected, false, false));
}
 
Example #2
Source File: TestSchema.java    From spork with Apache License 2.0 6 votes vote down vote up
@Test
public void testNewNormalNestedMerge2() throws Exception {
    LogicalSchema a = org.apache.pig.newplan.logical.Util.translateSchema(Utils.getSchemaFromString(
        "a1:(a11:chararray, a12:float), b1:(b11:chararray, b12:float), c1:long"));
    LogicalSchema b = org.apache.pig.newplan.logical.Util.translateSchema(Utils.getSchemaFromString(
        "a2:bytearray, b2:(b21:double, b22:long), c2:chararray"));

    LogicalSchema mergedSchema = LogicalSchema.merge(a, b, LogicalSchema.MergeMode.Union);
    LogicalSchema expected = org.apache.pig.newplan.logical.Util.translateSchema(Utils.getSchemaFromString(
        "a1:(a11:chararray, a12:float), b1:(), c1:bytearray"));
    expected.getField(1).schema = new LogicalSchema();
    assertTrue(LogicalSchema.equals(mergedSchema, expected, false, false));

    mergedSchema = LogicalSchema.merge(a, b, LogicalSchema.MergeMode.LoadForEach);
    expected = org.apache.pig.newplan.logical.Util.translateSchema(Utils.getSchemaFromString(
        "a1:(a11:chararray, a12:float), b1:(b11:chararray, b12:float), c1:long"));
    assertTrue(LogicalSchema.equals(mergedSchema, expected, false, false));

    mergedSchema = LogicalSchema.merge(b, a, LogicalSchema.MergeMode.LoadForEach);
    expected = org.apache.pig.newplan.logical.Util.translateSchema(Utils.getSchemaFromString(
            "a2:(a11:chararray, a12:float), b2:(b21:double, b22:long), c2:chararray"));
    assertTrue(LogicalSchema.equals(mergedSchema, expected, false, false));
}
 
Example #3
Source File: ExampleGenerator.java    From spork with Apache License 2.0 6 votes vote down vote up
private void readBaseData(List<Operator> loads) throws IOException, InterruptedException, FrontendException, ExecException {
    PhysicalPlan thisPhyPlan = new PhysicalPlan();
    for (Operator op : loads) {
        LogicalSchema schema = ((LOLoad) op).getSchema();
        if(schema == null) {
            throw new ExecException("Example Generator requires a schema. Please provide a schema while loading data.");
        }
        poLoadToSchemaMap.put((POLoad)logToPhyMap.get(op), schema);
        thisPhyPlan.add(logToPhyMap.get(op));
    }
    baseData = null;
    Map<Operator, DataBag> result = getData(thisPhyPlan);
    baseData = new HashMap<LOLoad, DataBag>();
    for (Operator lo : result.keySet()) {
        if (lo instanceof LOLoad) {
            baseData.put((LOLoad) lo, result.get(lo));
        }
    }
}
 
Example #4
Source File: ExampleGenerator.java    From spork with Apache License 2.0 6 votes vote down vote up
public ExampleGenerator(LogicalPlan plan, PigContext hadoopPigContext) {
        this.plan = plan;
//        pigContext = new PigContext(ExecType.LOCAL, hadoopPigContext
//                .getProperties());
        pigContext = hadoopPigContext;
        // pigContext.setExecType(ExecType.LOCAL);
        FileLocalizer.setInitialized(false);
        try {
            pigContext.connect();
        } catch (ExecException e) {
            log.error("Error connecting to the cluster "
                    + e.getLocalizedMessage());

        }
        execEngine = new MRExecutionEngine(pigContext);
        localMRRunner = new LocalMapReduceSimulator();
        poLoadToSchemaMap = new HashMap<POLoad, LogicalSchema>();
    }
 
Example #5
Source File: DereferenceExpression.java    From spork with Apache License 2.0 6 votes vote down vote up
private List<Integer> translateAliasToPos(LogicalSchema schema, List<Object> rawColumns) throws FrontendException {
    List<Integer> columns = new ArrayList<Integer>();
    for( Object rawColumn : rawColumns ) {
        if( rawColumn instanceof Integer ) {
        	if (schema!=null && ((Integer)rawColumn>=schema.size() || (Integer)rawColumn<0)) {
        	    throw new FrontendException("Index "+rawColumn + " out of range in schema:" + schema.toString(false), 1127);
        	}
            columns.add( (Integer)rawColumn );
        } else {
            int pos = schema.getFieldPosition((String)rawColumn);
            if( pos != -1) {
                columns.add( pos );
                continue;
            } else {
                throw new FrontendException("Cannot find field " + rawColumn + " in " + schema.toString(false), 1128);
            }
        }
    }
    return columns;
}
 
Example #6
Source File: ColumnPruneVisitor.java    From spork with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private void addForEachIfNecessary(LogicalRelationalOperator op) throws FrontendException {
    Set<Long> outputUids = (Set<Long>)op.getAnnotation(ColumnPruneHelper.OUTPUTUIDS);
    if (outputUids!=null) {
        LogicalSchema schema = op.getSchema();
        Set<Integer> columnsToDrop = new HashSet<Integer>();

        for (int i=0;i<schema.size();i++) {
            if (!outputUids.contains(schema.getField(i).uid))
                columnsToDrop.add(i);
        }

        if (!columnsToDrop.isEmpty()) {
            LOForEach foreach = Util.addForEachAfter((LogicalPlan)op.getPlan(), op, 0, columnsToDrop);
            foreach.getSchema();
        }
    }
}
 
Example #7
Source File: ResourceSchema.java    From spork with Apache License 2.0 6 votes vote down vote up
/**
 * Only for use by Pig internal code.
 * Construct a ResourceSchema from a {@link LogicalSchema}
 * @param pigSchema LogicalSchema to use
 * @param sortInfo information on how data is sorted
 */
@InterfaceAudience.Private
public ResourceSchema(LogicalSchema pigSchema, SortInfo sortInfo) {
    this(pigSchema);
    if (sortInfo!=null && sortInfo.getSortColInfoList().size()!=0) {
        sortKeys = new int[sortInfo.getSortColInfoList().size()];
        sortKeyOrders = new Order[sortInfo.getSortColInfoList().size()];
        for (int i=0;i<sortInfo.getSortColInfoList().size();i++) {
            SortColInfo colInfo = sortInfo.getSortColInfoList().get(i); 
            int index = colInfo.getColIndex();
            Order order;
            org.apache.pig.SortColInfo.Order origOrder = colInfo.getSortOrder();
            if (origOrder==org.apache.pig.SortColInfo.Order.ASCENDING) {
                order = Order.ASCENDING;
            } else {
                order = Order.DESCENDING;
            }
            sortKeys[i] = index;
            sortKeyOrders[i] = order;
        }
    }
}
 
Example #8
Source File: ColumnPruneHelper.java    From spork with Apache License 2.0 6 votes vote down vote up
protected Set<Integer> getColumns(LogicalSchema schema, Set<Long> uids) throws FrontendException {
    if (schema == null) {
        throw new SchemaNotDefinedException("Schema is not defined.");
    }

    Set<Integer> cols = new HashSet<Integer>();
    Iterator<Long> iter = uids.iterator();
    while(iter.hasNext()) {
        long uid = iter.next();
        int index = schema.findField(uid);
        if (index == -1) {
            throw new FrontendException("UID " + uid + " is not found in the schema " + schema, 2241);
        }

        cols.add(index);
    }

    return cols;
}
 
Example #9
Source File: TestNewPlanOperatorPlan.java    From spork with Apache License 2.0 6 votes vote down vote up
@Test
public void testLoadEqualityDifferentFuncSpecCtorArgs() throws FrontendException {
    LogicalPlan lp = new LogicalPlan();

    LogicalSchema aschema1 = new LogicalSchema();
    aschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad load1 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "y" })), aschema1, lp,
            conf);
    lp.add(load1);

    LOLoad load2 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "z" })), aschema1, lp,
            conf);
    lp.add(load2);

    assertFalse(load1.isEqual(load2));
}
 
Example #10
Source File: ColumnPruneHelper.java    From spork with Apache License 2.0 6 votes vote down vote up
@Override
public void visit(LOStore store) throws FrontendException {
    Set<Long> output = setOutputUids(store);

    if (output.isEmpty()) {
        // to deal with load-store-load-store case
        LogicalSchema s = store.getSchema();
        if (s == null) {
            throw new SchemaNotDefinedException("Schema for " + store.getName() + " is not defined.");
        }

        for(int i=0; i<s.size(); i++) {
            output.add(s.getField(i).uid);
        }
    }

    // for store, input uids are same as output uids
    store.annotate(INPUTUIDS, output);
}
 
Example #11
Source File: ColumnPruneHelper.java    From spork with Apache License 2.0 6 votes vote down vote up
@Override
public void visit(LOCross cross) throws FrontendException {
    Set<Long> output = setOutputUids(cross);
    // Since we do not change the topology of the plan, we keep
    // at least one input for each predecessor.
    List<Operator> preds = plan.getPredecessors(cross);
    for (Operator pred : preds) {
        LogicalSchema schema = ((LogicalRelationalOperator)pred).getSchema();
        Set<Long> uids = getAllUids(schema);
        boolean allPruned = true;
        for (Long uid : uids) {
            if (output.contains(uid))
                allPruned = false;
        }
        if (allPruned)
            output.add(schema.getField(0).uid);
    }
    cross.annotate(INPUTUIDS, output);
}
 
Example #12
Source File: ColumnPruneHelper.java    From spork with Apache License 2.0 6 votes vote down vote up
private Set<Long> getAllUids( LogicalSchema schema ) {
    Set<Long> uids = new HashSet<Long>();

    if( schema == null ) {
        return uids;
    }

    for( LogicalFieldSchema field : schema.getFields() ) {
        if( ( field.type == DataType.TUPLE || field.type == DataType.BAG )
                && field.schema != null ) {
           uids.addAll( getAllUids( field.schema ) );
        }
        uids.add( field.uid );
    }
    return uids;
}
 
Example #13
Source File: Util.java    From spork with Apache License 2.0 6 votes vote down vote up
/**
 * This function translates the new LogicalSchema into old Schema format required
 * by PhysicalOperators
 * @param schema LogicalSchema to be converted to Schema
 * @return Schema that is converted from LogicalSchema
 * @throws FrontendException
 */
public static Schema translateSchema(LogicalSchema schema) {
    if (schema == null) {
        return null;
    }

    Schema s2 = new Schema();
    List<LogicalSchema.LogicalFieldSchema> ll = schema.getFields();
    for (LogicalSchema.LogicalFieldSchema f: ll) {
        Schema.FieldSchema f2 = null;
        try {
            f2 = new Schema.FieldSchema(f.alias, translateSchema(f.schema), f.type);
            f2.canonicalName = ((Long)f.uid).toString();
            s2.add(f2);
        } catch (FrontendException e) {
        }
    }

    return s2;
}
 
Example #14
Source File: Util.java    From spork with Apache License 2.0 6 votes vote down vote up
public static LogicalSchema translateSchema(Schema schema) {
    if (schema == null) {
        return null;
    }

    LogicalSchema s2 = new LogicalSchema();
    List<Schema.FieldSchema> ll = schema.getFields();
    for (Schema.FieldSchema f: ll) {
        LogicalSchema.LogicalFieldSchema f2 =
            new LogicalSchema.LogicalFieldSchema(f.alias, translateSchema(f.schema), f.type);

        s2.addField(f2);
    }

    return s2;
}
 
Example #15
Source File: ColumnPruneHelper.java    From spork with Apache License 2.0 6 votes vote down vote up
private void collectUids(LogicalRelationalOperator currentOp, LogicalExpressionPlan exp, Set<Long> uids) throws FrontendException {
    List<Operator> ll = exp.getSinks();
    for(Operator op: ll) {
        if (op instanceof ProjectExpression) {
            if (!((ProjectExpression)op).isRangeOrStarProject()) {
                long uid = ((ProjectExpression)op).getFieldSchema().uid;
                uids.add(uid);
            } else {
                LogicalRelationalOperator ref = ((ProjectExpression)op).findReferent();
                LogicalSchema s = ref.getSchema();
                if (s == null) {
                    throw new SchemaNotDefinedException("Schema not defined for " + ref.getAlias());
                }
                for(LogicalFieldSchema f: s.getFields()) {
                    uids.add(f.uid);
                }
            }
        }
    }
}
 
Example #16
Source File: MapLookupExpression.java    From spork with Apache License 2.0 6 votes vote down vote up
public LogicalFieldSchema getFieldSchema() throws FrontendException {
    if (fieldSchema!=null)
        return fieldSchema;
    LogicalExpression successor = (LogicalExpression)plan.getSuccessors(this).get(0);
    LogicalFieldSchema predFS = successor.getFieldSchema();
    if (predFS!=null) {
        if (predFS.type==DataType.MAP && predFS.schema!=null) {
            return (predFS.schema.getField(0));
        }
        else {
            fieldSchema = new LogicalSchema.LogicalFieldSchema(null, null, DataType.BYTEARRAY);
            uidOnlyFieldSchema = fieldSchema.mergeUid(uidOnlyFieldSchema);
            return fieldSchema;
        }
    }
    return null;
}
 
Example #17
Source File: TestPlanGeneration.java    From spork with Apache License 2.0 6 votes vote down vote up
@Test
public void testEmptyBagDereference() throws Exception {
    String query = "A = load 'x' as ( u:bag{} );" +
            "B = foreach A generate u.$100;" +
            "store B into '111';";

    LogicalPlan lp = Util.parseAndPreprocess(query, pc);
    Util.optimizeNewLP(lp);
    LOStore loStore = (LOStore)lp.getSinks().get(0);
    LOForEach loForEach = (LOForEach)lp.getPredecessors(loStore).get(0);
    LogicalSchema schema = loForEach.getSchema();
    assertEquals(1, schema.size());
    LogicalFieldSchema bagFieldSchema = schema.getField(0);
    assertEquals(DataType.BAG, bagFieldSchema.type);
    LogicalFieldSchema tupleFieldSchema = bagFieldSchema.schema.getField(0);
    assertEquals(1, tupleFieldSchema.schema.size());
    assertEquals(DataType.BYTEARRAY, tupleFieldSchema.schema.getField(0).type);
}
 
Example #18
Source File: AugmentBaseDataVisitor.java    From spork with Apache License 2.0 6 votes vote down vote up
private boolean inInput(Tuple newTuple, DataBag input, LogicalSchema schema) throws ExecException {
    boolean result;
    for (Iterator<Tuple> iter = input.iterator(); iter.hasNext();) {
        result = true;
        Tuple tmp = iter.next();
        for (int i = 0; i < schema.size(); ++i)
            if (!newTuple.get(i).equals(tmp.get(i)))
            {
                result = false;
                break;
            }
        if (result)
            return true;
    }
    return false;
}
 
Example #19
Source File: TestLogicalPlanBuilder.java    From spork with Apache License 2.0 6 votes vote down vote up
@Test
public void testQuery85() throws Exception {
    LogicalPlan lp;
    String query = "a = load 'myfile' as (name, age, gpa);" +
             "b = group a by (name, age);";
    lp = buildPlan( query + "store b into 'output';");
    Operator store = lp.getSinks().get(0);
    LOCogroup cogroup = (LOCogroup) lp.getPredecessors(store).get(0);

    LogicalSchema actual = cogroup.getSchema();
    System.out.println( actual.toString( false ) );

    Assert.assertTrue(  actual.toString( false ).equals( "group:tuple(name:bytearray,age:bytearray),a:bag{:tuple(name:bytearray,age:bytearray,gpa:bytearray)}" ) );

    lp = buildPlan(query +
    		       "c = foreach b generate group.name, group.age, COUNT(a.gpa);" +
    		       "store c into 'output';");
    store = lp.getSinks().get(0);
    LOForEach foreach  = (LOForEach) lp.getPredecessors(store).get(0);


    Assert.assertTrue( foreach.getSchema().toString( false ).equals("name:bytearray,age:bytearray,:long") );
}
 
Example #20
Source File: TestNewPlanOperatorPlan.java    From spork with Apache License 2.0 6 votes vote down vote up
@Test
public void testLoadEqualityDifferentFileName() throws FrontendException {
    LogicalPlan lp = new LogicalPlan();
    LogicalSchema aschema1 = new LogicalSchema();
    aschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad load1 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "y" })), aschema1, lp,
            conf);
    lp.add(load1);

    // Different file name
    LOLoad load5 = newLOLoad(new FileSpec("/def",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "z" })), aschema1, lp,
            conf);
    lp.add(load5);

    assertFalse(load1.isEqual(load5));
}
 
Example #21
Source File: FilterAboveForeach.java    From spork with Apache License 2.0 6 votes vote down vote up
/**
 * checks if a relational operator contains all of the specified uids
 * @param op LogicalRelational operator that should contain the uid
 * @param uids Uids to check for
 * @return true if given LogicalRelationalOperator has all the given uids
 */
private boolean hasAll(LogicalRelationalOperator op, Pair<List<Long>,
        List<Byte>> uidWithTypes) throws FrontendException {
    LogicalSchema schema = op.getSchema();

    if (schema==null)
        return false;

    List<Long> uids = uidWithTypes.first;
    List<Byte> types = uidWithTypes.second;

    for (int i=0;i<uids.size();i++) {
        boolean found = false;
        for (LogicalSchema.LogicalFieldSchema fs : schema.getFields()) {
            if (fs.uid==uids.get(i) && fs.type==types.get(i))
                found = true;
        }
        if (!found)
            return false;
    }
    return true;
}
 
Example #22
Source File: AddExpression.java    From spork with Apache License 2.0 5 votes vote down vote up
@Override
public LogicalSchema.LogicalFieldSchema getFieldSchema() throws FrontendException {
    if (fieldSchema!=null)
        return fieldSchema;
    fieldSchema = new LogicalSchema.LogicalFieldSchema(null, null, getLhs().getType());
    uidOnlyFieldSchema = fieldSchema.mergeUid(uidOnlyFieldSchema);
    return fieldSchema;
}
 
Example #23
Source File: TestNewPlanLogToPhyTranslationVisitor.java    From spork with Apache License 2.0 5 votes vote down vote up
@Test    
public void testPlanwithisNull() throws Exception {
    String query = ("a = load 'd.txt' as (a:int, b:int);" +
    "b = filter a by a is null;" +        
    "store b into 'empty';");  
    
    LogicalPlan newLogicalPlan = buildPlan(query);
    LogicalRelationalOperator ld =  (LogicalRelationalOperator)newLogicalPlan.getSources().get(0);
    assertEquals( LOLoad.class, ld.getClass() );
    LOLoad load = (LOLoad)ld;
    LogicalSchema ls = load.getSchema();
    
    assertEquals(1, ls.getField(0).uid);
    assertEquals(2, ls.getField(1).uid);
    
    LogicalRelationalOperator fil = (LogicalRelationalOperator)
    newLogicalPlan.getSuccessors( newLogicalPlan.getSources().get(0) ).get(0);
    assertEquals( LOFilter.class, 
            fil.getClass() );
    LOFilter filter = (LOFilter)fil;
    
    LogicalExpressionPlan filPlan = filter.getFilterPlan();
    
    assertEquals( 1, filPlan.getSources().size() );
    assertEquals( 2, filPlan.size() );
    assertEquals( 1, filPlan.getSinks().size() );
    assertEquals( IsNullExpression.class, filPlan.getSources().get(0).getClass() );
    IsNullExpression isNull = (IsNullExpression)filPlan.getSources().get(0);
    assertTrue( ls.getField(0).uid != isNull.getFieldSchema().uid );
    assertTrue( ls.getField(1).uid != isNull.getFieldSchema().uid );
    
    assertEquals( ProjectExpression.class, isNull.getExpression().getClass() );
    ProjectExpression prj = (ProjectExpression) isNull.getExpression();
    assertEquals( ls.getField(0).uid, prj.getFieldSchema().uid );
}
 
Example #24
Source File: TestNewPlanLogToPhyTranslationVisitor.java    From spork with Apache License 2.0 5 votes vote down vote up
@Test    
public void testPlanwithisNotNull() throws Exception {
    String query = ("a = load 'd.txt' as (a:int, b:int);" +
    "b = filter a by a is not null;" +        
    "store b into 'empty';");  
    
    LogicalPlan newLogicalPlan = buildPlan(query);
    LogicalRelationalOperator ld =  (LogicalRelationalOperator)newLogicalPlan.getSources().get(0);
    assertEquals( LOLoad.class, ld.getClass() );
    LOLoad load = (LOLoad)ld;
    LogicalSchema ls = load.getSchema();
    
    assertEquals(1, ls.getField(0).uid);
    assertEquals(2, ls.getField(1).uid);
    
    LogicalRelationalOperator fil = (LogicalRelationalOperator)
    newLogicalPlan.getSuccessors( newLogicalPlan.getSources().get(0) ).get(0);
    assertEquals( LOFilter.class, 
            fil.getClass() );
    LOFilter filter = (LOFilter)fil;
    
    LogicalExpressionPlan filPlan = filter.getFilterPlan();
    
    assertEquals( 1, filPlan.getSources().size() );
    assertEquals( 3, filPlan.size() );
    assertEquals( 1, filPlan.getSinks().size() );
    assertEquals( NotExpression.class, filPlan.getSources().get(0).getClass() );
    NotExpression notExp = (NotExpression)filPlan.getSources().get(0);
    assertTrue( ls.getField(0).uid != notExp.getFieldSchema().uid );
    assertTrue( ls.getField(1).uid != notExp.getFieldSchema().uid );
    assertEquals( IsNullExpression.class, notExp.getExpression().getClass() );
    IsNullExpression isNull = (IsNullExpression)notExp.getExpression();
    assertTrue( ls.getField(0).uid != isNull.getFieldSchema().uid );
    assertTrue( ls.getField(1).uid != isNull.getFieldSchema().uid );
    
    assertEquals( ProjectExpression.class, isNull.getExpression().getClass() );
    ProjectExpression prj = (ProjectExpression) isNull.getExpression();
    assertEquals( ls.getField(0).uid, prj.getFieldSchema().uid );
}
 
Example #25
Source File: EqualExpression.java    From spork with Apache License 2.0 5 votes vote down vote up
@Override
public LogicalSchema.LogicalFieldSchema getFieldSchema() throws FrontendException {
    if (fieldSchema!=null)
        return fieldSchema;
    fieldSchema = new LogicalSchema.LogicalFieldSchema(null, null, DataType.BOOLEAN);
    uidOnlyFieldSchema = fieldSchema.mergeUid(uidOnlyFieldSchema);
    return fieldSchema;
}
 
Example #26
Source File: LineageFindRelVisitor.java    From spork with Apache License 2.0 5 votes vote down vote up
@Override
public void visit(LOInnerLoad innerLoad) throws FrontendException{
    LOForEach foreach = innerLoad.getLOForEach();
    LogicalRelationalOperator pred = 
        ((LogicalRelationalOperator)foreach.getPlan().getPredecessors(foreach).get(0));

    LogicalSchema predSchema = pred.getSchema();
    
    //if this has a schema, the lineage can be tracked using the uid in input schema
    if(innerLoad.getSchema() != null){
        if(innerLoad.getSchema().size() == 1 
                && innerLoad.getSchema().getField(0).type == DataType.BYTEARRAY
                && uid2LoadFuncMap.get(innerLoad.getSchema().getField(0).uid) == null
                && predSchema != null
        ){
            long inpUid = predSchema.getField(innerLoad.getProjection().getColNum()).uid;
            if(uid2LoadFuncMap.get(inpUid) != null){
                addUidLoadFuncToMap(innerLoad.getSchema().getField(0).uid, uid2LoadFuncMap.get(inpUid));
            }
            return;
        }

    }

    // associated load func could not be found using uid, use
    // the single load func associated with input relation (if any)
    if(getAssociatedLoadFunc(pred) != null){
        mapRelToPredLoadFunc(innerLoad, pred);
    }
}
 
Example #27
Source File: LineageFindRelVisitor.java    From spork with Apache License 2.0 5 votes vote down vote up
/**
 * map all uids in schema to funcSpec 
 * @param schema
 * @param funcSpec
 * @throws VisitorException
 */
private void setLoadFuncForUids(LogicalSchema schema, FuncSpec funcSpec)
throws VisitorException {
    if(schema == null){
        return;
    }
    for(LogicalFieldSchema fs : schema.getFields()){
        addUidLoadFuncToMap((Long) fs.uid, funcSpec);
        setLoadFuncForUids(fs.schema, funcSpec);
    }
    
}
 
Example #28
Source File: AugmentBaseDataVisitor.java    From spork with Apache License 2.0 5 votes vote down vote up
Tuple BackPropConstraint(Tuple outputConstraint, List<Integer> cols,
        LogicalSchema inputSchema, boolean cast) throws ExecException {
    Tuple inputConst = TupleFactory.getInstance().newTuple(
            inputSchema.getFields().size());

    Tuple inputConstraint = new ExampleTuple(inputConst);

    for (int outCol = 0; outCol < outputConstraint.size(); outCol++) {
        int inCol = cols.get(outCol);
        Object outVal = outputConstraint.get(outCol);
        Object inVal = inputConstraint.get(inCol);

        if (inVal == null && outVal != null) {
            // inputConstraint.set(inCol, outVal);
            inputConstraint.set(inCol, (cast) ? new DataByteArray(outVal
                    .toString().getBytes()) : outVal);

        } else {
            if (outVal != null) {
                // unable to back-propagate, due to conflicting column
                // constraints, so give up
                return null;
            }
        }
    }

    return inputConstraint;
}
 
Example #29
Source File: BinCondExpression.java    From spork with Apache License 2.0 5 votes vote down vote up
@Override
public LogicalSchema.LogicalFieldSchema getFieldSchema() throws FrontendException {
    if (fieldSchema!=null)
        return fieldSchema;

    //TypeCheckingExpVisitor will ensure that lhs and rhs have same schema
    LogicalFieldSchema argFs = getLhs().getFieldSchema();
    fieldSchema = argFs.deepCopy();
    fieldSchema.resetUid();

    uidOnlyFieldSchema = fieldSchema.mergeUid(uidOnlyFieldSchema);
    return fieldSchema;
}
 
Example #30
Source File: GreaterThanEqualExpression.java    From spork with Apache License 2.0 5 votes vote down vote up
@Override
public LogicalSchema.LogicalFieldSchema getFieldSchema() throws FrontendException {
    if (fieldSchema!=null)
        return fieldSchema;
    fieldSchema = new LogicalSchema.LogicalFieldSchema(null, null, DataType.BOOLEAN);
    uidOnlyFieldSchema = fieldSchema.mergeUid(uidOnlyFieldSchema);
    return fieldSchema;
}