Java Code Examples for org.apache.pig.newplan.logical.relational.LogicalSchema#addField()

public static LogicalSchema translateSchema(Schema schema) {
    if (schema == null) {
        return null;
    }

    LogicalSchema s2 = new LogicalSchema();
    List<Schema.FieldSchema> ll = schema.getFields();
    for (Schema.FieldSchema f: ll) {
        LogicalSchema.LogicalFieldSchema f2 =
            new LogicalSchema.LogicalFieldSchema(f.alias, translateSchema(f.schema), f.type);

        s2.addField(f2);
    }

    return s2;
}
 

@Test
public void testLoadEqualityDifferentFuncSpecCtorArgs() throws FrontendException {
    LogicalPlan lp = new LogicalPlan();

    LogicalSchema aschema1 = new LogicalSchema();
    aschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad load1 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "y" })), aschema1, lp,
            conf);
    lp.add(load1);

    LOLoad load2 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "z" })), aschema1, lp,
            conf);
    lp.add(load2);

    assertFalse(load1.isEqual(load2));
}
 

@Test
public void testLoadEqualityDifferentNumFuncSpecCstorArgs() throws FrontendException {
    LogicalPlan lp = new LogicalPlan();

    LogicalSchema aschema1 = new LogicalSchema();
    aschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad load1 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "y" })), aschema1, lp,
            conf);
    lp.add(load1);

    LOLoad load3 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), "x")), aschema1, lp, conf);
    lp.add(load3);

    assertFalse(load1.isEqual(load3));
}
 

@Test
public void testLoadEqualityDifferentFunctionNames() throws FrontendException {
    LogicalPlan lp = new LogicalPlan();

    LogicalSchema aschema1 = new LogicalSchema();
    aschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad load1 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "y" })), aschema1, lp,
            conf);
    lp.add(load1);

    // Different function names in FuncSpec
    LOLoad load4 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "z" })), aschema1, lp,
            conf);
    lp.add(load4);

    assertFalse(load1.isEqual(load4));
}
 

@Test
public void testLoadEqualityDifferentFileName() throws FrontendException {
    LogicalPlan lp = new LogicalPlan();
    LogicalSchema aschema1 = new LogicalSchema();
    aschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad load1 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "y" })), aschema1, lp,
            conf);
    lp.add(load1);

    // Different file name
    LOLoad load5 = newLOLoad(new FileSpec("/def",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "z" })), aschema1, lp,
            conf);
    lp.add(load5);

    assertFalse(load1.isEqual(load5));
}
 

@Test
public void testRelationalEqualityDifferentSchema() throws FrontendException {
    LogicalPlan lp = new LogicalPlan();
    LogicalSchema aschema1 = new LogicalSchema();
    aschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad load1 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "y" })), aschema1, lp,
            conf);
    lp.add(load1);

    // Different schema
    LogicalSchema aschema2 = new LogicalSchema();
    aschema2.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.CHARARRAY));

    LOLoad load6 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "z" })), aschema2, lp,
            conf);
    lp.add(load6);

    assertFalse(load1.isEqual(load6));
}
 

@Test
public void testRelationalEqualityOneNullOneNotNullSchema() throws FrontendException {
    LogicalPlan lp = new LogicalPlan();
    LogicalSchema aschema1 = new LogicalSchema();
    aschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad load1 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "y" })), aschema1, lp,
            conf);
    lp.add(load1);

    // Test that one with schema and one without breaks equality
    LOLoad load9 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "z" })), null, lp, conf);
    lp.add(load9);

    assertFalse(load1.isEqual(load9));
}
 

protected LogicalSchema determineSchema(LogicalRelationalOperator op)
        throws FrontendException {
    LogicalSchema determinedSchema = new LogicalSchema();
    for (int i = 0; i < op.getSchema().size(); i++) {
        determinedSchema.addField(new LogicalFieldSchema(
                null,
                null,
                DataType.BYTEARRAY));
    }
    return determinedSchema;
}
 

@Test
public void testFilterDifferentPredicates() throws FrontendException {
    LogicalPlan lp = new LogicalPlan();

    LogicalExpressionPlan fp1 = new LogicalExpressionPlan();
    ProjectExpression fy1 = new ProjectExpression(fp1, 0, 1, null);
    ConstantExpression fc1 = new ConstantExpression(fp1,
            new Integer(0));
    new EqualExpression(fp1, fy1, fc1);
    LOFilter D1 = new LOFilter(lp, fp1);
    LogicalSchema cschema = new LogicalSchema();
    cschema.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    cschema.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    D1.neverUseForRealSetSchema(cschema);
    lp.add(D1);

    LogicalExpressionPlan fp2 = new LogicalExpressionPlan();
    ProjectExpression fy2 = new ProjectExpression(fp2, 0, 1, null);
    ConstantExpression fc2 = new ConstantExpression(fp2,
            new Integer(1));
    new EqualExpression(fp2, fy2, fc2);
    LOFilter D2 = new LOFilter(lp, fp2);
    D2.neverUseForRealSetSchema(cschema);
    lp.add(D2);

    assertFalse(D1.isEqual(D2));
}
 

@Override
public LogicalSchema.LogicalFieldSchema getFieldSchema() throws FrontendException {
    if (fieldSchema!=null)
        return fieldSchema;

    LogicalSchema inputSchema = new LogicalSchema();
    List<Operator> succs = plan.getSuccessors(this);

    if (succs!=null) {
        for(Operator lo : succs){
            if (((LogicalExpression)lo).getFieldSchema()==null) {
                inputSchema = null;
                break;
            }
            inputSchema.addField(((LogicalExpression)lo).getFieldSchema());
        }
    }

    if (lazilyInitializeInvokerFunction) {
        initializeInvokerFunction();
    }

    // Since ef only set one time, we never change its value, so we can optimize it by instantiate only once.
    // This significantly optimize the performance of frontend (PIG-1738)
    if (ef==null) {
        ef = (EvalFunc<?>) PigContext.instantiateFuncFromSpec(mFuncSpec);
    }

    ef.setUDFContextSignature(signature);
    Properties props = UDFContext.getUDFContext().getUDFProperties(ef.getClass());
    Schema translatedInputSchema = Util.translateSchema(inputSchema);
    if(translatedInputSchema != null) {
        props.put("pig.evalfunc.inputschema."+signature, translatedInputSchema);
    }
    // Store inputSchema into the UDF context
    ef.setInputSchema(translatedInputSchema);

    Schema udfSchema = ef.outputSchema(translatedInputSchema);
    if (udfSchema != null && udfSchema.size() > 1) {
        throw new FrontendException("Given UDF returns an improper Schema. Schema should only contain one field of a Tuple, Bag, or a single type. Returns: " + udfSchema);
    }

    //TODO appendability should come from a setting
    SchemaTupleFrontend.registerToGenerateIfPossible(translatedInputSchema, false, GenContext.UDF);
    SchemaTupleFrontend.registerToGenerateIfPossible(udfSchema, false, GenContext.UDF);

    if (udfSchema != null) {
        Schema.FieldSchema fs;
        if(udfSchema.size() == 0) {
            fs = new Schema.FieldSchema(null, null, DataType.findType(ef.getReturnType()));
        } else if(udfSchema.size() == 1) {
            fs = new Schema.FieldSchema(udfSchema.getField(0));
        } else {
            fs = new Schema.FieldSchema(null, udfSchema, DataType.TUPLE);
        }
        fieldSchema = Util.translateFieldSchema(fs);
        fieldSchema.normalize();
    } else {
        fieldSchema = new LogicalSchema.LogicalFieldSchema(null, null, DataType.findType(ef.getReturnType()));
    }
    uidOnlyFieldSchema = fieldSchema.mergeUid(uidOnlyFieldSchema);
    return fieldSchema;
}
 

@Test
public void testJoinDifferentJoinTypes() throws FrontendException {
    LogicalPlan lp = new LogicalPlan();
    LogicalSchema jaschema1 = new LogicalSchema();
    jaschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad A1 = newLOLoad(new FileSpec("/abc",
            new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
                            "x", "y" })), jaschema1, lp, conf);
    lp.add(A1);

    // B = load
    LogicalSchema jbschema1 = new LogicalSchema();
    jbschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LOLoad B1 = newLOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema1, lp, conf);
    lp.add(B1);

    // C = join
    LogicalSchema jcschema1 = new LogicalSchema();
    jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LogicalExpressionPlan aprojplan1 = new LogicalExpressionPlan();
    new ProjectExpression(aprojplan1, 0, 0, null);
    LogicalExpressionPlan bprojplan1 = new LogicalExpressionPlan();
    new ProjectExpression(bprojplan1, 1, 0, null);
    MultiMap<Integer, LogicalExpressionPlan> mm1 =
            new MultiMap<Integer, LogicalExpressionPlan>();
    mm1.put(0, aprojplan1);
    mm1.put(1, bprojplan1);
    LOJoin C1 = new LOJoin(lp, mm1, JOINTYPE.HASH, new boolean[] { true, true });
    C1.neverUseForRealSetSchema(jcschema1);
    lp.add(C1);
    lp.connect(A1, C1);
    lp.connect(B1, C1);

    // A = load
    LogicalSchema jaschema2 = new LogicalSchema();
    jaschema2.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad A2 = newLOLoad(new FileSpec("/abc",
            new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
                            "x", "y" })), jaschema2, lp, conf);
    lp.add(A2);

    // B = load
    LogicalSchema jbschema2 = new LogicalSchema();
    jbschema2.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LOLoad B2 = newLOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema2, lp, conf);
    lp.add(B2);

    // C = join
    LogicalSchema jcschema2 = new LogicalSchema();
    jcschema2.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    jcschema2.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LogicalExpressionPlan aprojplan2 = new LogicalExpressionPlan();
    new ProjectExpression(aprojplan2, 0, 0, null);
    LogicalExpressionPlan bprojplan2 = new LogicalExpressionPlan();
    new ProjectExpression(bprojplan2, 1, 0, null);
    MultiMap<Integer, LogicalExpressionPlan> mm2 =
            new MultiMap<Integer, LogicalExpressionPlan>();
    mm2.put(0, aprojplan2);
    mm2.put(1, bprojplan2);
    LOJoin C2 = new LOJoin(lp, mm2, JOINTYPE.SKEWED, new boolean[] { true, true });
    C2.neverUseForRealSetSchema(jcschema2);
    lp.add(C2);
    lp.connect(A2, C2);
    lp.connect(B2, C2);

    assertFalse(C1.isEqual(C2));
}
 

@Test
public void testJoinDifferentInner() throws FrontendException {
    LogicalPlan lp = new LogicalPlan();
    LogicalSchema jaschema1 = new LogicalSchema();
    jaschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad A1 = newLOLoad(new FileSpec("/abc",
            new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
                            "x", "y" })), jaschema1, lp, conf);
    lp.add(A1);

    // B = load
    LogicalSchema jbschema1 = new LogicalSchema();
    jbschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LOLoad B1 = newLOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema1, lp, conf);
    lp.add(B1);

    // C = join
    LogicalSchema jcschema1 = new LogicalSchema();
    jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LogicalExpressionPlan aprojplan1 = new LogicalExpressionPlan();
    new ProjectExpression(aprojplan1, 0, 0, null);
    LogicalExpressionPlan bprojplan1 = new LogicalExpressionPlan();
    new ProjectExpression(bprojplan1, 1, 0, null);
    MultiMap<Integer, LogicalExpressionPlan> mm1 =
            new MultiMap<Integer, LogicalExpressionPlan>();
    mm1.put(0, aprojplan1);
    mm1.put(1, bprojplan1);
    LOJoin C1 = new LOJoin(lp, mm1, JOINTYPE.HASH, new boolean[] { true, true });
    C1.neverUseForRealSetSchema(jcschema1);
    lp.add(C1);
    lp.connect(A1, C1);
    lp.connect(B1, C1);

    // Test different inner status
    // A = load
    LogicalSchema jaschema3 = new LogicalSchema();
    jaschema3.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad A3 = newLOLoad(new FileSpec("/abc",
            new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
                            "x", "y" })), jaschema3, lp, conf);
    lp.add(A3);

    // B = load
    LogicalSchema jbschema3 = new LogicalSchema();
    jbschema3.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LOLoad B3 = newLOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema3, lp, conf);
    lp.add(B3);

    // C = join
    LogicalSchema jcschema3 = new LogicalSchema();
    jcschema3.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    jcschema3.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LogicalExpressionPlan aprojplan3 = new LogicalExpressionPlan();
    new ProjectExpression(aprojplan3, 0, 0, null);
    LogicalExpressionPlan bprojplan3 = new LogicalExpressionPlan();
    new ProjectExpression(bprojplan3, 1, 0, null);
    MultiMap<Integer, LogicalExpressionPlan> mm3 =
            new MultiMap<Integer, LogicalExpressionPlan>();
    mm3.put(0, aprojplan3);
    mm3.put(1, bprojplan3);
    LOJoin C3 = new LOJoin(lp, mm3, JOINTYPE.HASH, new boolean[] { true, false });
    C3.neverUseForRealSetSchema(jcschema3);
    lp.add(C3);
    lp.connect(A3, C3);
    lp.connect(B3, C3);

    assertFalse(C1.isEqual(C3));
}
 

@Test
public void testJoinDifferentNumInputs() throws FrontendException {
    LogicalPlan lp = new LogicalPlan();
    LogicalSchema jaschema1 = new LogicalSchema();
    jaschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad A1 = newLOLoad(new FileSpec("/abc",
            new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
                            "x", "y" })), jaschema1, lp, conf);
    lp.add(A1);

    // B = load
    LogicalSchema jbschema1 = new LogicalSchema();
    jbschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LOLoad B1 = newLOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema1, lp, conf);
    lp.add(B1);

    // C = join
    LogicalSchema jcschema1 = new LogicalSchema();
    jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LogicalExpressionPlan aprojplan1 = new LogicalExpressionPlan();
    new ProjectExpression(aprojplan1, 0, 0, null);
    LogicalExpressionPlan bprojplan1 = new LogicalExpressionPlan();
    new ProjectExpression(bprojplan1, 1, 0, null);
    MultiMap<Integer, LogicalExpressionPlan> mm1 =
            new MultiMap<Integer, LogicalExpressionPlan>();
    mm1.put(0, aprojplan1);
    mm1.put(1, bprojplan1);
    LOJoin C1 = new LOJoin(lp, mm1, JOINTYPE.HASH, new boolean[] { true, true });
    C1.neverUseForRealSetSchema(jcschema1);
    lp.add(C1);
    lp.connect(A1, C1);
    lp.connect(B1, C1);

    // A = load
    LogicalSchema jaschema5 = new LogicalSchema();
    jaschema5.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad A5 = newLOLoad(new FileSpec("/abc",
            new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
                            "x", "y" })), jaschema5, lp, conf);
    lp.add(A5);

    // B = load
    LogicalSchema jbschema5 = new LogicalSchema();
    jbschema5.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LOLoad B5 = newLOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema5, lp, conf);
    lp.add(B5);

    // Beta = load
    LogicalSchema jbetaschema5 = new LogicalSchema();
    jbetaschema5.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LOLoad Beta5 = newLOLoad(new FileSpec("/ghi",
            new FuncSpec("PigStorage", "\t")), jbetaschema5, lp, conf);
    lp.add(Beta5);

    // C = join
    LogicalSchema jcschema5 = new LogicalSchema();
    jcschema5.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    jcschema5.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LogicalExpressionPlan aprojplan5 = new LogicalExpressionPlan();
    new ProjectExpression(aprojplan5, 0, 0, null);
    LogicalExpressionPlan bprojplan5 = new LogicalExpressionPlan();
    new ProjectExpression(bprojplan5, 1, 0, null);
    LogicalExpressionPlan betaprojplan5 = new LogicalExpressionPlan();
    new ProjectExpression(betaprojplan5, 1, 0, null);
    MultiMap<Integer, LogicalExpressionPlan> mm5 =
            new MultiMap<Integer, LogicalExpressionPlan>();
    mm5.put(0, aprojplan5);
    mm5.put(1, bprojplan5);
    mm5.put(2, betaprojplan5);
    LOJoin C5 = new LOJoin(lp, mm5, JOINTYPE.HASH, new boolean[] { true, true });
    C5.neverUseForRealSetSchema(jcschema5);
    lp.add(C5);
    lp.connect(A5, C5);
    lp.connect(B5, C5);
    lp.connect(Beta5, C5);

    assertFalse(C1.isEqual(C5));
}
 

@Test
public void testJoinDifferentJoinKeys() throws FrontendException {
    LogicalPlan lp = new LogicalPlan();

    // Test different join keys
    LogicalSchema jaschema6 = new LogicalSchema();
    jaschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad A6 = newLOLoad(new FileSpec("/abc",
            new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
                            "x", "y" })), jaschema6, lp, conf);
    lp.add(A6);

    // B = load
    LogicalSchema jbschema6 = new LogicalSchema();
    jbschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    jbschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "z", null, DataType.LONG));
    LOLoad B6 = newLOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema6, lp, conf);
    lp.add(B6);

    // C = join
    LogicalSchema jcschema6 = new LogicalSchema();
    jcschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    jcschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LogicalExpressionPlan aprojplan6 = new LogicalExpressionPlan();
    new ProjectExpression(aprojplan6, 0, 0, null);
    LogicalExpressionPlan bprojplan6 = new LogicalExpressionPlan();
    new ProjectExpression(bprojplan6, 1, 0, null);
    LogicalExpressionPlan b2projplan6 = new LogicalExpressionPlan();
    new ProjectExpression(b2projplan6, 1, 1, null);
    MultiMap<Integer, LogicalExpressionPlan> mm6 =
            new MultiMap<Integer, LogicalExpressionPlan>();
    mm6.put(0, aprojplan6);
    mm6.put(1, bprojplan6);
    mm6.put(1, b2projplan6);
    LOJoin C6 = new LOJoin(lp, mm6, JOINTYPE.HASH, new boolean[] { true, true });
    C6.neverUseForRealSetSchema(jcschema6);
    lp.add(C6);
    lp.connect(A6, C6);
    lp.connect(B6, C6);

    LogicalSchema jaschema7 = new LogicalSchema();
    jaschema7.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad A7 = newLOLoad(new FileSpec("/abc",
            new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
                            "x", "y" })), jaschema7, lp, conf);
    lp.add(A7);

    // B = load
    LogicalSchema jbschema7 = new LogicalSchema();
    jbschema7.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    jbschema7.addField(new LogicalSchema.LogicalFieldSchema(
            "z", null, DataType.LONG));
    LOLoad B7 = newLOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema7, lp, conf);
    lp.add(B7);

    // C = join
    LogicalSchema jcschema7 = new LogicalSchema();
    jcschema7.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    jcschema7.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LogicalExpressionPlan aprojplan7 = new LogicalExpressionPlan();
    new ProjectExpression(aprojplan7, 0, 0, null);
    LogicalExpressionPlan bprojplan7 = new LogicalExpressionPlan();
    new ProjectExpression(bprojplan7, 1, 1, null);
    LogicalExpressionPlan b2projplan7 = new LogicalExpressionPlan();
    new ProjectExpression(b2projplan7, 1, 0, null);
    MultiMap<Integer, LogicalExpressionPlan> mm7 =
            new MultiMap<Integer, LogicalExpressionPlan>();
    mm7.put(0, aprojplan7);
    mm7.put(1, bprojplan7);
    mm7.put(1, b2projplan7);
    LOJoin C7 = new LOJoin(lp, mm7, JOINTYPE.HASH, new boolean[] { true, true });
    C7.neverUseForRealSetSchema(jcschema7);
    lp.add(C7);
    lp.connect(A7, C7);
    lp.connect(B7, C7);

    assertFalse(C6.isEqual(C7));
}
 

@Test
public void testJoinDifferentNumJoinKeys() throws FrontendException {
    LogicalPlan lp = new LogicalPlan();

    // Test different join keys
    LogicalSchema jaschema6 = new LogicalSchema();
    jaschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad A6 = newLOLoad(new FileSpec("/abc",
            new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
                            "x", "y" })), jaschema6, lp, conf);
    lp.add(A6);

    // B = load
    LogicalSchema jbschema6 = new LogicalSchema();
    jbschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    jbschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "z", null, DataType.LONG));
    LOLoad B6 = newLOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema6, lp, conf);
    lp.add(B6);

    // C = join
    LogicalSchema jcschema6 = new LogicalSchema();
    jcschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    jcschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LogicalExpressionPlan aprojplan6 = new LogicalExpressionPlan();
    new ProjectExpression(aprojplan6, 0, 0, null);
    LogicalExpressionPlan bprojplan6 = new LogicalExpressionPlan();
    new ProjectExpression(bprojplan6, 1, 0, null);
    LogicalExpressionPlan b2projplan6 = new LogicalExpressionPlan();
    new ProjectExpression(b2projplan6, 1, 1, null);
    MultiMap<Integer, LogicalExpressionPlan> mm6 =
            new MultiMap<Integer, LogicalExpressionPlan>();
    mm6.put(0, aprojplan6);
    mm6.put(1, bprojplan6);
    mm6.put(1, b2projplan6);
    LOJoin C6 = new LOJoin(lp, mm6, JOINTYPE.HASH, new boolean[] { true, true });
    C6.neverUseForRealSetSchema(jcschema6);
    lp.add(C6);
    lp.connect(A6, C6);
    lp.connect(B6, C6);

    // Test different different number of join keys
    LogicalSchema jaschema8 = new LogicalSchema();
    jaschema8.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad A8 = newLOLoad(new FileSpec("/abc",
            new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
                            "x", "y" })), jaschema8, lp, conf);
    lp.add(A8);

    // B = load
    LogicalSchema jbschema8 = new LogicalSchema();
    jbschema8.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    jbschema8.addField(new LogicalSchema.LogicalFieldSchema(
            "z", null, DataType.LONG));
    LOLoad B8 = newLOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema8, lp, conf);
    lp.add(B8);

    // C = join
    LogicalSchema jcschema8 = new LogicalSchema();
    jcschema8.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    jcschema8.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LogicalExpressionPlan aprojplan8 = new LogicalExpressionPlan();
    new ProjectExpression(aprojplan8, 0, 0, null);
    LogicalExpressionPlan bprojplan8 = new LogicalExpressionPlan();
    new ProjectExpression(bprojplan8, 1, 0, null);
    MultiMap<Integer, LogicalExpressionPlan> mm8 =
            new MultiMap<Integer, LogicalExpressionPlan>();
    mm8.put(0, aprojplan8);
    mm8.put(1, bprojplan8);
    LOJoin C8 = new LOJoin(lp, mm8, JOINTYPE.HASH, new boolean[] { true, true });
    C8.neverUseForRealSetSchema(jcschema8);
    lp.add(C8);
    lp.connect(A8, C8);
    lp.connect(B8, C8);

    assertFalse(C6.isEqual(C8));
}
 

@Test
public void testRelationalSameOpDifferentPreds() throws FrontendException {
    LogicalPlan lp1 = new LogicalPlan();
    LogicalSchema aschema1 = new LogicalSchema();
    aschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad A1 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "y" })), aschema1, lp1,
            conf);
    lp1.add(A1);

    LogicalExpressionPlan fp1 = new LogicalExpressionPlan();
    ProjectExpression fy1 = new ProjectExpression(fp1, 0, 0, null);
    ConstantExpression fc1 = new ConstantExpression(fp1,
            new Integer(0));
    new EqualExpression(fp1, fy1, fc1);
    LOFilter D1 = new LOFilter(lp1, fp1);
    LogicalSchema cschema = new LogicalSchema();
    cschema.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    D1.neverUseForRealSetSchema(cschema);
    lp1.add(D1);
    lp1.connect(A1, D1);

    LogicalPlan lp2 = new LogicalPlan();
    LOLoad A2 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "z" })), null, lp2,
            conf);
    lp2.add(A2);

    LogicalExpressionPlan fp2 = new LogicalExpressionPlan();
    ProjectExpression fy2 = new ProjectExpression(fp2, 0, 0, null);
    ConstantExpression fc2 = new ConstantExpression(fp2,
            new Integer(0));
    new EqualExpression(fp2, fy2, fc2);
    LOFilter D2 = new LOFilter(lp2, fp2);
    D2.neverUseForRealSetSchema(cschema);
    lp2.add(D2);
    lp2.connect(A2, D2);

    assertTrue(D1.isEqual(D2));
}
 

@Before
public void setUp() throws Exception {
    // Build a plan that looks like it has just been transformed
    // It is roughly the logical plan for
    // A = load 'bla' as (x);
    // B = load 'morebla' as (y);
    // C = join A on x, B on y;
    // D = filter C by y > 0;
    // The plan is built with the filter having been pushed above the join
    // but the listners not yet having been called.
    // A = load
    lp = new LogicalPlan();
    LogicalSchema aschema = new LogicalSchema();
    aschema.addField(new LogicalSchema.LogicalFieldSchema(
        "x", null, DataType.INTEGER));
    aschema.getField(0).uid = 1;
    LOLoad A = new LOLoad(null, lp);
    A.neverUseForRealSetSchema(aschema);
    lp.add(A);

    // B = load
    LogicalSchema bschema = new LogicalSchema();
    bschema.addField(new LogicalSchema.LogicalFieldSchema(
        "y", null, DataType.INTEGER));
    bschema.getField(0).uid = 2;
    LOLoad B = new LOLoad(null, lp);
    B.neverUseForRealSetSchema(bschema);
    lp.add(B);

    // C = join
    LogicalSchema cschema = new LogicalSchema();
    cschema.addField(new LogicalSchema.LogicalFieldSchema(
        "x", null, DataType.INTEGER));
    cschema.addField(new LogicalSchema.LogicalFieldSchema(
        "y", null, DataType.INTEGER));
    cschema.getField(0).uid = 1;
    cschema.getField(1).uid = 2;

    MultiMap<Integer, LogicalExpressionPlan> mm =
        new MultiMap<Integer, LogicalExpressionPlan>();
    LOJoin C = new LOJoin(lp, mm, JOINTYPE.HASH, new boolean[] {true, true});

    LogicalExpressionPlan aprojplan = new LogicalExpressionPlan();
    ProjectExpression x = new ProjectExpression(aprojplan, 0, 0, C);
    x.neverUseForRealSetFieldSchema(new LogicalFieldSchema(null, null,
            DataType.INTEGER, 1));
    LogicalExpressionPlan bprojplan = new LogicalExpressionPlan();
    ProjectExpression y = new ProjectExpression(bprojplan, 1, 0, C);
    y.neverUseForRealSetFieldSchema(new LogicalFieldSchema(null, null,
            DataType.INTEGER, 2));
    mm.put(0, aprojplan);
    mm.put(1, bprojplan);

    C.neverUseForRealSetSchema(cschema);
    // Don't add it to the plan quite yet

    // D = filter
    LogicalExpressionPlan filterPlan = new LogicalExpressionPlan();
    LOFilter D = new LOFilter(lp, filterPlan);
    ProjectExpression fy = new ProjectExpression(filterPlan, 0, 1, D);
    fy.neverUseForRealSetFieldSchema(new LogicalFieldSchema(null, null,
            DataType.INTEGER, 2));
    ConstantExpression fc = new ConstantExpression(filterPlan, new Integer(0));
    new EqualExpression(filterPlan, fy, fc);

    D.neverUseForRealSetSchema(cschema);
    // Connect D to B, since the transform has happened.
    lp.add(D);
    lp.connect(B, D);

    // Now add in C, connected to A and D.
    lp.add(C);
    lp.connect(A, C);
    lp.connect(D, C);

    changedPlan = new LogicalPlan();
    changedPlan.add(D);
    changedPlan.add(C);
    changedPlan.connect(D, C);
}
 

@Test    
public void testForeachPlan2() throws Exception {
    String query = ("a = load 'd.txt' as (id, c:bag{t:(s,v)});" +
    "b = foreach a generate id, flatten(c);" +        
    "store b into 'empty';");  
    
    LogicalPlan newLogicalPlan = buildPlan(query);
    LogicalRelationalOperator ld =  (LogicalRelationalOperator)newLogicalPlan.getSources().get(0);
    LogicalRelationalOperator fe = (LogicalRelationalOperator)newLogicalPlan.getSuccessors(ld).get(0);
    LogicalSchema ls = fe.getSchema();
    assertEquals(1, ls.getField(0).uid);
    assertEquals(4, ls.getField(1).uid);
    assertEquals(5, ls.getField(2).uid);
    
    LogicalSchema expected = new LogicalSchema();
    expected.addField(new LogicalFieldSchema("id", null, DataType.BYTEARRAY));
    expected.addField(new LogicalFieldSchema("s", null, DataType.BYTEARRAY));
    expected.addField(new LogicalFieldSchema("v", null, DataType.BYTEARRAY));
    assertTrue(expected.isEqual(ls));
    
    
    PhysicalPlan phyPlan = translatePlan(newLogicalPlan);
    
    assertEquals(phyPlan.size(), 3);
    assertEquals(phyPlan.getLeaves().get(0).getClass(), POStore.class);
    POForEach foreach = (POForEach)phyPlan.getSuccessors(phyPlan.getRoots().get(0)).get(0);
    
    assertEquals(foreach.getInputPlans().size(), 2);
    
    PhysicalPlan inner = foreach.getInputPlans().get(0);
    assertEquals(inner.size(), 1);
    POProject prj = (POProject)inner.getRoots().get(0);
    assertEquals(prj.getColumn(), 0);
    assertEquals(prj.getInputs(), null);
    
    inner = foreach.getInputPlans().get(1);
    assertEquals(inner.size(), 1);
    prj = (POProject)inner.getRoots().get(0);
    assertEquals(prj.getColumn(), 1);
    assertEquals(prj.getInputs(), null);
    Boolean[] flat = foreach.getToBeFlattened().toArray(new Boolean[0]);
    assertFalse(flat[0]);
    assertTrue(flat[1]);
}