org.apache.pig.impl.logicalLayer.schema.Schema#getPigSchema

Source File: TestResourceSchema.java From spork with Apache License 2.0

6 votes

/**
 * Test that ResourceSchema is correctly created given a
 * pig.Schema and vice versa 
 */
@Test
public void testResourceFlatSchemaCreation() 
throws ExecException, SchemaMergeException, FrontendException {
    String [] aliases ={"f1", "f2"};
    byte[] types = {DataType.CHARARRAY, DataType.INTEGER};
    Schema origSchema = TypeCheckingTestUtil.genFlatSchema(
            aliases,types);
    ResourceSchema rsSchema = new ResourceSchema(origSchema);
    assertEquals("num fields", aliases.length, rsSchema.getFields().length);
    ResourceSchema.ResourceFieldSchema[] fields = rsSchema.getFields();
    for (int i=0; i<fields.length; i++) {
        assertEquals(fields[i].getName(), aliases[i]);
        assertEquals(fields[i].getType(), types[i]);
    }
    Schema genSchema = Schema.getPigSchema(rsSchema);
    assertTrue("generated schema equals original", 
            Schema.equals(genSchema, origSchema, true, false));
}

Source File: TestResourceSchema.java From spork with Apache License 2.0

6 votes

/**
 * Test that ResourceSchema is correctly created given a
 * pig.Schema and vice versa 
 */
@Test
public void testResourceFlatSchemaCreation2() 
throws ExecException, SchemaMergeException, FrontendException {
    String [] aliases ={"f1", "f2"};
    byte[] types = {DataType.CHARARRAY, DataType.INTEGER};
    
    Schema origSchema = new Schema(
            new Schema.FieldSchema("t1", 
                    new Schema(
                            new Schema.FieldSchema("t0", 
                                    TypeCheckingTestUtil.genFlatSchema(
                                            aliases,types), 
                                            DataType.TUPLE)), DataType.BAG));
                    
    ResourceSchema rsSchema = new ResourceSchema(origSchema);

    Schema genSchema = Schema.getPigSchema(rsSchema);
    assertTrue("generated schema equals original", 
            Schema.equals(genSchema, origSchema, true, false));
}

Source File: TestResourceSchema.java From spork with Apache License 2.0

6 votes

/**
 * Test one-level Pig Schema: multiple fields for a bag
 */
@Test
public void testResourceSchemaWithInvalidPigSchema() 
throws FrontendException {
    String [] aliases ={"f1", "f2"};
    byte[] types = {DataType.CHARARRAY, DataType.INTEGER};
    Schema level0 = TypeCheckingTestUtil.genFlatSchema(
            aliases,types);
    Schema.FieldSchema fld0 = 
        new Schema.FieldSchema("f0", level0, DataType.BAG);
    Schema level1 = new Schema(fld0);
    try {
        Schema.getPigSchema(new ResourceSchema(level1));
        Assert.fail();
    } catch(FrontendException e) {
        assertTrue(e.getErrorCode()==2218);
    }
}

Source File: TestResourceSchema.java From spork with Apache License 2.0

6 votes

/**
 * Test one-level Pig Schema: bag without tuple field
 */
@Test
public void testResourceSchemaWithInvalidPigSchema2() 
throws FrontendException {
    String [] aliases ={"f1"};
    byte[] types = {DataType.INTEGER};
    Schema level0 = TypeCheckingTestUtil.genFlatSchema(
            aliases,types);
    Schema.FieldSchema fld0 = 
        new Schema.FieldSchema("f0", level0, DataType.BAG);
    Schema level1 = new Schema(fld0);
    try {
        Schema.getPigSchema(new ResourceSchema(level1));
        Assert.fail();
    } catch (FrontendException e) {
        assertTrue(e.getErrorCode()==2218);
    }
}

Source File: VespaStorage.java From vespa with Apache License 2.0

5 votes

private String createDocumentOperation(Tuple tuple) throws IOException {
    if (tuple == null || tuple.size() == 0) {
        return null;
    }
    if (resourceSchema == null) {
        return null;
    }

    Map<String, Object> fields = TupleTools.tupleMap(resourceSchema, tuple);
    String docId = TupleTools.toString(fields, template);

    Schema schema = Schema.getPigSchema(resourceSchema);
    return VespaDocumentOperation.create(operation, docId, fields, properties, schema);
}

Source File: LOLoad.java From spork with Apache License 2.0

5 votes

private LogicalSchema getSchemaFromMetaData() throws FrontendException {
    if (getLoadFunc()!=null && getLoadFunc() instanceof LoadMetadata) {
        try {
            ResourceSchema resourceSchema = ((LoadMetadata)loadFunc).getSchema(getFileSpec().getFileName(), new Job(conf));
            Schema oldSchema = Schema.getPigSchema(resourceSchema);
            return Util.translateSchema(oldSchema);
        } catch (IOException e) {
            throw new FrontendException( this, "Cannot get schema from loadFunc " + loadFunc.getClass().getName(), 2245, e);
        }
    }
    return null;
}

Source File: JythonFunction.java From spork with Apache License 2.0

5 votes

@Override
public Schema outputSchema(Schema input) {
    if(schema != null) {
        return schema;
    } else {
        if(outputSchemaFunc != null) {
            PyFunction pf;
            try {
                pf = JythonScriptEngine.getFunction(scriptFilePath, outputSchemaFunc);
                // this should be a schema function
                PyObject schemaFunctionDef = pf.__findattr__("schemaFunction".intern());
                if(schemaFunctionDef == null) {
                    throw new IllegalStateException("Function: "
                            + outputSchemaFunc + " is not a schema function");
                }
                Object newSchema = ((pf.__call__(Py.java2py(input))).__tojava__(Object.class));
                if (newSchema instanceof ResourceSchema) {
                    return(Schema.getPigSchema((ResourceSchema) newSchema));
                }
                else if (newSchema instanceof Schema) {
                    return (Schema) newSchema;
                }
                else {
                    return Utils.getSchemaFromString(newSchema.toString());
                }
            } catch (IOException ioe) {
                throw new IllegalStateException("Could not find function: "
                    + outputSchemaFunc + "()", ioe);
            }
        } else {
            return new Schema(new Schema.FieldSchema(null, DataType.BYTEARRAY));
        }
    }
}

Source File: TestResourceSchema.java From spork with Apache License 2.0

5 votes

/**
 * Test that ResourceSchema is correctly with SortInfo
 */
@Test
public void testResourceFlatSchemaCreationWithSortInfo() 
throws ExecException, SchemaMergeException, FrontendException {
    String [] aliases ={"f1", "f2"};
    byte[] types = {DataType.CHARARRAY, DataType.INTEGER};
    
    Schema origSchema = new Schema(
            new Schema.FieldSchema("t1", 
                    new Schema(
                            new Schema.FieldSchema("t0", 
                                    TypeCheckingTestUtil.genFlatSchema(
                                            aliases,types), 
                                            DataType.TUPLE)), DataType.BAG));
    List<SortColInfo> colList = new ArrayList<SortColInfo>();
    SortColInfo col1 = new SortColInfo("f1", 0, SortColInfo.Order.ASCENDING);
    SortColInfo col2 = new SortColInfo("f1", 1, SortColInfo.Order.DESCENDING);
    colList.add(col1);
    colList.add(col2);
    SortInfo sortInfo = new SortInfo(colList);
                    
    ResourceSchema rsSchema = new ResourceSchema(origSchema, sortInfo);

    Schema genSchema = Schema.getPigSchema(rsSchema);
    assertTrue("generated schema equals original", 
            Schema.equals(genSchema, origSchema, true, false));
    assertTrue(rsSchema.getSortKeys()[0]==0);
    assertTrue(rsSchema.getSortKeys()[1]==1);
    assertTrue(rsSchema.getSortKeyOrders()[0]==ResourceSchema.Order.ASCENDING);
    assertTrue(rsSchema.getSortKeyOrders()[1]==ResourceSchema.Order.DESCENDING);
}

Source File: TestSchema.java From spork with Apache License 2.0

4 votes

@Test
public void testResourceSchemaToSchema() throws ParserException,FrontendException{
    Schema s1 = Utils.getSchemaFromString("b:bag{t:tuple(name:chararray,age:int)}");
    Schema s2 = Schema.getPigSchema(new ResourceSchema(s1));
    assertEquals(s1, s2);
}

Source File: TestResourceSchema.java From spork with Apache License 2.0

4 votes

/**
 * Test that Pig Schema is correctly created given a
 * ResourceSchema and vice versa. Test also that 
 * TwoLevelAccess flag is set for Pig Schema when needed.
 * @throws IOException 
 */
@Test
public void testToPigSchemaWithTwoLevelAccess() throws IOException {
    ResourceFieldSchema[] level0 = 
        new ResourceFieldSchema[] {
            new ResourceFieldSchema()
                .setName("fld0").setType(DataType.CHARARRAY),
            new ResourceFieldSchema()
                .setName("fld1").setType(DataType.DOUBLE),
            new ResourceFieldSchema()
                .setName("fld2").setType(DataType.INTEGER)
    };
           
    ResourceSchema rSchema0 = new ResourceSchema()
        .setFields(level0);
    
    ResourceFieldSchema[] level1 = 
        new ResourceFieldSchema[] {
            new ResourceFieldSchema()
                .setName("t1").setType(DataType.TUPLE)
                .setSchema(rSchema0)
    };
    
    ResourceSchema rSchema1 = new ResourceSchema()
        .setFields(level1);
    
    ResourceFieldSchema[] level2 = 
        new ResourceFieldSchema[] {
            new ResourceFieldSchema()
                .setName("t2").setType(DataType.BAG)
                .setSchema(rSchema1)
    };
    
    ResourceSchema origSchema = new ResourceSchema()
        .setFields(level2);        
    
    Schema pSchema = Schema.getPigSchema(origSchema);
            
    assertTrue(!CheckTwoLevelAccess(pSchema));
            
    assertTrue(ResourceSchema.equals(origSchema, new ResourceSchema(pSchema)));
}

Java Code Examples for org.apache.pig.impl.logicalLayer.schema.Schema#getPigSchema()