Java Code Examples for org.apache.pig.impl.PigContext#instantiateFuncFromSpec()

private LogicalPlan genNewLoadStorePlan(String inputFile,
                                        String outputFile, DataStorage dfs)
                                    throws Throwable {
    LogicalPlan plan = new LogicalPlan() ;
    FileSpec filespec1 =
        new FileSpec(inputFile, new FuncSpec("org.apache.pig.builtin.PigStorage")) ;
    FileSpec filespec2 =
        new FileSpec(outputFile, new FuncSpec("org.apache.pig.builtin.PigStorage"));
    LOLoad load = newLOLoad( filespec1, null, plan,
            ConfigurationUtil.toConfiguration(dfs.getConfiguration())) ;
    LOStore store = new LOStore(plan, filespec2, (StoreFuncInterface)PigContext.instantiateFuncFromSpec(filespec2.getFuncSpec()), null) ;

    plan.add(load) ;
    plan.add(store) ;

    plan.connect(load, store) ;

    return plan ;
}
 

private void instantiateFunc(FuncSpec fSpec) {
    this.func = (EvalFunc) PigContext.instantiateFuncFromSpec(fSpec);
    this.setSignature(signature);
    this.setFuncInputSchema(signature);
    if (func.getClass().isAnnotationPresent(MonitoredUDF.class)) {
        executor = new MonitoredUDFExecutor(func);
    }
    //the next couple of initializations do not work as intended for the following reasons
    //the reporter and pigLogger are member variables of PhysicalOperator
    //when instanitateFunc is invoked at deserialization time, both
    //reporter and pigLogger are null. They are set during map and reduce calls,
    //making the initializations here basically useless. Look at the processInput
    //method where these variables are re-initialized. At that point, the PhysicalOperator
    //is set up correctly with the reporter and pigLogger references
    this.func.setReporter(getReporter());
    this.func.setPigLogger(pigLogger);
}
 

private void seekInRightStream(Object firstLeftKey) throws IOException{
    rightLoader = (LoadFunc)PigContext.instantiateFuncFromSpec(rightLoaderFuncSpec);

    // check if hadoop distributed cache is used
    if (indexFile != null && rightLoader instanceof DefaultIndexableLoader) {
        DefaultIndexableLoader loader = (DefaultIndexableLoader)rightLoader;
        loader.setIndexFile(indexFile);
    }
    
    // Pass signature of the loader to rightLoader
    // make a copy of the conf to use in calls to rightLoader.
    rightLoader.setUDFContextSignature(signature);
    Job job = new Job(new Configuration(PigMapReduce.sJobConfInternal.get()));
    rightLoader.setLocation(rightInputFileName, job);
    ((IndexableLoadFunc)rightLoader).initialize(job.getConfiguration());
    ((IndexableLoadFunc)rightLoader).seekNear(
            firstLeftKey instanceof Tuple ? (Tuple)firstLeftKey : mTupleFactory.newTuple(firstLeftKey));
}
 

private void initRightLoader(int [] splitsToBeRead) throws IOException{
    PigContext pc = (PigContext) ObjectSerializer
            .deserialize(PigMapReduce.sJobConfInternal.get().get("pig.pigContext"));
    
    Configuration conf = ConfigurationUtil.toConfiguration(pc.getProperties());
    
    // Hadoop security need this property to be set
    if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) {
        conf.set(MRConfiguration.JOB_CREDENTIALS_BINARY, 
                System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
    }
    
    //create ReadToEndLoader that will read the given splits in order
    loader = new ReadToEndLoader((LoadFunc)PigContext.instantiateFuncFromSpec(rightLoaderFuncSpec),
            conf, inpLocation, splitsToBeRead);
}
 

/**
 * This helper function instantiates an EvalFunc given its String class name.
 */
private EvalFunc<?> makeEvalFunc(String base) {
    StringBuffer sb = new StringBuffer();
    sb.append(base).append("(");

    boolean first = true;
    for (String s : constructorArgs) {
        if (!first) sb.append(",");
        else first = false;
        sb.append("'").append(s).append("'");
    }

    sb.append(")");

    return (EvalFunc<?>)PigContext.instantiateFuncFromSpec(sb.toString());
}
 

@Test
public void testSpecCtorClassName() {
    String pigStorage = PigStorage.class.getName();
    FuncSpec fs = new FuncSpec(pigStorage);
    Object o = PigContext.instantiateFuncFromSpec(fs);
    assertTrue(o instanceof PigStorage);
}
 

@Test
public void testCtorClassNameArgs() {
    String dummy = DummyClass.class.getName();
    String[] args = new String[]{":"};
    FuncSpec fs = new FuncSpec(dummy, args);
    Object o = PigContext.instantiateFuncFromSpec(fs);
    assertTrue(o instanceof DummyClass);
    assertEquals((byte)':', ((DummyClass)o).delim);
}
 

@Override
public void visitCast(POCast cast) {
    if (cast.getFuncSpec()!=null) {
        Object obj = PigContext.instantiateFuncFromSpec(cast.getFuncSpec());
        if (obj instanceof LoadFunc) {
            LoadFunc loadFunc = (LoadFunc)obj;
            if (loadFunc.getCacheFiles()!=null) {
                cacheFiles.addAll(loadFunc.getCacheFiles());
            }
            if (loadFunc.getShipFiles()!=null) {
                shipFiles.addAll(loadFunc.getShipFiles());
            }
        }
    }
}
 

@Test
public void testInstantiate5() throws Throwable {
    try {
        PigContext.instantiateFuncFromSpec(TEST_CLASS2 + "('a','b','c')") ;
        fail("expect error here") ;
    } catch(Exception e) {
        // good
    }
}
 

@SuppressWarnings("unchecked")
private void instantiateFunc() {
    if(mUserComparisonFunc != null) {
        this.mUserComparisonFunc = (ComparisonFunc) PigContext.instantiateFuncFromSpec(this.mUserComparisonFuncSpec);
        this.mUserComparisonFunc.setReporter(reporter);
        this.mComparator = mUserComparisonFunc;
    }
}
 

public LoadFunc getLoadFunc(){
    if (this.loader==null) {
        this.loader = (LoadFunc)PigContext.instantiateFuncFromSpec(lFile.getFuncSpec());
        this.loader.setUDFContextSignature(signature);
    }
    return this.loader;
}
 

String buildStoreOp(SourceLocation loc, String alias, String inputAlias, String filename, FuncSpec funcSpec)
throws ParserValidationException {
    try {
        // Load StoreFunc class from default properties if funcSpec is null. Fallback on PigStorage if StoreFunc is not specified in properties.
        funcSpec = funcSpec == null ? new FuncSpec(pigContext.getProperties().getProperty(
                PigConfiguration.PIG_DEFAULT_STORE_FUNC, PigStorage.class.getName())) : funcSpec;
        StoreFuncInterface stoFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec(funcSpec);
        String fileNameKey = inputAlias + "_" + (storeIndex++) ;

        String signature = inputAlias + "_" + newOperatorKey();
        stoFunc.setStoreFuncUDFContextSignature(signature);

        String absolutePath = fileNameMap.get(fileNameKey);
        if (absolutePath == null) {
            absolutePath = stoFunc.relToAbsPathForStoreLocation(
                    filename,
                    QueryParserUtils.getCurrentDir(pigContext));
            if (absolutePath!=null) {
                QueryParserUtils.setHdfsServers(absolutePath, pigContext);
            }
            fileNameMap.put(fileNameKey, absolutePath);
        }
        FileSpec fileSpec = new FileSpec(absolutePath, funcSpec);

        LOStore op = new LOStore(plan, fileSpec, stoFunc, signature);
        return buildOp(loc, op, alias, inputAlias, null);
    } catch(Exception ex) {
        throw new ParserValidationException(intStream, loc, ex);
    }
}
 

/**
 * @param inputIndex
 * @param conf
 * @return
 * @throws IOException
 */
@SuppressWarnings("unchecked")
private static LoadFunc getLoadFunc(int inputIndex, Configuration conf) throws IOException {
    ArrayList<FileSpec> inputs =
            (ArrayList<FileSpec>) ObjectSerializer.deserialize(
                    conf.get(PIG_INPUTS));
    FuncSpec loadFuncSpec = inputs.get(inputIndex).getFuncSpec();
    return (LoadFunc) PigContext.instantiateFuncFromSpec(loadFuncSpec);
}
 

public SampleLoader(String funcSpec) {
    funcSpec = funcSpec.replaceAll("\\\\'", "'");
    loader = (LoadFunc)PigContext.instantiateFuncFromSpec(funcSpec);
}
 

@SuppressWarnings("unchecked")
@Override
public void visit( UserFuncExpression op ) throws FrontendException {       
    Object f = PigContext.instantiateFuncFromSpec(op.getFuncSpec());
    PhysicalOperator p;
    if (f instanceof EvalFunc) {
        p = new POUserFunc(new OperatorKey(DEFAULT_SCOPE, nodeGen
                .getNextNodeId(DEFAULT_SCOPE)), -1,
                null, op.getFuncSpec(), (EvalFunc) f);
        ((POUserFunc)p).setSignature(op.getSignature());
        //reinitialize input schema from signature
        if (((POUserFunc)p).getFunc().getInputSchema() == null) {
            ((POUserFunc)p).setFuncInputSchema(op.getSignature());
        }
        List<String> cacheFiles = ((EvalFunc)f).getCacheFiles();
        if (cacheFiles != null) {
            ((POUserFunc)p).setCacheFiles(cacheFiles);
        }
    } else {
        p = new POUserComparisonFunc(new OperatorKey(DEFAULT_SCOPE, nodeGen
                .getNextNodeId(DEFAULT_SCOPE)), -1,
                null, op.getFuncSpec(), (ComparisonFunc) f);
    }
    p.setResultType(op.getType());
    currentPlan.add(p);
    List<LogicalExpression> fromList = op.getArguments();
    if(fromList!=null){
        for (LogicalExpression inputOperator : fromList) {
            PhysicalOperator from = logToPhyMap.get(inputOperator);
            try {
                currentPlan.connect(from, p);
            } catch (PlanException e) {
                int errCode = 2015;
                String msg = "Invalid physical operators in the physical plan" ;
                throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);
            }
        }
    }
    logToPhyMap.put(op, p);
    
    //We need to track all the scalars
    if( op instanceof ScalarExpression ) {
        Operator refOp = ((ScalarExpression)op).getImplicitReferencedOperator();
        ((POUserFunc)p).setReferencedOperator( logToPhyMap.get( refOp ) );
    }

}
 

@Test
public void testInstantiate3() throws Throwable {
    PigContext.instantiateFuncFromSpec(TEST_CLASS + "('a','b','c')") ;
}
 

@Test
public void testInstantiate2() throws Throwable {
    PigContext.instantiateFuncFromSpec(TEST_CLASS + "('a','b')") ;
}
 

public DefaultInputHandler(HandleSpec spec) {
    serializer = (PigToStream)PigContext.instantiateFuncFromSpec(spec.spec);
}
 

@Test
public void testHadoopExceptionCreation() throws Exception {
    Object object = PigContext
            .instantiateFuncFromSpec("org.apache.hadoop.mapred.FileAlreadyExistsException");
    assertTrue(object instanceof FileAlreadyExistsException);
}
 

@Override
public LogicalSchema.LogicalFieldSchema getFieldSchema() throws FrontendException {
    if (fieldSchema!=null)
        return fieldSchema;

    LogicalSchema inputSchema = new LogicalSchema();
    List<Operator> succs = plan.getSuccessors(this);

    if (succs!=null) {
        for(Operator lo : succs){
            if (((LogicalExpression)lo).getFieldSchema()==null) {
                inputSchema = null;
                break;
            }
            inputSchema.addField(((LogicalExpression)lo).getFieldSchema());
        }
    }

    if (lazilyInitializeInvokerFunction) {
        initializeInvokerFunction();
    }

    // Since ef only set one time, we never change its value, so we can optimize it by instantiate only once.
    // This significantly optimize the performance of frontend (PIG-1738)
    if (ef==null) {
        ef = (EvalFunc<?>) PigContext.instantiateFuncFromSpec(mFuncSpec);
    }

    ef.setUDFContextSignature(signature);
    Properties props = UDFContext.getUDFContext().getUDFProperties(ef.getClass());
    Schema translatedInputSchema = Util.translateSchema(inputSchema);
    if(translatedInputSchema != null) {
        props.put("pig.evalfunc.inputschema."+signature, translatedInputSchema);
    }
    // Store inputSchema into the UDF context
    ef.setInputSchema(translatedInputSchema);

    Schema udfSchema = ef.outputSchema(translatedInputSchema);
    if (udfSchema != null && udfSchema.size() > 1) {
        throw new FrontendException("Given UDF returns an improper Schema. Schema should only contain one field of a Tuple, Bag, or a single type. Returns: " + udfSchema);
    }

    //TODO appendability should come from a setting
    SchemaTupleFrontend.registerToGenerateIfPossible(translatedInputSchema, false, GenContext.UDF);
    SchemaTupleFrontend.registerToGenerateIfPossible(udfSchema, false, GenContext.UDF);

    if (udfSchema != null) {
        Schema.FieldSchema fs;
        if(udfSchema.size() == 0) {
            fs = new Schema.FieldSchema(null, null, DataType.findType(ef.getReturnType()));
        } else if(udfSchema.size() == 1) {
            fs = new Schema.FieldSchema(udfSchema.getField(0));
        } else {
            fs = new Schema.FieldSchema(null, udfSchema, DataType.TUPLE);
        }
        fieldSchema = Util.translateFieldSchema(fs);
        fieldSchema.normalize();
    } else {
        fieldSchema = new LogicalSchema.LogicalFieldSchema(null, null, DataType.findType(ef.getReturnType()));
    }
    uidOnlyFieldSchema = fieldSchema.mergeUid(uidOnlyFieldSchema);
    return fieldSchema;
}