Java Code Examples for org.apache.pig.impl.PigContext#resolveClassName()

public Invoker(String fullName, String paramSpecsStr, String isStatic)
throws ClassNotFoundException, FrontendException, SecurityException, NoSuchMethodException {
    String className = fullName.substring(0, fullName.lastIndexOf('.'));
    String methodName = fullName.substring(fullName.lastIndexOf('.')+1);
    Class<?> klazz;
    try {
        klazz = PigContext.resolveClassName(className);
    } catch (IOException e) {
        // the amusing part is that PigContext throws this to wrap one of
        // the exceptions we declare!
        throw new FrontendException(e);
    }
    String[] paramSpecs = "".equals(paramSpecsStr) ? new String[0] : paramSpecsStr.split(" ");
    isStatic_ = "static".equalsIgnoreCase(isStatic) || "true".equals(isStatic);
    paramClasses_ = new Class<?>[paramSpecs.length];
    for (int i = 0; i < paramSpecs.length; i++) {
        paramClasses_[i] = stringToClass(paramSpecs[i]);
    }
    if (!isStatic_) {
        selfClass_ = paramClasses_[0];
    }
    method_ = klazz.getMethod(methodName, (isStatic_ ? paramClasses_ : dropFirstClass(paramClasses_)));
    returnType_ = method_.getGenericReturnType();
}
 

private void setOutputFormat(org.apache.hadoop.mapreduce.Job job) {
    // the OutputFormat we report to Hadoop is always PigOutputFormat which
    // can be wrapped with LazyOutputFormat provided if it is supported by
    // the Hadoop version and PigConfiguration.PIG_OUTPUT_LAZY is set
    if ("true".equalsIgnoreCase(job.getConfiguration().get(PigConfiguration.PIG_OUTPUT_LAZY))) {
        try {
            Class<?> clazz = PigContext
                    .resolveClassName("org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat");
            Method method = clazz.getMethod("setOutputFormatClass",
                    org.apache.hadoop.mapreduce.Job.class, Class.class);
            method.invoke(null, job, PigOutputFormatTez.class);
        } catch (Exception e) {
            job.setOutputFormatClass(PigOutputFormatTez.class);
            log.warn(PigConfiguration.PIG_OUTPUT_LAZY
                    + " is set but LazyOutputFormat couldn't be loaded. Default PigOutputFormat will be used");
        }
    } else {
        job.setOutputFormatClass(PigOutputFormatTez.class);
    }
}
 

public static void setOutputFormat(org.apache.hadoop.mapreduce.Job job) {
    // the OutputFormat we report to Hadoop is always PigOutputFormat which
    // can be wrapped with LazyOutputFormat provided if it is supported by
    // the Hadoop version and PigConfiguration.PIG_OUTPUT_LAZY is set
    if ("true".equalsIgnoreCase(job.getConfiguration().get(PigConfiguration.PIG_OUTPUT_LAZY))) {
        try {
            Class<?> clazz = PigContext
                    .resolveClassName("org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat");
            Method method = clazz.getMethod("setOutputFormatClass",
                    org.apache.hadoop.mapreduce.Job.class, Class.class);
            method.invoke(null, job, PigOutputFormat.class);
        } catch (Exception e) {
            job.setOutputFormatClass(PigOutputFormat.class);
            log.warn(PigConfiguration.PIG_OUTPUT_LAZY
                    + " is set but LazyOutputFormat couldn't be loaded. Default PigOutputFormat will be used");
        }
    } else {
        job.setOutputFormatClass(PigOutputFormat.class);
    }
}
 

public boolean isDeterministic() throws FrontendException{
    Class<?> udfClass;
    try {
        udfClass = PigContext.resolveClassName(getFuncSpec().getClassName());
    }catch(IOException ioe) {
        throw new FrontendException("Cannot instantiate: " + getFuncSpec(), ioe) ;
    }

    if (udfClass.getAnnotation(Nondeterministic.class) == null) {
        return true;
    }
    return false;

}
 

private void validateFuncSpec(SourceLocation loc, FuncSpec funcSpec, byte ft) throws RecognitionException {
    switch (ft) {
    case FunctionType.COMPARISONFUNC:
    case FunctionType.LOADFUNC:
    case FunctionType.STOREFUNC:
    case FunctionType.STREAMTOPIGFUNC:
    case FunctionType.PIGTOSTREAMFUNC:
        try{
            Class<?> func = PigContext.resolveClassName(funcSpec.getClassName());
            FunctionType.tryCasting(func, ft);
        } catch(Exception ex){
            throw new ParserValidationException(intStream, loc, ex);
        }
    }
}
 

private Class<?>[] getArgumentClassArray(String[] argumentTypes) {
    Class<?>[] arguments = new Class<?>[argumentTypes.length];
    for (int i = 0; i < argumentTypes.length; i++) {
        try {
            arguments[i]= nameToClassObjectMap.get(argumentTypes[i]);
            if (arguments[i] == null) {
                arguments[i] = PigContext.resolveClassName(argumentTypes[i]);
            }
        } catch (IOException e) {
            throw new RuntimeException("Unable to find class in PigContext: " + argumentTypes[i], e);
        }
    }
    return arguments;
}
 

static public void setCustomPartitioner(String customPartitioner, TezOperator tezOp) throws IOException {
    if (customPartitioner != null) {
        for (TezEdgeDescriptor edge : tezOp.inEdges.values()) {
            edge.partitionerClass = PigContext.resolveClassName(customPartitioner);
        }
    }
}
 

@Test
public void testClassCache() throws IOException {
    Class<?> c = PigContext.resolveClassName("TOMAP");
    Assert.assertEquals(TOMAP.class, c);
    
    c = PigContext.resolveClassName("TOMAP");
    Assert.assertEquals(TOMAP.class, c);
    
    c = PigContext.resolveClassName("org.apache.pig.builtin.REPLACE");
    Assert.assertEquals(REPLACE.class, c);
}
 

/**
 * Check if an operator is qualified to be under POForEach to turn on
 * accumulator. The operator must be in the following list or an
 * <code>POUserFunc</code>.
 *
 * If the operator has sub-operators, they must also belong to this list.
 * <li>ConstantExpression</li>
 * <li>POProject, whose result type is not BAG, or TUPLE and overloaded</li>
 * <li>POMapLookup</li>
 * <li>POCase</li>
 * <li>UnaryExpressionOperator</li>
 * <li>BinaryExpressionOperator</li>
 * <li>POBinCond</li>
 *
 * If the operator is <code>POUserFunc</code>, it must implement
 * <code>Accumulator</code> interface and its inputs pass the check by
 * calling <code>checkUDFInput()</code>
 *
 * @param po the operator to be checked on
 * @return <code>true</code> if it is ok, <code>false</code> if not.
 */
private static boolean check(PhysicalOperator po) {
    if (po instanceof ConstantExpression) {
        return true;
    }

    if (po instanceof POCast) {
        return check(po.getInputs().get(0));
    }

    if (po instanceof POMapLookUp) {
        return check(po.getInputs().get(0));
    }

    if (po instanceof POProject) {
        // POProject can not project data bag
        if (((POProject)po).getResultType() == DataType.BAG) {
            return false;
        }

        // POProject can not overload a data bag
        if (((POProject)po).getResultType() == DataType.TUPLE && ((POProject)po).isOverloaded()) {
            return false;
        }

        return true;
    }

    if (po instanceof UnaryExpressionOperator) {
        return check(((UnaryExpressionOperator)po).getExpr());
    }

    if (po instanceof BinaryExpressionOperator) {
        return check(((BinaryExpressionOperator)po).getLhs()) &&
                check(((BinaryExpressionOperator)po).getRhs());
    }

    if (po instanceof POBinCond) {
        return check(((POBinCond)po).getLhs()) &&
            check(((POBinCond)po).getRhs()) && check(((POBinCond)po).getCond());
    }

    if (po instanceof POUserFunc) {
        String className = ((POUserFunc)po).getFuncSpec().getClassName();
        @SuppressWarnings("rawtypes")
        Class c = null;
        try {
            c = PigContext.resolveClassName(className);
        } catch (Exception e) {
            return false;
        }
        if (!Accumulator.class.isAssignableFrom(c)) {
            return false;
        }

        // check input of UDF
        List<PhysicalOperator> inputs = po.getInputs();
        for (PhysicalOperator p: inputs) {
            if (!checkUDFInput(p)) {
                return false;
            }
        }

        return true;
    }

    return false;
}