Java Code Examples for org.apache.pig.tools.pigstats.PigStats#get()

private PigStats exec(String query) throws IOException {
    LOG.info("Query to run:\n" + query);
    List<PigProgressNotificationListener> listeners = ScriptState.get().getAllListeners();
    PigContext pc = scriptContext.getPigContext();
    ScriptState scriptState = pc.getExecutionEngine().instantiateScriptState();
    ScriptState.start(scriptState);
    ScriptState.get().setScript(query);
    for (PigProgressNotificationListener listener : listeners) {
        ScriptState.get().registerListener(listener);
    }
    PigServer pigServer = new PigServer(scriptContext.getPigContext(), false);
    pigServer.setBatchOn();
    GruntParser grunt = new GruntParser(new StringReader(query), pigServer);
    grunt.setInteractive(false);
    try {
        grunt.parseStopOnError(true);
    } catch (ParseException e) {
        throw new IOException("Failed to parse script " + e.getMessage(), e);
    }
    pigServer.executeBatch();
    return PigStats.get();
}
 

@Override
public PigStats call() throws Exception {
    LOG.info("Query to run:\n" + query);
    PigContext pc = scriptContext.getPigContext();
    ScriptState scriptState = pc.getExecutionEngine().instantiateScriptState();
    ScriptState.start(scriptState);
    ScriptState.get().setScript(query);
    ScriptState.get().registerListener(adaptor);
    PigServer pigServer = new PigServer(ctx, true);
    pigServer.setBatchOn();
    GruntParser grunt = new GruntParser(new StringReader(query), pigServer);
    grunt.setInteractive(false);
    try {
        grunt.parseStopOnError(true);
    } catch (ParseException e) {
        throw new IOException("Failed to parse script", e);
    }
    pigServer.executeBatch();
    return PigStats.get();
}
 

/**
 * Compile and execute the current plan.
 * @return
 * @throws IOException
 */
private PigStats execute() throws IOException {
    pigContext.getProperties().setProperty( PigContext.JOB_NAME, jobName );
    if( jobPriority != null ) {
        pigContext.getProperties().setProperty( PigContext.JOB_PRIORITY, jobPriority );
    }

    // In this plan, all stores in the plan will be executed. They should be ignored if the plan is reused.
    currDAG.countExecutedStores();

    currDAG.compile();

    if( currDAG.lp.size() == 0 ) {
       return PigStats.get();
    }

    pigContext.getProperties().setProperty("pig.logical.plan.signature", currDAG.lp.getSignature());

    PigStats stats = executeCompiledLogicalPlan();

    return stats;
}
 

/**
 * Starts collecting statistics for the given MR plan
 *
 * @param pc the Pig context
 * @param client the Hadoop job client
 * @param jcc the job compiler
 * @param plan the MR plan
 */
public static void startCollection(PigContext pc, JobClient client,
        JobControlCompiler jcc, MROperPlan plan) {
    SimplePigStats ps = (SimplePigStats)PigStats.get();
    ps.initialize(pc, client, jcc, plan);

    MRScriptState.get().emitInitialPlanNotification(plan);
    MRScriptState.get().emitLaunchStartedNotification(plan.size());
}
 

/**
 * Stops collecting statistics for a MR plan
 *
 * @param display if true, log collected statistics in the Pig log
 *      file at INFO level
 */
public static void stopCollection(boolean display) {
    SimplePigStats ps = (SimplePigStats)PigStats.get();
    ps.finish();
    if (!ps.isSuccessful()) {
        LOG.error(ps.getNumberFailedJobs() + " map reduce job(s) failed!");
        String errMsg = ps.getErrorMessage();
        if (errMsg != null) {
            LOG.error("Error message: " + errMsg);
        }
    }
    MRScriptState.get().emitLaunchCompletedNotification(
            ps.getNumberSuccessfulJobs());
    if (display) ps.display();
}
 

private void executeBatch() throws IOException {
    if (mPigServer.isBatchOn()) {
        if (mExplain != null) {
            explainCurrentBatch();
        }

        if (!mLoadOnly) {
            mPigServer.executeBatch();
            PigStats stats = PigStats.get();
            JobGraph jg = stats.getJobGraph();
            Iterator<JobStats> iter = jg.iterator();
            while (iter.hasNext()) {
                JobStats js = iter.next();
                if (!js.isSuccessful()) {
                    mNumFailedJobs++;
                    Exception exp = (js.getException() != null) ? js.getException()
                            : new ExecException(
                                    "Job " + (js.getJobId() == null ? "" : js.getJobId() + " ") +
                                    "failed, hadoop does not return any error message",
                                    2244);
                    LogUtils.writeLog(exp,
                            mPigServer.getPigContext().getProperties().getProperty("pig.logfile"),
                            log,
                            "true".equalsIgnoreCase(mPigServer.getPigContext().getProperties().getProperty("verbose")),
                            "Pig Stack Trace");
                } else {
                    mNumSucceededJobs++;
                }
            }
        }
    }
}
 

@Test
public void testRun_setsErrorThrowableOnPigStats() {
    File outputFile = null;
    try {
        String filename = this.getClass().getSimpleName() + "_" + "testRun_setsErrorThrowableOnPigStats";
        outputFile = File.createTempFile(filename, ".out");
        outputFile.delete();
        
        File scriptFile = File.createTempFile(filename, ".pig");
        BufferedWriter bw = new BufferedWriter(new FileWriter(scriptFile));
        bw.write("a = load 'test/org/apache/pig/test/data/passwd';\n");
        bw.write("b = group a by $0\n");
        bw.write("c = foreach b generate group, COUNT(a) as cnt;\n");
        bw.write("store c into 'out'\n");
        bw.close();
        
        Main.run(new String[]{"-x", "local", scriptFile.getAbsolutePath()}, null);
        PigStats stats = PigStats.get();
        
        Throwable t = stats.getErrorThrowable();
        assertTrue(t instanceof FrontendException);
        
        FrontendException fe = (FrontendException) t;
        SourceLocation sl = fe.getSourceLocation();
        assertEquals(2, sl.line());
        assertEquals(15, sl.offset());
        
        Throwable cause = fe.getCause();
        assertTrue(cause instanceof ParserException);
        
    } catch (Exception e) {
        log.error("Encountered exception", e);
        fail("Encountered Exception");
    }
}
 

@Test
public void testRun_setsErrorThrowableForParamSubstitution() {
    File outputFile = null;
    try {
        String filename = this.getClass().getSimpleName() + "_" + "testRun_setsErrorThrowableForParamSubstitution";
        outputFile = File.createTempFile(filename, ".out");
        outputFile.delete();

        File scriptFile = File.createTempFile(filename, ".pig");
        BufferedWriter bw = new BufferedWriter(new FileWriter(scriptFile));
        bw.write("a = load '$NOEXIST';\n");
        bw.write("b = group a by $0\n");
        bw.write("c = foreach b generate group, COUNT(a) as cnt;\n");
        bw.write("store c into 'out'\n");
        bw.close();
        
        Main.run(new String[]{"-x", "local", scriptFile.getAbsolutePath()}, null);
        PigStats stats = PigStats.get();
        
        Throwable t = stats.getErrorThrowable();
        assertTrue(t instanceof IOException);
        
        Throwable cause = t.getCause();
        assertTrue(cause instanceof ParameterSubstitutionException);

        ParameterSubstitutionException pse = (ParameterSubstitutionException) cause;
        assertTrue(pse.getMessage().contains("NOEXIST"));
    } catch (Exception e) {
        log.error("Encountered exception", e);
        fail("Encountered Exception");
    }
}
 

@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
  // '-' is invalid for pig alias
  String alias = "paragraph_" + context.getParagraphId().replace("-", "_");
  String[] lines = st.split("\n");
  List<String> queries = new ArrayList<>();
  for (int i = 0; i < lines.length; ++i) {
    if (i == lines.length - 1) {
      lines[i] = alias + " = " + lines[i];
    }
    queries.add(lines[i]);
  }

  StringBuilder resultBuilder = new StringBuilder("%table ");
  try {
    pigServer.setJobName(createJobName(st, context));
    File tmpScriptFile = PigUtils.createTempPigScript(queries);
    // each thread should its own ScriptState & PigStats
    ScriptState.start(pigServer.getPigContext().getExecutionEngine().instantiateScriptState());
    // reset PigStats, otherwise you may get the PigStats of last job in the same thread
    // because PigStats is ThreadLocal variable
    PigStats.start(pigServer.getPigContext().getExecutionEngine().instantiatePigStats());
    PigScriptListener scriptListener = new PigScriptListener();
    ScriptState.get().registerListener(scriptListener);
    listenerMap.put(context.getParagraphId(), scriptListener);
    pigServer.registerScript(tmpScriptFile.getAbsolutePath());
    Schema schema = pigServer.dumpSchema(alias);
    boolean schemaKnown = (schema != null);
    if (schemaKnown) {
      for (int i = 0; i < schema.size(); ++i) {
        Schema.FieldSchema field = schema.getField(i);
        resultBuilder.append(field.alias != null ? field.alias : "col_" + i);
        if (i != schema.size() - 1) {
          resultBuilder.append("\t");
        }
      }
      resultBuilder.append("\n");
    }
    Iterator<Tuple> iter = pigServer.openIterator(alias);
    boolean firstRow = true;
    int index = 0;
    while (iter.hasNext() && index < maxResult) {
      index++;
      Tuple tuple = iter.next();
      if (firstRow && !schemaKnown) {
        for (int i = 0; i < tuple.size(); ++i) {
          resultBuilder.append("c_" + i + "\t");
        }
        resultBuilder.append("\n");
        firstRow = false;
      }
      resultBuilder.append(StringUtils.join(tuple.iterator(), "\t"));
      resultBuilder.append("\n");
    }
    if (index >= maxResult && iter.hasNext()) {
      resultBuilder.append("\n");
      resultBuilder.append(ResultMessages.getExceedsLimitRowsMessage(maxResult, MAX_RESULTS));
    }
  } catch (IOException e) {
    // Extract error in the following order
    // 1. catch FrontendException, FrontendException happens in the query compilation phase.
    // 2. catch ParseException for syntax error
    // 3. PigStats, This is execution error
    // 4. Other errors.
    if (e instanceof FrontendException) {
      FrontendException fe = (FrontendException) e;
      if (!fe.getMessage().contains("Backend error :")) {
        LOGGER.error("Fail to run pig query.", e);
        return new InterpreterResult(Code.ERROR, ExceptionUtils.getStackTrace(e));
      }
    }
    if (e.getCause() instanceof ParseException) {
      return new InterpreterResult(Code.ERROR, e.getMessage());
    }
    PigStats stats = PigStats.get();
    if (stats != null) {
      String errorMsg = stats.getDisplayString();
      if (errorMsg != null) {
        return new InterpreterResult(Code.ERROR, errorMsg);
      }
    }
    LOGGER.error("Fail to run pig query.", e);
    return new InterpreterResult(Code.ERROR, ExceptionUtils.getStackTrace(e));
  } finally {
    listenerMap.remove(context.getParagraphId());
  }
  return new InterpreterResult(Code.SUCCESS, resultBuilder.toString());
}
 

/**
 * Updates the {@link JobGraph} of the {@link PigStats}. The initial
 * {@link JobGraph} is created without job ids using {@link MROperPlan},
 * before any job is submitted for execution. The {@link JobGraph} then
 * is updated with job ids after jobs are executed.
 *
 * @param jobMroMap the map that maps {@link Job}s to {@link MapReduceOper}s
 */
public static void updateJobMroMap(Map<Job, MapReduceOper> jobMroMap) {
    SimplePigStats ps = (SimplePigStats)PigStats.get();
    for (Map.Entry<Job, MapReduceOper> entry : jobMroMap.entrySet()) {
        MapReduceOper mro = entry.getValue();
        ps.mapMROperToJob(mro, entry.getKey());
    }
}
 

/**
 * Add stats for a new Job, which doesn't yet need to be completed.
 *
 * @param job the job being run
 * @return JobStats for the job
 */
public static JobStats addJobStats(Job job) {
    SimplePigStats ps = (SimplePigStats)PigStats.get();
    return ps.addMRJobStats(job);
}