org.apache.oozie.client.WorkflowJob Java Examples

/**
 * Copy WorkflowJob information in OOZIE to Oozie we defined
 * @param job WorkflowJob
 * @return OozieJob
 */
public static OozieJob setWorkflowjob(WorkflowJob job) {
	OozieJob oozieJob = new OozieJob();
	oozieJob.setAcl(job.getAcl());
	for (WorkflowAction temp : job.getActions()) {
		OozieAction a = asAction(temp, job.getId() );
		oozieJob.addAction(a);
	}
	oozieJob.setAppName(job.getAppName());
	oozieJob.setAppPath(job.getAppPath());
	oozieJob.setConf(job.getConf());
	oozieJob.setConsoleUrl(job.getConsoleUrl());
	oozieJob.setCreatedTime(job.getCreatedTime());
	oozieJob.setEndTime(job.getEndTime());
	oozieJob.setExternalId(job.getExternalId());
	oozieJob.setGroup(job.getGroup());
	oozieJob.setId(job.getId());
	oozieJob.setLastModifiedTime(job.getLastModifiedTime());
	oozieJob.setParentId(job.getParentId());
	oozieJob.setRun(job.getRun());
	oozieJob.setStartTime(job.getStartTime());
	oozieJob.setStatus(job.getStatus().toString());
	oozieJob.setUser(job.getUser());
	return oozieJob;
}
 

@Override
public boolean didSucceed() throws OozieClientException {
  try {
    return oozieClient.getJobInfo( id ).getStatus().equals( WorkflowJob.Status.SUCCEEDED );
  } catch ( org.apache.oozie.client.OozieClientException e ) {
    throw new OozieClientException( e, e.getErrorCode() );
  }
}
 

@Override
public boolean isRunning() throws OozieClientException {
  try {
    return oozieClient.getJobInfo( id ).getStatus().equals( WorkflowJob.Status.RUNNING );
  } catch ( org.apache.oozie.client.OozieClientException e ) {
    throw new OozieClientException( e, e.getErrorCode() );
  }
}
 

@Override
public ExternalStatus getStatus(SlotID unused, String jobId) throws ExternalServiceException {
    try {
        WorkflowJob jobInfo = client.getJobInfo(jobId);
        String status = jobInfo.getStatus().toString();
        return new OozieExternalStatus(status);
    } catch (OozieClientException e) {
        throw new ExternalServiceException(e);
    }
}
 

@Test
public void testNotificationSent() throws OozieClientException, MessagingException {
  WorkflowJob job = mock(WorkflowJob.class);
  when(oozieClient.getJobInfo(JOB_ID)).thenReturn(job);
  when(job.getId()).thenReturn(JOB_ID);
  when(job.getCreatedTime()).thenReturn(new Date());
  when(job.getConf())
    .thenReturn(
      "<configuration>"
        + "<property><name>"
        + Constants.USER_PROPERTY
        + "</name>"
        + "<value>test</value></property>"

        + "<property><name>"
        + Constants.NOTIFICATION_PROPERTY
        + "</name>"
        + "<value>[email protected]</value></property>"

        + "<property><name>"
        + Constants.FILTER_PROPERTY
        + "</name>"
        + "<value>{\"type\":\"equals\",\"key\":\"DATASET_KEY\",\"value\":\"8575f23e-f762-11e1-a439-00145eb45e9a\"}</value></property>"
        + "</configuration>");

  service.processCallback(JOB_ID, SUCCEEDED);
}
 

@Test
public void testSubmitWorkflow() throws Exception {

    LOG.info("OOZIE: Test Submit Workflow Start");

    FileSystem hdfsFs = hdfsLocalCluster.getHdfsFileSystemHandle();
    OozieClient oozie = oozieLocalServer.getOozieClient();

    Path appPath = new Path(hdfsFs.getHomeDirectory(), "testApp");
    hdfsFs.mkdirs(new Path(appPath, "lib"));
    Path workflow = new Path(appPath, "workflow.xml");

    // Setup input directory and file
    hdfsFs.mkdirs(new Path(TEST_INPUT_DIR));
    hdfsFs.copyFromLocalFile(
            new Path(getClass().getClassLoader().getResource(TEST_INPUT_FILE).toURI()), new Path(TEST_INPUT_DIR));

    //write workflow.xml
    String wfApp = "<workflow-app name=\"sugar-option-decision\" xmlns=\"uri:oozie:workflow:0.5\">\n" +
            "  <global>\n" +
            "    <job-tracker>${jobTracker}</job-tracker>\n" +
            "    <name-node>${nameNode}</name-node>\n" +
            "    <configuration>\n" +
            "      <property>\n" +
            "        <name>mapreduce.output.fileoutputformat.outputdir</name>\n" +
            "        <value>" + TEST_OUTPUT_DIR + "</value>\n" +
            "      </property>\n" +
            "      <property>\n" +
            "        <name>mapreduce.input.fileinputformat.inputdir</name>\n" +
            "        <value>" + TEST_INPUT_DIR + "</value>\n" +
            "      </property>\n" +
            "    </configuration>\n" +
            "  </global>\n" +
            "  <start to=\"first\"/>\n" +
            "  <action name=\"first\">\n" +
            "    <map-reduce> <prepare><delete path=\"" + TEST_OUTPUT_DIR + "\"/></prepare></map-reduce>\n" +
            "    <ok to=\"decision-second-option\"/>\n" +
            "    <error to=\"kill\"/>\n" +
            "  </action>\n" +
            "  <decision name=\"decision-second-option\">\n" +
            "    <switch>\n" +
            "      <case to=\"option\">${doOption}</case>\n" +
            "      <default to=\"second\"/>\n" +
            "    </switch>\n" +
            "  </decision>\n" +
            "  <action name=\"option\">\n" +
            "    <map-reduce> <prepare><delete path=\"" + TEST_OUTPUT_DIR + "\"/></prepare></map-reduce>\n" +
            "    <ok to=\"second\"/>\n" +
            "    <error to=\"kill\"/>\n" +
            "  </action>\n" +
            "  <action name=\"second\">\n" +
            "    <map-reduce> <prepare><delete path=\"" + TEST_OUTPUT_DIR + "\"/></prepare></map-reduce>\n" +
            "    <ok to=\"end\"/>\n" +
            "    <error to=\"kill\"/>\n" +
            "  </action>\n" +
            "  <kill name=\"kill\">\n" +
            "    <message>\n" +
            "      Failed to workflow, error message[${wf: errorMessage (wf: lastErrorNode ())}]\n" +
            "    </message>\n" +
            "  </kill>\n" +
            "  <end name=\"end\"/>\n" +
            "</workflow-app>";

    Writer writer = new OutputStreamWriter(hdfsFs.create(workflow));
    writer.write(wfApp);
    writer.close();

    //write job.properties
    Properties conf = oozie.createConfiguration();
    conf.setProperty(OozieClient.APP_PATH, workflow.toString());
    conf.setProperty(OozieClient.USER_NAME, UserGroupInformation.getCurrentUser().getUserName());
    conf.setProperty("nameNode", "hdfs://localhost:" + hdfsLocalCluster.getHdfsNamenodePort());
    conf.setProperty("jobTracker", mrLocalCluster.getResourceManagerAddress());
    conf.setProperty("doOption", "true");

    //submit and check
    final String jobId = oozie.run(conf);
    WorkflowJob wf = oozie.getJobInfo(jobId);
    assertNotNull(wf);
    assertEquals(WorkflowJob.Status.RUNNING, wf.getStatus());


    while(true){
        Thread.sleep(1000);
        wf = oozie.getJobInfo(jobId);
        if(wf.getStatus() == WorkflowJob.Status.FAILED || wf.getStatus() == WorkflowJob.Status.KILLED || wf.getStatus() == WorkflowJob.Status.PREP || wf.getStatus() == WorkflowJob.Status.SUCCEEDED){
            break;
        }
    }

    wf = oozie.getJobInfo(jobId);
    assertEquals(WorkflowJob.Status.SUCCEEDED, wf.getStatus());

    LOG.info("OOZIE: Workflow: {}", wf.toString());
    hdfsFs.close();

}
 

private void submitWorkflowJob(String workFlowRoot)
		throws OozieClientException, InterruptedException {
	String oozieURL = System.getProperty("oozie.base.url");
	LOG.debug("Oozie BaseURL is: {} ", oozieURL);
	OozieClient client = new OozieClient(oozieURL);

	DateTime now = new DateTime();
	int monthOfYear = now.getMonthOfYear();
	int dayOfMonth = now.getDayOfMonth();
	int hourOfDay = now.getHourOfDay();
	String year = String.valueOf(now.getYear());
	String month = monthOfYear < 10 ? "0" + String.valueOf(monthOfYear)
			: String.valueOf(monthOfYear);
	String day = dayOfMonth < 10 ? "0" + String.valueOf(dayOfMonth)
			: String.valueOf(dayOfMonth);
	String hour = hourOfDay < 10 ? "0" + String.valueOf(hourOfDay) : String
			.valueOf(hourOfDay);

	Properties conf = client.createConfiguration();
	conf.setProperty(OozieClient.APP_PATH, workFlowRoot
			+ "/hive-action-add-partition.xml");
	conf.setProperty("nameNode", hadoopClusterService.getHDFSUri());
	conf.setProperty("jobTracker", hadoopClusterService.getJobTRackerUri());
	conf.setProperty("workflowRoot", workFlowRoot);
	conf.setProperty("YEAR", year);
	conf.setProperty("MONTH", month);
	conf.setProperty("DAY", day);
	conf.setProperty("HOUR", hour);
	conf.setProperty("oozie.use.system.libpath", "true");

	// submit and start the workflow job
	client.setDebugMode(1);
	// client.dryrun(conf);
	String jobId = client.run(conf);// submit(conf);

	LOG.debug("Workflow job submitted");
	// wait until the workflow job finishes printing the status every 10
	// seconds
	int retries = 3;
	for (int i = 1; i <= retries; i++) {
		// Sleep 60 sec./ 3 mins
		Thread.sleep(60 * 1000);

		WorkflowJob jobInfo = client.getJobInfo(jobId);
		Status jobStatus = jobInfo.getStatus();
		LOG.debug("Workflow job running ...");
		LOG.debug("HiveActionWorkflowJob Status Try: {}", i);
		LOG.debug("HiveActionWorkflowJob Id: {}", jobInfo.getId());
		LOG.debug("HiveActionWorkflowJob StartTime: {}",
				jobInfo.getStartTime());
		LOG.debug("HiveActionWorkflowJob EndTime: {}", jobInfo.getEndTime());
		LOG.debug("HiveActionWorkflowJob ConsoleURL: {}",
				jobInfo.getConsoleUrl());
		LOG.debug("HiveActionWorkflowJob Status: {}", jobInfo.getStatus());

		WorkflowAction workflowAction = jobInfo.getActions().get(0);

		LOG.debug("HiveActionWorkflowJob Action consoleURL: {}",
				workflowAction.getConsoleUrl());
		LOG.debug("HiveActionWorkflowJob Action Name: {}",
				workflowAction.getName());
		LOG.debug("HiveActionWorkflowJob Action error message: {}",
				workflowAction.getErrorMessage());
		LOG.debug("HiveActionWorkflowJob Action Status: {}",
				workflowAction.getStats());
		LOG.debug("HiveActionWorkflowJob Action data: {}",
				workflowAction.getData());
		LOG.debug("HiveActionWorkflowJob Action conf: {}",
				workflowAction.getConf());
		LOG.debug("HiveActionWorkflowJob Action retries: {}",
				workflowAction.getRetries());
		LOG.debug("HiveActionWorkflowJob Action id: {}",
				workflowAction.getId());
		LOG.debug("HiveActionWorkflowJob Action start time: {}",
				workflowAction.getStartTime());
		LOG.debug("HiveActionWorkflowJob Action end time: {}",
				workflowAction.getEndTime());
		LOG.debug("HiveActionWorkflowJob Oozie Url: {}",
				client.getOozieUrl());

		if (jobStatus == WorkflowJob.Status.SUCCEEDED) {
			LOG.info("Oozie workflow job was successful!" + jobStatus);
			break;
		} else if (jobStatus == WorkflowJob.Status.PREP
				|| jobStatus == WorkflowJob.Status.RUNNING) {
			if (i == retries) {
				throw new RuntimeException("Error executing workflow job!"
						+ jobStatus);
			} else {
				continue;
			}
		} else {
			throw new RuntimeException("Error executing workflow job!"
					+ jobStatus);
		}
	}
}
 

@Test
public void testDidSucceed() throws Exception {
  when( workflowJob.getStatus() ).thenReturn( WorkflowJob.Status.SUCCEEDED );
  assertTrue( oozieJobInfo.didSucceed() );
}
 

@Test
public void testDidntSucceed() throws Exception {
  when( workflowJob.getStatus() ).thenReturn( WorkflowJob.Status.FAILED );
  assertFalse( oozieJobInfo.didSucceed() );
}
 

@Test
public void testIsRunning() throws Exception {
  when( workflowJob.getStatus() ).thenReturn( WorkflowJob.Status.RUNNING );
  assertThat( oozieJobInfo.isRunning(), is( true ) );

}