Java Code Examples for org.apache.helix.task.TaskDriver#delete()

private void cleanUpJobs(HelixManager helixManager) {
  // Clean up existing jobs
  TaskDriver taskDriver = new TaskDriver(helixManager);

  Map<String, WorkflowConfig> workflows = taskDriver.getWorkflows();

  log.debug("cleanUpJobs workflow count {} workflows {}", workflows.size(), workflows.keySet());

  boolean cleanupDistJobs = ConfigUtils.getBoolean(this.config,
      GobblinClusterConfigurationKeys.CLEAN_ALL_DIST_JOBS,
      GobblinClusterConfigurationKeys.DEFAULT_CLEAN_ALL_DIST_JOBS);

  for (Map.Entry<String, WorkflowConfig> entry : workflows.entrySet()) {
    String workflowName = entry.getKey();

    if (workflowName.contains(GobblinClusterConfigurationKeys.PLANNING_JOB_NAME_PREFIX)
        || workflowName.contains(GobblinClusterConfigurationKeys.ACTUAL_JOB_NAME_PREFIX)) {
      if (!cleanupDistJobs) {
        log.info("Distributed job {} won't be deleted.", workflowName);
        continue;
      }
    }

    WorkflowConfig workflowConfig = entry.getValue();

    // request delete if not already requested
    if (workflowConfig.getTargetState() != TargetState.DELETE) {
      taskDriver.delete(workflowName);

      log.info("Requested delete of workflowName {}", workflowName);
    }
  }
}
 

@DELETE
@Path("{workflowId}")
public Response deleteWorkflow(@PathParam("clusterId") String clusterId,
    @PathParam("workflowId") String workflowId,
    @QueryParam("force") @DefaultValue("false") String forceDelete) {
  boolean force = Boolean.valueOf(forceDelete);
  TaskDriver driver = getTaskDriver(clusterId);
  try {
    driver.delete(workflowId, force);
  } catch (HelixException e) {
    return badRequest(String
        .format("Failed to delete workflow %s for reason : %s", workflowId, e.getMessage()));
  }
  return OK();
}
 

/**
 * Parses the first argument as a driver command and the rest of the
 * arguments are parsed based on that command. Constructs a Helix
 * message and posts it to the controller
 */
public static void main(String[] args) throws Exception {
  String[] cmdArgs = Arrays.copyOfRange(args, 1, args.length);
  CommandLine cl = parseOptions(cmdArgs, constructOptions(), args[0]);
  String zkAddr = cl.getOptionValue(ZK_ADDRESS);
  String clusterName = cl.getOptionValue(CLUSTER_NAME_OPTION);
  String workflow = cl.getOptionValue(RESOURCE_OPTION);

  if (zkAddr == null || clusterName == null || workflow == null) {
    printUsage(constructOptions(), "[cmd]");
    throw new IllegalArgumentException(
        "zk, cluster, and resource must all be non-null for all commands");
  }

  HelixManager helixMgr =
      HelixManagerFactory.getZKHelixManager(clusterName, "Admin", InstanceType.ADMINISTRATOR,
          zkAddr);
  helixMgr.connect();
  TaskDriver driver = new TaskDriver(helixMgr);
  try {
    TaskDriver.DriverCommand cmd = TaskDriver.DriverCommand.valueOf(args[0]);
    switch (cmd) {
    case start:
      if (cl.hasOption(WORKFLOW_FILE_OPTION)) {
        driver.start(Workflow.parse(new File(cl.getOptionValue(WORKFLOW_FILE_OPTION))));
      } else {
        throw new IllegalArgumentException("Workflow file is required to start flow.");
      }
      break;
    case stop:
      driver.stop(workflow);
      break;
    case resume:
      driver.resume(workflow);
      break;
    case delete:
      driver.delete(workflow);
      break;
    case list:
      list(driver, workflow);
      break;
    case flush:
      driver.flushQueue(workflow);
      break;
    case clean:
      driver.cleanupQueue(workflow);
      break;
    default:
      throw new IllegalArgumentException("Unknown command " + args[0]);
    }
  } catch (IllegalArgumentException e) {
    LOG.error("Unknown driver command " + args[0]);
    throw e;
  }

  helixMgr.disconnect();
}
 

/**
 * Start a new job in a job queue, or stop/resume/persistDataChanges/delete a job queue
 * <p>
 * Usage:
 * <p>
 * <li>Start a new job in a job queue:
 * <code>curl -d @'./{input.txt}' -H 'Content-Type: application/json'
 * http://{host:port}/clusters/{clusterName}/jobQueues/{jobQueue}
 * <p>
 * input.txt: <code>jsonParameters={"command":"start"}&newJob={newJobConfig.yaml}
 * <p>
 * For newJobConfig.yaml, see {@link Workflow#parse(String)}
 * <li>Stop/resume/persistDataChanges/delete a job queue:
 * <code>curl -d 'jsonParameters={"command":"{stop/resume/persistDataChanges/delete}"}'
 * -H "Content-Type: application/json" http://{host:port}/clusters/{clusterName}/jobQueues/{jobQueue}
 */
@Override
public Representation post(Representation entity) {
  String clusterName =
      ResourceUtil.getAttributeFromRequest(getRequest(), ResourceUtil.RequestKey.CLUSTER_NAME);
  String jobQueueName =
      ResourceUtil.getAttributeFromRequest(getRequest(), ResourceUtil.RequestKey.JOB_QUEUE);
  ZkClient zkClient =
      ResourceUtil.getAttributeFromCtx(getContext(), ResourceUtil.ContextKey.ZKCLIENT);
  try {
    TaskDriver driver = new TaskDriver(zkClient, clusterName);

    Form form = new Form(entity);
    JsonParameters jsonParameters = new JsonParameters(form);

    TaskDriver.DriverCommand cmd = TaskDriver.DriverCommand.valueOf(jsonParameters.getCommand());
    switch (cmd) {
    case start: {
      // Get the job queue and submit it
      String yamlPayload =
          ResourceUtil.getYamlParameters(form, ResourceUtil.YamlParamKey.NEW_JOB);
      if (yamlPayload == null) {
        throw new HelixException("Yaml job config is required!");
      }
      Workflow workflow = Workflow.parse(yamlPayload);

      for (String jobName : workflow.getJobConfigs().keySet()) {
        Map<String, String> jobCfgMap = workflow.getJobConfigs().get(jobName);
        JobConfig.Builder jobCfgBuilder = JobConfig.Builder.fromMap(jobCfgMap);
        if (workflow.getTaskConfigs() != null && workflow.getTaskConfigs().containsKey(jobName)) {
          jobCfgBuilder.addTaskConfigs(workflow.getTaskConfigs().get(jobName));
        }
        driver.enqueueJob(jobQueueName, TaskUtil.getDenamespacedJobName(jobQueueName, jobName),
            jobCfgBuilder);
      }
      break;
    }
    case stop: {
      driver.stop(jobQueueName);
      break;
    }
    case resume: {
      driver.resume(jobQueueName);
      break;
    }
    case flush: {
      driver.flushQueue(jobQueueName);
      break;
    }
    case delete: {
      driver.delete(jobQueueName);
      break;
    }
    case clean: {
      driver.cleanupQueue(jobQueueName);
      break;
    }
    default:
      throw new HelixException("Unsupported job queue command: " + cmd);
    }
    getResponse().setEntity(getHostedEntitiesRepresentation(clusterName, jobQueueName));
    getResponse().setStatus(Status.SUCCESS_OK);
  } catch (Exception e) {
    getResponse().setEntity(ClusterRepresentationUtil.getErrorAsJsonStringFromException(e),
        MediaType.APPLICATION_JSON);
    getResponse().setStatus(Status.SUCCESS_OK);
    LOG.error("Error in posting job queue: " + entity, e);
  }
  return null;
}