org.apache.mesos.Protos.TaskInfo Java Examples

public void record(
        Protos.Offer.Operation operation,
        Protos.Offer offer) throws Exception {
    if (operation.getType() == Protos.Offer.Operation.Type.LAUNCH) {
        LOGGER.info("Persisting Launch Operation: {}", TextFormat.shortDebugString(operation));
        for (TaskInfo taskInfo : operation.getLaunch().getTaskInfosList()) {
            LOGGER.debug("Recording operation: {} for task: {}", operation, taskInfo);
            try {
                cassandraState.update(TaskUtils.unpackTaskInfo(taskInfo), offer);
            } catch (Exception e) {
                LOGGER.error("Error updating task in recorder with exception: ", e);
                throw e;
            }
        }
    }
}
 

@Override
public EvaluationOutcome filter(
    Offer offer,
    PodInstance podInstance,
    Collection<TaskInfo> tasks)
{
  if (offer.getSlaveId().getValue().equals(agentId)) {
    return EvaluationOutcome.pass(this, "Offer matches required Agent ID '%s'", agentId).build();
  } else {
    // agent mismatch: return empty offer
    return EvaluationOutcome.fail(
        this,
        "Offer lacks required Agent ID. Wanted: '%s' Got: '%s'",
        agentId,
        offer.getSlaveId().getValue())
        .build();
  }
}
 

@Test
public void testReserveTaskDynamicPort() throws Exception {
    Protos.Resource offeredPorts = ResourceTestUtils.getUnreservedPorts(10000, 10000);
    PodInstanceRequirement podInstanceRequirement = PodInstanceRequirementTestUtils.getPortRequirement(0);

    List<OfferRecommendation> recommendations =
            evaluator.evaluate(podInstanceRequirement, OfferTestUtils.getCompleteOffers(offeredPorts));

    Assert.assertEquals(6, recommendations.size());

    Protos.Offer.Operation launchOperation = recommendations.get(4).getOperation().get();
    Assert.assertFalse(recommendations.get(5).getOperation().isPresent());

    Protos.TaskInfo taskInfo = launchOperation.getLaunchGroup().getTaskGroup().getTasks(0);
    Protos.Resource fulfilledPortResource = taskInfo.getResources(0);
    Assert.assertFalse(getResourceId(fulfilledPortResource).isEmpty());

    Map<String, Protos.Environment.Variable> envvars = EnvUtils.toMap(taskInfo.getCommand().getEnvironment());
    Assert.assertEquals(envvars.toString(),
            String.valueOf(10000), envvars.get(TestConstants.PORT_ENV_NAME + "_0").getValue());
}
 

/**
 * Builds a TaskInfo from the given jobspec
 * @param job - JobSpec to TaskInfo-ify
 * @param offer - offer add extra data (SlaveId)
 * @return TaskInfo fully formed
 */
private TaskInfo getTaskInfo(JobSpec job,Offer offer) {
    TaskID taskId = TaskID.newBuilder().setValue(job.getJob().getId()).build();
    TaskInfo info = TaskInfo.newBuilder().setName("task " + taskId.getValue())
             .setTaskId(taskId)
             .setSlaveId(offer.getSlaveId())
             .addResources(Resource.newBuilder()
                           .setName("cpus")
                           .setType(Value.Type.SCALAR)
                           .setScalar(Value.Scalar.newBuilder().setValue(job.getJob().getLoadValue()*1.0)))
             .addResources(Resource.newBuilder()
                           .setName("mem")
                           .setType(Value.Type.SCALAR)
                           .setScalar(Value.Scalar.newBuilder().setValue(job.getJob().getLoadValue()*1024.0)))
             .setExecutor(ExecutorInfo.newBuilder(executor)).setData(MesosUtilities.jobSpecToByteString(job)).build();
    return info;
}
 

/**
 * Setup testing target & sample data.
 */
public LaunchTaskTest()
{
  SlaveID slaveID = SlaveID.newBuilder().setValue("s1").build();
  this.sampleTaskInfo =
      TaskInfo.newBuilder()
              .setName("t1").setSlaveId(slaveID)
              .setTaskId(TaskID.newBuilder().setValue("id2"))
              .setExecutor(
                  ExecutorInfo.newBuilder()
                              .setExecutorId(ExecutorID.newBuilder().setValue("e1"))
                              .setCommand(CommandInfo.getDefaultInstance()))
              .build();
  this.sampleOffer =
      Offer.newBuilder()
           .setHostname("h1").setSlaveId(slaveID)
           .setId(OfferID.newBuilder().setValue("id1"))
           .setFrameworkId(FrameworkID.newBuilder().setValue("f1").build())
           .build();
  this.target = new LaunchTask(sampleTaskInfo, sampleOffer);
}
 

private static PodSpec getPodSpec(ConfigStore<ServiceSpec> configStore, Protos.TaskInfo taskInfo)
    throws TaskException
{
  UUID configId = new TaskLabelReader(taskInfo).getTargetConfiguration();
  ServiceSpec serviceSpec;

  try {
    serviceSpec = configStore.fetch(configId);
  } catch (ConfigStoreException e) {
    throw new TaskException(String.format(
        "Unable to retrieve ServiceSpecification ID %s referenced by TaskInfo[%s]",
        configId, taskInfo.getName()), e);
  }

  Optional<PodSpec> podSpecOptional = getPodSpec(serviceSpec, taskInfo);
  if (!podSpecOptional.isPresent()) {
    throw new TaskException(String.format(
        "No TaskSpecification found for TaskInfo[%s]", taskInfo.getName()));
  } else {
    return podSpecOptional.get();
  }
}
 

@Override
public EvaluationOutcome filter(
    Offer offer,
    PodInstance podInstance,
    Collection<TaskInfo> tasks)
{
  return EvaluationOutcome.pass(this, "Passthrough rule always passes.").build();
}
 

private static TaskInfo getTask(String id, Offer offer) {
    TaskInfo.Builder taskBuilder = TaskTestUtils.getTaskInfo(Collections.emptyList()).toBuilder();
    taskBuilder.getTaskIdBuilder().setValue(id);
    try {
        taskBuilder.setName(CommonIdUtils.toTaskName(taskBuilder.getTaskId()));
    } catch (Exception e) {
        throw new IllegalStateException(e);
    }
    taskBuilder.setLabels(new TaskLabelWriter(taskBuilder).setOfferAttributes(offer).toProto());
    return taskBuilder.build();
}
 

protected boolean matchTask(Task task, Collection<OfferID> offerIds, Collection<TaskInfo> taskInfos) {
	for (Host host : cluster) {
		if (host instanceof HostInifinte) {
			// This represents the head node (skip)
		} else if (matchTask(task, host, offerIds, taskInfos)) {
			// Can this task be run on this host?
			return true;
		}
	}

	return false;
}
 

/**
 * Returns whether the provided {@link TaskInfo}, representing a previously-launched task,
 * is in the same provided pod provided in the {@link PodInstance}.
 */
public static boolean areEquivalent(TaskInfo taskInfo, PodInstance podInstance) {
  try {
    TaskLabelReader reader = new TaskLabelReader(taskInfo);
    return reader.getIndex() == podInstance.getIndex()
        && reader.getType().equals(podInstance.getPod().getType());
  } catch (TaskException e) {
    LOGGER.warn("Unable to extract pod type or index from TaskInfo", e);
    return false;
  }
}
 

private void setTaskInfoList(List<TaskInfo> taskInfoList) {
    LOGGER.debug("Writing executor state list: " + logTaskList(taskInfoList));
    try {
        new StatePath(zooKeeperStateDriver).mkdir(getKey());
        zooKeeperStateDriver.set(getKey(), taskInfoList);
    } catch (IOException ex) {
        LOGGER.error("Could not write list of executor states to zookeeper: ", ex);
    }
}
 

@Test
public void testUpdateStaticToStaticPort() throws Exception {
    // Launch for the first time: get port 555
    Resource reserveResource = recordLaunchWithCompleteOfferedResources(
            PodInstanceRequirementTestUtils.getPortRequirement(555),
            ResourceTestUtils.getUnreservedPorts(555, 555)).get(3);
    String resourceId = getResourceId(reserveResource);
    Collection<Resource> expectedResources = getExpectedExecutorResources(
            stateStore.fetchTasks().iterator().next().getExecutor());
    expectedResources.addAll(Arrays.asList(
            ResourceTestUtils.getReservedPorts(555, 555, resourceId),
            ResourceTestUtils.getUnreservedPorts(666, 666)));

    // Now lets move to port 666:
    List<OfferRecommendation> recommendations = evaluator.evaluate(
            PodInstanceRequirementTestUtils.getPortRequirement(666),
            Arrays.asList(OfferTestUtils.getOffer(expectedResources)));

    // UNRESERVE, RESERVE, LAUNCH
    Assert.assertEquals(Arrays.asList(
            Protos.Offer.Operation.Type.UNRESERVE,
            Protos.Offer.Operation.Type.RESERVE,
            Offer.Operation.Type.LAUNCH_GROUP,
            null),
            recommendations.stream()
                    .map(rec -> rec.getOperation().isPresent() ? rec.getOperation().get().getType() : null)
                    .collect(Collectors.toList()));

    Operation launchOperation = recommendations.get(2).getOperation().get();
    TaskInfo taskInfo = launchOperation.getLaunchGroup().getTaskGroup().getTasks(0);

    Map<String, Protos.Environment.Variable> envvars = EnvUtils.toMap(taskInfo.getCommand().getEnvironment());
    Assert.assertEquals(String.valueOf(666), envvars.get(TestConstants.PORT_ENV_NAME + "_666").getValue());
}
 

/**
 * Pretty-print mesos protobuf TaskInfo.
 * <p/>
 * XXX(erikdw): not including command, container (+data), nor health_check.
 */
public static String taskInfoToString(TaskInfo task) {
  Map<String, String> map = new LinkedHashMap<>();
  map.put("task_id", task.getTaskId().getValue());
  map.put("slave_id", task.getSlaveId().getValue());
  map.putAll(resourcesToOrderedMap(task.getResourcesList()));
  map.put("executor_id", task.getExecutor().getExecutorId().getValue());
  return JSONValue.toJSONString(map);
}
 

public void removeTask(TaskInfo taskInfo) throws InvalidParameterException {
    List<TaskInfo> taskList = getTaskList();
    LOGGER.debug("Removing TaskInfo from cluster for task: " + taskInfo.getTaskId().getValue());
    if (!taskList.remove(taskInfo)) {
        throw new InvalidParameterException("TaskInfo does not exist in list: " + taskInfo.getTaskId().getValue());
    }
    getStatus(taskInfo).destroy(); // Destroy task status in ZK.
    setTaskInfoList(taskList); // Remove from cluster state list
}
 

/**
 * We assume a long-running Mesos Task that manages a REEF Evaluator process, leveraging Mesos Executor's interface.
 */
@Override
public void launchTask(final ExecutorDriver driver, final TaskInfo task) {
  driver.sendStatusUpdate(TaskStatus.newBuilder()
      .setTaskId(TaskID.newBuilder().setValue(this.mesosExecutorId).build())
      .setState(TaskState.TASK_STARTING)
      .setSlaveId(task.getSlaveId())
      .setMessage(this.mesosRemoteManager.getMyIdentifier())
      .build());
}
 

@Test
public void assertLaunchTaskWithWrongElasticJobClass() {
    TaskInfo taskInfo = buildWrongElasticJobClass();
    TaskExecutor.TaskThread taskThread = new TaskExecutor().new TaskThread(executorDriver, taskInfo);
    try {
        taskThread.run();
    } catch (final JobSystemException ex) {
        Assert.assertTrue(ex.getMessage().startsWith("Elastic-Job: Class 'org.apache.shardingsphere.elasticjob.cloud.executor.TaskExecutorThreadTest' must implements ElasticJob interface."));
    }
}
 

@Test
public void testReserveStaticPort() throws Exception {
    PodInstanceRequirement podInstanceRequirement = PodInstanceRequirementTestUtils.getPortRequirement(555);
    Protos.Resource offeredPorts = ResourceTestUtils.getUnreservedPorts(555, 555);

    List<OfferRecommendation> recommendations =
            evaluator.evaluate(podInstanceRequirement, OfferTestUtils.getCompleteOffers(offeredPorts));

    Assert.assertEquals(Arrays.asList(
            Protos.Offer.Operation.Type.RESERVE,
            Protos.Offer.Operation.Type.RESERVE,
            Protos.Offer.Operation.Type.RESERVE,
            Protos.Offer.Operation.Type.RESERVE,
            Offer.Operation.Type.LAUNCH_GROUP,
            null),
            recommendations.stream()
                    .map(rec -> rec.getOperation().isPresent() ? rec.getOperation().get().getType() : null)
                    .collect(Collectors.toList()));

    Protos.Offer.Operation launchOperation = recommendations.get(4).getOperation().get();
    Protos.TaskInfo taskInfo = launchOperation.getLaunchGroup().getTaskGroup().getTasks(0);
    Protos.Resource fulfilledPortResource = taskInfo.getResources(0);
    Assert.assertFalse(getResourceId(fulfilledPortResource).isEmpty());

    Map<String, Protos.Environment.Variable> envvars = EnvUtils.toMap(taskInfo.getCommand().getEnvironment());
    Assert.assertEquals(String.valueOf(555), envvars.get(TestConstants.PORT_ENV_NAME + "_555").getValue());
}
 

private SingularityTaskExecutorData readExecutorData(
  ObjectMapper objectMapper,
  Protos.TaskInfo taskInfo
) {
  try {
    Preconditions.checkState(taskInfo.hasData(), "TaskInfo was missing executor data");

    return objectMapper.readValue(
      taskInfo.getData().toByteArray(),
      SingularityTaskExecutorData.class
    );
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 

@Subscribe
public void runTask(final RunTaskMessageEnvelope envelope)
{
    checkState(envelope.getRecipient().equals(context.getDriverUPID()), "Received a remote message for local delivery");

    if (context.isStateMachine(DRIVER_ABORTED)) {
        LOG.warn("driver is aborted!");
        return;
    }

    final RunTaskMessage message = envelope.getMessage();

    final TaskInfo task = message.getTask();

    checkState(!tasks.containsKey(task.getTaskId()), "Task %s already started!", task.getTaskId().getValue());

    tasks.put(task.getTaskId(), task);

    eventBus.post(new ExecutorCallback() {
        @Override
        public Runnable getCallback(final Executor executor, final ExecutorDriver executorDriver)
        {
            return new Runnable() {
                @Override
                public void run()
                {
                    executor.launchTask(executorDriver, task);
                }

                @Override
                public String toString()
                {
                    return "callback for launchTask()";
                }
            };
        }
    });
}
 

private String logTaskList(List<TaskInfo> taskInfoList) {
    List<String> res = new ArrayList<>();
    for (TaskInfo t : taskInfoList) {
        res.add(t.getTaskId().getValue());
    }
    return Arrays.toString(res.toArray());
}
 

private static TaskInfo getTaskInfo(String taskName, String attrName, String attrVal) {
    TaskInfo.Builder infoBuilder = TaskTestUtils.getTaskInfo(Collections.emptyList()).toBuilder()
            .setName(taskName)
            .setTaskId(CommonIdUtils.toTaskId(TestConstants.SERVICE_NAME, taskName));
    infoBuilder.setLabels(new TaskLabelWriter(infoBuilder)
            .setOfferAttributes(offerWithAttribute(attrName, attrVal))
            .toProto());
    return infoBuilder.build();
}
 

/**
 * @see LabelReader#LabelReader(String, Labels)
 */
public TaskLabelReader(TaskInfo.Builder taskInfoBuilder) {
  reader = new LabelReader(
      String.format("Task %s", taskInfoBuilder.getName()),
      taskInfoBuilder.getLabels()
  );
}
 

@Override
public List<String> getFile(TaskInfo taskInfo) throws FileNotFoundException, IOException {

    List<String> paths = getFileName(taskInfo);
    if (log.isDebugEnabled())
        log.debug("############## ComposeFileList.getFile, paths: ##############" + paths);
    validateFiles(paths);

    return paths;
}
 

/**
 * Implementation of task type colocation. Considers the presence of tasks in the cluster to
 * determine whether the provided task can be launched against a given offer. This rule requires
 * that the offer be located on an agent which currently has an instance of the specified task
 * type.
 */
private EvaluationOutcome filterColocate(
    Offer offer,
    PodInstance podInstance,
    Collection<TaskInfo> tasksToColocate)
{

  for (TaskInfo taskToColocate : tasksToColocate) {
    if (TaskUtils.areEquivalent(taskToColocate, podInstance)) {
      // This is stale data for the same task that we're currently evaluating for
      // placement. Don't worry about colocating with it. This occurs when we're
      // redeploying a given task with a new configuration (old data not deleted yet).
      continue;
    }
    if (taskToColocate.getSlaveId().equals(offer.getSlaveId())) {
      // The offer is for an agent which has a task to colocate with. Approved!
      return EvaluationOutcome.pass(
          this,
          "Found a task matching colocated type '%s' on this agent.",
          typeToFind)
          .build();
    }
  }
  // The offer doesn't match any tasks to colocate with. Denied!
  return EvaluationOutcome.fail(
      this,
      "Didn't find a task matching colocated type '%s' on this agent.", typeToFind)
      .build();
}
 

/**
 * Returns the task type embedded in the provided {@link TaskInfo}'s labels.
 *
 * @throws IllegalArgumentException if the provided task doesn't have a task type label
 */
@Override
public String getTaskType(TaskInfo taskInfo) {
  try {
    return new TaskLabelReader(taskInfo).getType();
  } catch (TaskException e) {
    throw new IllegalArgumentException(String.format(
        "Unable to extract task type label from provided TaskInfo: %s", taskInfo), e);
  }
}
 

/**
 * Returns a value to round robin against from the provided {@link TaskInfo}.
 */
protected String getKey(TaskInfo task) {
  try {
    return new TaskLabelReader(task).getHostname();
  } catch (TaskException e) {
    LOGGER.warn("Unable to extract hostname from task for filtering", e);
    return null;
  }
}
 

@Override
protected String getKey(TaskInfo task) {
  for (String taskAttributeString : new TaskLabelReader(task).getOfferAttributeStrings()) {
    AttributeStringUtils.NameValue taskAttributeNameValue =
        AttributeStringUtils.split(taskAttributeString);
    if (taskAttributeNameValue.name.equalsIgnoreCase(attributeName)) {
      return taskAttributeNameValue.value;
    }
  }
  return null;
}
 

@Override
public EvaluationOutcome filter(
    Offer offer,
    PodInstance podInstance,
    Collection<TaskInfo> tasks)
{
  EvaluationOutcome child = rule.filter(offer, podInstance, tasks);
  String reason = "Returning opposite of child rule";
  if (child.isPassing()) {
    return EvaluationOutcome.fail(this, reason).addChild(child).build();
  } else {
    return EvaluationOutcome.pass(this, reason).addChild(child).build();
  }
}
 

/**
 * Match a task to the offer/s
 */
protected synchronized boolean matchTask(Task task, Host host, Collection<OfferID> offerIds, Collection<TaskInfo> taskInfos) {
	// Not enough resources in this host?
	if (!host.hasResourcesAvailable(task.getResources())) return false;

	if (verbose) Gpr.debug("Matching task: " + task.getId() + "\t resources: " + task.getResources() + "\thost:" + host.getHostName() + ", resources: " + host.getResourcesAvaialble());

	// OK, we should be able to run 'task' in hostName
	String hostName = host.toString();
	Set<Offer> offers = offersByHost.get(hostName);
	if (offers == null) {
		Gpr.debug("Offer accounting problem in host '" + hostName + "': This should ever happen!");
		cluster.remove(host); // Remove any resources since we don't really seem to have any
		return false;
	}

	// Select offers and add taskInfo
	HostResources tr = new HostResources(task.getResources());
	for (Offer offer : offers) {
		// Consume resources until offers fulfill task requirements
		HostResources or = parseOffer(offer);
		tr.consume(or);

		// Add offer and taskInfo to collections
		offerIds.add(offer.getId());
		TaskInfo taskInfo = taskInfo(offer, task);
		taskInfos.add(taskInfo);

		// Are all resources needed for this task satisfied?
		if (tr.isConsumed()) return true;
	}

	Gpr.debug("Resource accounting problem in host '" + hostName + "': This should ever happen!");
	return false;
}
 

@Override
public Status launchTasks(Collection<OfferID> offerIds, Collection<TaskInfo> tasks) {
  assertNotStopped();

  OfferID id = Iterables.getOnlyElement(offerIds);
  Offer offer = sentOffers.remove(id);
  checkState(offer != null, "Offer " + id + " is invalid.");

  final TaskInfo task = Iterables.getOnlyElement(tasks);
  synchronized (activeTasks) {
    checkState(
        !activeTasks.containsKey(task.getTaskId()),
        "Task " + task.getTaskId() + " already exists.");
    activeTasks.put(task.getTaskId(), new Task(offer, task));
  }

  executor.schedule(
      () -> Futures.getUnchecked(schedulerFuture).statusUpdate(
          this,
          TaskStatus.newBuilder()
              .setTaskId(task.getTaskId())
              .setState(TaskState.TASK_RUNNING)
              .build()),
      1,
      TimeUnit.SECONDS);

  return Status.DRIVER_RUNNING;
}