org.apache.mesos.v1.scheduler.Protos Java Examples

public static void decline(final @NotNull Protocol protocol, final @NotNull String streamId, final @NotNull org.apache.mesos.v1.Protos.FrameworkID frameworkId,
                           final @NotNull Protos.Call call, final @NotNull String url) {
    try {
        SendUtils.sendCall(call, protocol, streamId, url);
    } catch (IOException e) {
        log.error("send decline call to mesos error!");
        log.error("frameworkId : " + frameworkId.getValue());
        log.error("call : " + call);
        try {
            Thread.sleep(5000);
        } catch (InterruptedException e1) {
            e1.printStackTrace();
        }
        getSendErrors().push(new SendErrors(frameworkId, call));
    }
}
 

private Protos.Call subscribeCall() {

        String hostName = System.getenv("HOST");
        if (StringUtils.isBlank(hostName)) {
            try {
                hostName = InetAddress.getLocalHost().getHostAddress();
            } catch (UnknownHostException e) {
                hostName = "unknown host";
                e.printStackTrace();
            }
        }

        return SchedulerCalls.subscribe(
                FrameworkInfo.newBuilder()
                        .setId(frameworkId)
                        .setHostname(hostName)
                        .setUser(Optional.ofNullable(System.getenv("user")).orElse("root"))
                        .setName(MESOS_SCHEDULER_NAME)
                        .setFailoverTimeout(FRAMEWORK_FAILOVER_TIMEOUT)
                        .setRole(MESOS_FRAMEWORK_ROLE)
                        .build());
    }
 

/**
 * Utility method to more succinctly construct an {@link Protos.Event Event} of type
 * {@link Protos.Event.Type#UPDATE UPDATE}.
 *
 * @param agentId        The {@link org.apache.mesos.v1.Protos.TaskStatus#getAgentId() agentId} to be set on the
 *                       {@link org.apache.mesos.v1.Protos.TaskStatus TaskStatus}.
 * @param executorId     The {@link org.apache.mesos.v1.Protos.TaskStatus#getExecutorId() executorId} to be set on the
 *                       {@link org.apache.mesos.v1.Protos.TaskStatus TaskStatus}.
 * @param taskId         The {@link org.apache.mesos.v1.Protos.TaskStatus#getTaskId() taskId} to be set on the
 *                       {@link org.apache.mesos.v1.Protos.TaskStatus TaskStatus}.
 * @param state          The {@link org.apache.mesos.v1.Protos.TaskState TaskState} to be set on the
 *                       {@link org.apache.mesos.v1.Protos.TaskStatus TaskStatus}.
 * @param uuid           The {@link org.apache.mesos.v1.Protos.TaskStatus#getUuid() uuid} to be set on the
 *                       {@link org.apache.mesos.v1.Protos.TaskStatus TaskStatus}.
 * @return  A {@link Protos.Call Call} with a configured {@link Protos.Call.Acknowledge Acknowledge}.
 */
@NotNull
public static Protos.Event update(
    @NotNull final String agentId,
    @NotNull final String executorId,
    @NotNull final String taskId,
    @NotNull final org.apache.mesos.v1.Protos.TaskState state,
    @Nullable final ByteString uuid
) {
    return update(
        org.apache.mesos.v1.Protos.AgentID.newBuilder().setValue(agentId).build(),
        org.apache.mesos.v1.Protos.ExecutorID.newBuilder().setValue(executorId).build(),
        org.apache.mesos.v1.Protos.TaskID.newBuilder().setValue(taskId).build(),
        state,
        uuid
    );
}
 

/**
 * Utility method to more succinctly construct an {@link Protos.Event Event} of type
 * {@link Protos.Event.Type#MESSAGE MESSAGE}.
 *
 * @param agentId        The {@link Protos.Event.Message#getAgentId() agentId} to be set on the
 *                       {@link Protos.Event.Message Message}.
 * @param executorId     The {@link Protos.Event.Message#getExecutorId() executorId} to be set on the
 *                       {@link Protos.Event.Message Message}.
 * @param data           The {@link Protos.Event.Message#getData() data} to be set on the
 *                       {@link Protos.Event.Message Message}.
 * @return  A {@link Protos.Call Call} with a configured {@link Protos.Call.Acknowledge Acknowledge}.
 */
@NotNull
public static Protos.Event message(
    @NotNull final org.apache.mesos.v1.Protos.AgentID agentId,
    @NotNull final org.apache.mesos.v1.Protos.ExecutorID executorId,
    @NotNull final ByteString data
) {
    return Protos.Event.newBuilder()
        .setType(Protos.Event.Type.MESSAGE)
        .setMessage(
            Protos.Event.Message.newBuilder()
                .setAgentId(agentId)
                .setExecutorId(executorId)
                .setData(data)
        )
        .build();
}
 

@Override
public Object getEvent(byte[] buffer, Class<?> classz) throws Exception {
    if(classz.equals(Protos.Event.class)) {
        return Protos.Event.parseFrom(buffer);
    }

    return Event.parseFrom(buffer);
}
 

public static void kill(@NotNull Map<Long, String> killmap, String agentId, @NotNull Protocol protocol, final @NotNull String streamId, final @NotNull FrameworkID frameworkID, @NotNull String url, final @NotNull long taskId) {
    if (killmap.containsKey(taskId)) {
        fixedAuxiliaryPool.submit(() -> {
            JuiceTask juiceTask = daoUtils.queryTask(taskId);
            if (null != juiceTask) {
                if (juiceTask.getTaskStatus() > TaskResult.Result.RUNNING.getType()) {
                    killmap.remove(taskId);
                    return;
                }
                String value = killmap.get(taskId);
                if (StringUtils.isBlank(value)) {
                    if (StringUtils.isNotBlank(juiceTask.getAgentId())) {
                        value = juiceTask.getAgentId();
                    } else if (StringUtils.isNotBlank(agentId)) {
                        value = agentId;
                    }
                }
                if (StringUtils.isNotBlank(value)) {
                    TaskManagement.TaskAgentRel taskAgentRel = new TaskManagement.TaskAgentRel(taskId, juiceTask.getTaskName(), juiceTask.getRetry(), value);
                    Protos.Call call = SchedulerCalls.kill(frameworkID, taskAgentRel);
                    try {
                        SendUtils.sendCall(call, protocol, streamId, url);
                        killmap.remove(taskId);
                    } catch (IOException e) {
                        log.warn("send kill task call error due to : " + e.getCause());
                    }
                }
            }
        });
    }
}
 

public static void accept(final @NotNull Protocol protocol, final @NotNull String streamId, final @NotNull org.apache.mesos.v1.Protos.OfferID offerID,
                          final @NotNull org.apache.mesos.v1.Protos.FrameworkID frameworkID, final @NotNull List<org.apache.mesos.v1.Protos.TaskInfo> taskInfos,
                          final @NotNull String url) {
    Protos.Call call = SchedulerCalls.accept(frameworkID, offerID, taskInfos);
    log.info("accept --> Launching {} tasks", taskInfos.size());

    try {
        SendUtils.sendCall(call, protocol, streamId, url);
    } catch (IOException e) {
        log.error("send accept call to mesos error!");
        log.error("frameworkId : " + frameworkID.getValue() + ", offerId : " + offerID.getValue());
        log.error("call : " + call);
        taskInfos.forEach(
                taskInfo -> {
                    getTaskErrors().add(new TaskResult(com.hujiang.juice.common.model.Task.splitTaskNameId(taskInfo.getTaskId().getValue()), TaskResult.Result.ERROR, "send task to mesos error!"));
                }
        );
        try {
            Thread.sleep(5000);
        } catch (InterruptedException e1) {
            e1.printStackTrace();
        }
        List<org.apache.mesos.v1.Protos.OfferID> offerIDS = Lists.newArrayList();
        offerIDS.add(offerID);

        //  if accept error, then offer will be decline
        getSendErrors().push(new SendErrors(frameworkID, SchedulerCalls.decline(frameworkID, offerIDS)));
    }
}
 

private void connecting() throws Exception {
    InputStream stream = null;
    Response res = null;

    try {
        Protos.Call call = subscribeCall();
        res = Restty.create(getUrl())
                .addAccept(protocol.mediaType())
                .addMediaType(protocol.mediaType())
                .addKeepAlive()
                .requestBody(protocol.getSendBytes(call))
                .post();

        streamId = res.header(STREAM_ID);
        stream = res.body().byteStream();
        log.info("send subscribe, frameworkId : " + frameworkId + " , url " + getUrl() + ", streamId : " + streamId);
        log.debug("subscribe call : " + call);
        if (null == stream) {
            log.warn("stream is null");
            throw new DriverException("stream is null");
        }
        while (true) {
            int size = SendUtils.readChunkSize(stream);
            byte[] event = SendUtils.readChunk(stream, size);

            onEvent(event);
        }
    } catch (Exception e) {
        log.error("service handle error, due to : " + e);
        throw e;
    } finally {
        if (null != stream) {
            stream.close();
        }
        if (null != res) {
            res.close();
        }
        streamId = null;
    }
}
 

@Before
public void setUp() throws Exception {
    subject = BehaviorSubject.create();
    sim = new MesosServerSimulation<>(
        subject,
        ProtobufMessageCodecs.SCHEDULER_EVENT,
        ProtobufMessageCodecs.SCHEDULER_CALL,
        (e) -> e.getType() == Protos.Call.Type.SUBSCRIBE
    );
    final int serverPort = sim.start();
    uri = URI.create(String.format("http://localhost:%d/api/v1/scheduler", serverPort));
}
 

/**
 * @return  An initial {@link MesosClientBuilder} that will use protobuf
 *          for the {@link org.apache.mesos.v1.scheduler.Protos.Call Call} and
 *          {@link org.apache.mesos.v1.scheduler.Protos.Event Event} messages.
 */
@NotNull
public static MesosClientBuilder<Protos.Call, Protos.Event> schedulerUsingProtos() {
    return MesosClientBuilder.<Protos.Call, Protos.Event>newBuilder()
        .sendCodec(ProtobufMessageCodecs.SCHEDULER_CALL)
        .receiveCodec(ProtobufMessageCodecs.SCHEDULER_EVENT)
        ;
}
 

/**
 * @return  An initial {@link MesosClientBuilder} that will use protobuf
 *          for the {@link org.apache.mesos.v1.executor.Protos.Call Call} and
 *          {@link org.apache.mesos.v1.executor.Protos.Event Event} messages.
 */
@NotNull
public static MesosClientBuilder<
    org.apache.mesos.v1.executor.Protos.Call,
    org.apache.mesos.v1.executor.Protos.Event
    > executorUsingProtos() {
    return MesosClientBuilder.<org.apache.mesos.v1.executor.Protos.Call, org.apache.mesos.v1.executor.Protos.Event>newBuilder()
        .sendCodec(ProtobufMessageCodecs.EXECUTOR_CALL)
        .receiveCodec(ProtobufMessageCodecs.EXECUTOR_EVENT)
        ;
}
 

/**
 * Utility method to more succinctly construct an {@link Protos.Event Event} of type
 * {@link Protos.Event.Type#SUBSCRIBED SUBSCRIBED}.
 *
 * @param frameworkId              The frameworkId to be set on the
 *                                 {@link Protos.Event.Subscribed} message.
 * @param heartbeatIntervalSeconds The heartbeatIntervalSeconds to be set on the
 *                                 {@link Protos.Event.Subscribed} message.
 * @return An instance of {@link Protos.Event Event} of type
 * {@link Protos.Event.Type#SUBSCRIBED SUBSCRIBED} and
 * {@link Protos.Event#getSubscribed() subscribed} set based on the provide parameters.
 */
@NotNull
public static Protos.Event subscribed(@NotNull final String frameworkId, final int heartbeatIntervalSeconds) {
    return Protos.Event.newBuilder()
        .setType(Protos.Event.Type.SUBSCRIBED)
        .setSubscribed(
            Protos.Event.Subscribed.newBuilder()
                .setFrameworkId(org.apache.mesos.v1.Protos.FrameworkID.newBuilder()
                    .setValue(frameworkId)
                )
                .setHeartbeatIntervalSeconds(heartbeatIntervalSeconds)
        )
        .build();
}
 

/**
 * Utility method to more succinctly construct an {@link Protos.Event Event} of type
 * {@link Protos.Event.Type#UPDATE UPDATE}.
 *
 * @param agentId        The {@link org.apache.mesos.v1.Protos.TaskStatus#getAgentId() agentId} to be set on the
 *                       {@link org.apache.mesos.v1.Protos.TaskStatus TaskStatus}.
 * @param executorId     The {@link org.apache.mesos.v1.Protos.TaskStatus#getExecutorId() executorId} to be set on the
 *                       {@link org.apache.mesos.v1.Protos.TaskStatus TaskStatus}.
 * @param taskId         The {@link org.apache.mesos.v1.Protos.TaskStatus#getTaskId() taskId} to be set on the
 *                       {@link org.apache.mesos.v1.Protos.TaskStatus TaskStatus}.
 * @param state          The {@link org.apache.mesos.v1.Protos.TaskState TaskState} to be set on the
 *                       {@link org.apache.mesos.v1.Protos.TaskStatus TaskStatus}.
 * @param uuid           The {@link org.apache.mesos.v1.Protos.TaskStatus#getUuid() uuid} to be set on the
 *                       {@link org.apache.mesos.v1.Protos.TaskStatus TaskStatus}.
 * @return  A {@link Protos.Call Call} with a configured {@link Protos.Call.Acknowledge Acknowledge}.
 */
@NotNull
public static Protos.Event update(
    @NotNull final org.apache.mesos.v1.Protos.AgentID agentId,
    @NotNull final org.apache.mesos.v1.Protos.ExecutorID executorId,
    @NotNull final org.apache.mesos.v1.Protos.TaskID taskId,
    @NotNull final org.apache.mesos.v1.Protos.TaskState state,
    @Nullable final ByteString uuid
) {
    final org.apache.mesos.v1.Protos.TaskStatus.Builder builder = org.apache.mesos.v1.Protos.TaskStatus.newBuilder()
        .setAgentId(agentId)
        .setExecutorId(executorId)
        .setTaskId(taskId)
        .setState(state);

    final org.apache.mesos.v1.Protos.TaskStatus status;
    if (uuid != null) {
        status = builder.setUuid(uuid).build();
    } else {
        status = builder.build();
    }

    return Protos.Event.newBuilder()
        .setType(Protos.Event.Type.UPDATE)
        .setUpdate(
            Protos.Event.Update.newBuilder()
                .setStatus(status)
        )
        .build();
}
 

@Test
public void offerSimulation() throws Throwable {
    final String fwId = "sleepy-" + UUID.randomUUID();
    final org.apache.mesos.v1.Protos.FrameworkID frameworkID = org.apache.mesos.v1.Protos.FrameworkID.newBuilder()
        .setValue(fwId)
        .build();
    final Protos.Call subscribe = SchedulerCalls.subscribe(
        frameworkID,
        org.apache.mesos.v1.Protos.FrameworkInfo.newBuilder()
            .setId(frameworkID)
            .setUser("root")
            .setName("sleepy")
            .setFailoverTimeout(0)
            .setRole("*")
            .build()
    );

    async.run("sleepy-framework", () -> Sleepy._main(fwId, uri.toString(), "1", "*"));

    subject.onNext(subscribed(fwId, 15));
    sim.awaitSubscribeCall();
    final List<Protos.Call> callsReceived1 = sim.getCallsReceived();
    assertThat(callsReceived1).hasSize(1);
    assertThat(callsReceived1.get(0)).isEqualTo(subscribe);

    // send a heartbeat
    subject.onNext(HEARTBEAT);
    // send an offer
    subject.onNext(resourceOffer("host-1", "offer-1", "agent-1", fwId, 4, 16 * 1024, 100 * 1024));

    sim.awaitCall();  // wait for accept to reach the server
    final List<Protos.Call> callsReceived2 = sim.getCallsReceived();
    assertThat(callsReceived2).hasSize(2);
    final Protos.Call.Accept accept = callsReceived2.get(1).getAccept();
    assertThat(accept).isNotNull();
    assertThat(
        accept.getOfferIdsList().stream()
            .map(org.apache.mesos.v1.Protos.OfferID::getValue)
            .collect(Collectors.toList())
    ).isEqualTo(newArrayList("offer-1"));
    final List<org.apache.mesos.v1.Protos.TaskInfo> taskInfos = accept.getOperationsList().stream()
        .map(org.apache.mesos.v1.Protos.Offer.Operation::getLaunch)
        .flatMap(l -> l.getTaskInfosList().stream())
        .collect(Collectors.toList());
    assertThat(taskInfos).hasSize(4);
    assertThat(
        taskInfos.stream()
            .map(t -> t.getTaskId().getValue())
            .collect(Collectors.toSet())
    ).isEqualTo(newHashSet("task-1-1", "task-1-2", "task-1-3", "task-1-4"));

    // send task status updates
    final ByteString uuid1 = ByteString.copyFromUtf8(UUID.randomUUID().toString());
    final ByteString uuid2 = ByteString.copyFromUtf8(UUID.randomUUID().toString());
    final ByteString uuid3 = ByteString.copyFromUtf8(UUID.randomUUID().toString());
    final ByteString uuid4 = ByteString.copyFromUtf8(UUID.randomUUID().toString());
    subject.onNext(update("agent-1", "task-1-1", "task-1-1", TASK_RUNNING, uuid1));
    subject.onNext(update("agent-1", "task-1-2", "task-1-2", TASK_RUNNING, uuid2));
    subject.onNext(update("agent-1", "task-1-3", "task-1-3", TASK_RUNNING, uuid3));
    subject.onNext(update("agent-1", "task-1-4", "task-1-4", TASK_RUNNING, uuid4));

    // wait for the task status updates to be ack'd
    sim.awaitCall(4);
    final List<Protos.Call> callsReceived3 = sim.getCallsReceived();
    assertThat(callsReceived3).hasSize(6);
    final List<Protos.Call> ackCalls = callsReceived3.subList(2, 6);
    final Set<ByteString> ackdUuids = ackCalls.stream()
        .map(c -> c.getAcknowledge().getUuid())
        .collect(Collectors.toSet());

    assertThat(ackdUuids).isEqualTo(newHashSet(uuid1, uuid2, uuid3, uuid4));

    // send another offer with too little cpu for a task to run
    subject.onNext(resourceOffer("host-1", "offer-2", "agent-1", fwId, 0.9, 15 * 1024, 100 * 1024));

    // wait for the decline of the offer
    sim.awaitCall();
    final List<Protos.Call> callsReceived4 = sim.getCallsReceived();
    assertThat(callsReceived4).hasSize(7);
    final Protos.Call.Decline decline = callsReceived4.get(6).getDecline();
    assertThat(
        decline.getOfferIdsList().stream()
            .map(org.apache.mesos.v1.Protos.OfferID::getValue)
            .collect(Collectors.toList())
    ).isEqualTo(newArrayList("offer-2"));

    subject.onCompleted();
    sim.awaitSendingEvents();
}
 

/**
 * Utility method to more succinctly construct an {@link Protos.Event Event} of type
 * {@link Protos.Event.Type#OFFERS OFFERS}.
 *
 * @param hostname    The hostname to set on the offer.
 * @param offerId     The offerId to set on the offer.
 * @param agentId     The agentId to set on the offer.
 * @param frameworkId The frameworkId to set on the offer.
 * @param cpus        The number of cpus the offer will have.
 * @param mem         The number of megabytes of memory the offer will have.
 * @param disk        The number of megabytes of disk the offer will have.
 * @return An {@link Protos.Event Event} of type
 * {@link Protos.Event.Type#OFFERS OFFERS} containing a single
 * {@link org.apache.mesos.v1.Protos.Offer Offer} using the specified parameters as the values of the offer.
 */
@NotNull
public static Protos.Event resourceOffer(
    @NotNull final String hostname,
    @NotNull final String offerId,
    @NotNull final String agentId,
    @NotNull final String frameworkId,
    final double cpus,
    final long mem,
    final long disk
) {
    return Protos.Event.newBuilder()
        .setType(Protos.Event.Type.OFFERS)
        .setOffers(
            Protos.Event.Offers.newBuilder()
                .addAllOffers(Collections.singletonList(
                    org.apache.mesos.v1.Protos.Offer.newBuilder()
                        .setHostname(hostname)
                        .setId(org.apache.mesos.v1.Protos.OfferID.newBuilder().setValue(offerId))
                        .setAgentId(org.apache.mesos.v1.Protos.AgentID.newBuilder().setValue(agentId))
                        .setFrameworkId(org.apache.mesos.v1.Protos.FrameworkID.newBuilder().setValue(frameworkId))
                        .addResources(org.apache.mesos.v1.Protos.Resource.newBuilder()
                            .setName("cpus")
                            .setRole("*")
                            .setType(org.apache.mesos.v1.Protos.Value.Type.SCALAR)
                            .setScalar(org.apache.mesos.v1.Protos.Value.Scalar.newBuilder().setValue(cpus)))
                        .addResources(org.apache.mesos.v1.Protos.Resource.newBuilder()
                            .setName("mem")
                            .setRole("*")
                            .setType(org.apache.mesos.v1.Protos.Value.Type.SCALAR)
                            .setScalar(org.apache.mesos.v1.Protos.Value.Scalar.newBuilder().setValue(mem)))
                        .addResources(org.apache.mesos.v1.Protos.Resource.newBuilder()
                            .setName("disk")
                            .setRole("*")
                            .setType(org.apache.mesos.v1.Protos.Value.Type.SCALAR)
                            .setScalar(org.apache.mesos.v1.Protos.Value.Scalar.newBuilder().setValue(disk)))
                        .build()
                ))
        )
        .build();
}
 

private Protos.Call acknowledgeCall(TaskStatus status) {
    return SchedulerCalls.ackUpdate(frameworkId, status.getUuid(), status.getAgentId(), status.getTaskId());
}
 

public static void declineOffer(final @NotNull Protocol protocol, final @NotNull String streamId, final @NotNull FrameworkID frameworkID,
                                final @NotNull Protos.Call call, final @NotNull String url) {
    fixedSendPool.submit(() -> decline(protocol, streamId, frameworkID, call, url));
}
 

/**
 * Utility method to more succinctly construct an {@link Protos.Event Event} of type
 * {@link Protos.Event.Type#MESSAGE MESSAGE}.
 *
 * @param agentId        The {@link org.apache.mesos.v1.Protos.AgentID#getValue() value} of the
 *                       {@link Protos.Event.Message#getAgentId() agentId} to be set on the
 *                       {@link Protos.Event.Message Message}.
 * @param executorId     The {@link org.apache.mesos.v1.Protos.ExecutorID#getValue() value} of the
 *                       {@link Protos.Event.Message#getExecutorId() executorId} to be set on the
 *                       {@link Protos.Event.Message Message}.
 * @param data           The {@link Protos.Event.Message#getData() data} to be set on the
 *                       {@link Protos.Event.Message Message}.
 * @return  A {@link Protos.Call Call} with a configured {@link Protos.Call.Acknowledge Acknowledge}.
 */
@NotNull
public static Protos.Event message(
    @NotNull final String agentId,
    @NotNull final String executorId,
    @NotNull final ByteString data
) {
    return message(
        org.apache.mesos.v1.Protos.AgentID.newBuilder().setValue(agentId).build(),
        org.apache.mesos.v1.Protos.ExecutorID.newBuilder().setValue(executorId).build(),
        data
    );
}