org.apache.flink.runtime.taskexecutor.AccumulatorReport Java Examples
The following examples show how to use
org.apache.flink.runtime.taskexecutor.AccumulatorReport.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: JobMasterTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testJobFailureWhenTaskExecutorHeartbeatTimeout() throws Exception { final TestingHeartbeatServices testingHeartbeatService = new TestingHeartbeatServices(heartbeatInterval, heartbeatTimeout); runJobFailureWhenTaskExecutorTerminatesTest( testingHeartbeatService, (localTaskManagerLocation, jobMasterGateway) -> testingHeartbeatService.triggerHeartbeatTimeout( jmResourceId, localTaskManagerLocation.getResourceID()), (jobMasterGateway, taskManagerResourceId) -> (resourceId, ignored) -> { jobMasterGateway.heartbeatFromTaskManager(taskManagerResourceId, new AccumulatorReport(Collections.emptyList())); } ); }
Example #2
Source File: JobMaster.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public void reportPayload(ResourceID resourceID, AccumulatorReport payload) { validateRunsInMainThread(); for (AccumulatorSnapshot snapshot : payload.getAccumulatorSnapshots()) { executionGraph.updateAccumulators(snapshot); } }
Example #3
Source File: JobMasterTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testJobFailureWhenTaskExecutorHeartbeatTimeout() throws Exception { final AtomicBoolean respondToHeartbeats = new AtomicBoolean(true); runJobFailureWhenTaskExecutorTerminatesTest( fastHeartbeatServices, (localTaskManagerLocation, jobMasterGateway) -> respondToHeartbeats.set(false), (jobMasterGateway, taskManagerResourceId) -> (resourceId, ignored) -> { if (respondToHeartbeats.get()) { jobMasterGateway.heartbeatFromTaskManager(taskManagerResourceId, new AccumulatorReport(Collections.emptyList())); } } ); }
Example #4
Source File: JobMaster.java From flink with Apache License 2.0 | 5 votes |
@Override public void reportPayload(ResourceID resourceID, AccumulatorReport payload) { validateRunsInMainThread(); for (AccumulatorSnapshot snapshot : payload.getAccumulatorSnapshots()) { schedulerNG.updateAccumulators(snapshot); } }
Example #5
Source File: JobMaster.java From flink with Apache License 2.0 | 5 votes |
@Override public void reportPayload(ResourceID resourceID, AccumulatorReport payload) { validateRunsInMainThread(); for (AccumulatorSnapshot snapshot : payload.getAccumulatorSnapshots()) { schedulerNG.updateAccumulators(snapshot); } }
Example #6
Source File: JobMasterTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testJobFailureWhenTaskExecutorHeartbeatTimeout() throws Exception { final AtomicBoolean respondToHeartbeats = new AtomicBoolean(true); runJobFailureWhenTaskExecutorTerminatesTest( fastHeartbeatServices, (localTaskManagerLocation, jobMasterGateway) -> respondToHeartbeats.set(false), (jobMasterGateway, taskManagerResourceId) -> (resourceId, ignored) -> { if (respondToHeartbeats.get()) { jobMasterGateway.heartbeatFromTaskManager(taskManagerResourceId, new AccumulatorReport(Collections.emptyList())); } } ); }
Example #7
Source File: TestingJobMasterGatewayBuilder.java From flink with Apache License 2.0 | 4 votes |
public TestingJobMasterGatewayBuilder setTaskManagerHeartbeatConsumer(BiConsumer<ResourceID, AccumulatorReport> taskManagerHeartbeatConsumer) { this.taskManagerHeartbeatConsumer = taskManagerHeartbeatConsumer; return this; }
Example #8
Source File: TestingJobMasterGateway.java From flink with Apache License 2.0 | 4 votes |
@Override public void heartbeatFromTaskManager(ResourceID resourceID, AccumulatorReport accumulatorReport) { taskManagerHeartbeatConsumer.accept(resourceID, accumulatorReport); }
Example #9
Source File: TestingJobMasterGateway.java From flink with Apache License 2.0 | 4 votes |
public TestingJobMasterGateway( @Nonnull String address, @Nonnull String hostname, @Nonnull Supplier<CompletableFuture<Acknowledge>> cancelFunction, @Nonnull Function<TaskExecutionState, CompletableFuture<Acknowledge>> updateTaskExecutionStateFunction, @Nonnull BiFunction<JobVertexID, ExecutionAttemptID, CompletableFuture<SerializedInputSplit>> requestNextInputSplitFunction, @Nonnull BiFunction<IntermediateDataSetID, ResultPartitionID, CompletableFuture<ExecutionState>> requestPartitionStateFunction, @Nonnull Function<ResultPartitionID, CompletableFuture<Acknowledge>> scheduleOrUpdateConsumersFunction, @Nonnull Function<ResourceID, CompletableFuture<Acknowledge>> disconnectTaskManagerFunction, @Nonnull Consumer<ResourceManagerId> disconnectResourceManagerConsumer, @Nonnull BiFunction<ResourceID, Collection<SlotOffer>, CompletableFuture<Collection<SlotOffer>>> offerSlotsFunction, @Nonnull TriConsumer<ResourceID, AllocationID, Throwable> failSlotConsumer, @Nonnull BiFunction<String, UnresolvedTaskManagerLocation, CompletableFuture<RegistrationResponse>> registerTaskManagerFunction, @Nonnull BiConsumer<ResourceID, AccumulatorReport> taskManagerHeartbeatConsumer, @Nonnull Consumer<ResourceID> resourceManagerHeartbeatConsumer, @Nonnull Supplier<CompletableFuture<JobDetails>> requestJobDetailsSupplier, @Nonnull Supplier<CompletableFuture<ArchivedExecutionGraph>> requestJobSupplier, @Nonnull BiFunction<String, Boolean, CompletableFuture<String>> triggerSavepointFunction, @Nonnull BiFunction<String, Boolean, CompletableFuture<String>> stopWithSavepointFunction, @Nonnull Function<JobVertexID, CompletableFuture<OperatorBackPressureStatsResponse>> requestOperatorBackPressureStatsFunction, @Nonnull BiConsumer<AllocationID, Throwable> notifyAllocationFailureConsumer, @Nonnull Consumer<Tuple5<JobID, ExecutionAttemptID, Long, CheckpointMetrics, TaskStateSnapshot>> acknowledgeCheckpointConsumer, @Nonnull Consumer<DeclineCheckpoint> declineCheckpointConsumer, @Nonnull Supplier<JobMasterId> fencingTokenSupplier, @Nonnull BiFunction<JobID, String, CompletableFuture<KvStateLocation>> requestKvStateLocationFunction, @Nonnull Function<Tuple6<JobID, JobVertexID, KeyGroupRange, String, KvStateID, InetSocketAddress>, CompletableFuture<Acknowledge>> notifyKvStateRegisteredFunction, @Nonnull Function<Tuple4<JobID, JobVertexID, KeyGroupRange, String>, CompletableFuture<Acknowledge>> notifyKvStateUnregisteredFunction, @Nonnull TriFunction<String, Object, byte[], CompletableFuture<Object>> updateAggregateFunction, @Nonnull TriFunction<ExecutionAttemptID, OperatorID, SerializedValue<OperatorEvent>, CompletableFuture<Acknowledge>> operatorEventSender, @Nonnull BiFunction<OperatorID, SerializedValue<CoordinationRequest>, CompletableFuture<CoordinationResponse>> deliverCoordinationRequestFunction) { this.address = address; this.hostname = hostname; this.cancelFunction = cancelFunction; this.updateTaskExecutionStateFunction = updateTaskExecutionStateFunction; this.requestNextInputSplitFunction = requestNextInputSplitFunction; this.requestPartitionStateFunction = requestPartitionStateFunction; this.scheduleOrUpdateConsumersFunction = scheduleOrUpdateConsumersFunction; this.disconnectTaskManagerFunction = disconnectTaskManagerFunction; this.disconnectResourceManagerConsumer = disconnectResourceManagerConsumer; this.offerSlotsFunction = offerSlotsFunction; this.failSlotConsumer = failSlotConsumer; this.registerTaskManagerFunction = registerTaskManagerFunction; this.taskManagerHeartbeatConsumer = taskManagerHeartbeatConsumer; this.resourceManagerHeartbeatConsumer = resourceManagerHeartbeatConsumer; this.requestJobDetailsSupplier = requestJobDetailsSupplier; this.requestJobSupplier = requestJobSupplier; this.triggerSavepointFunction = triggerSavepointFunction; this.stopWithSavepointFunction = stopWithSavepointFunction; this.requestOperatorBackPressureStatsFunction = requestOperatorBackPressureStatsFunction; this.notifyAllocationFailureConsumer = notifyAllocationFailureConsumer; this.acknowledgeCheckpointConsumer = acknowledgeCheckpointConsumer; this.declineCheckpointConsumer = declineCheckpointConsumer; this.fencingTokenSupplier = fencingTokenSupplier; this.requestKvStateLocationFunction = requestKvStateLocationFunction; this.notifyKvStateRegisteredFunction = notifyKvStateRegisteredFunction; this.notifyKvStateUnregisteredFunction = notifyKvStateUnregisteredFunction; this.updateAggregateFunction = updateAggregateFunction; this.operatorEventSender = operatorEventSender; this.deliverCoordinationRequestFunction = deliverCoordinationRequestFunction; }
Example #10
Source File: TestingJobMasterGatewayBuilder.java From flink with Apache License 2.0 | 4 votes |
public TestingJobMasterGatewayBuilder setTaskManagerHeartbeatConsumer(BiConsumer<ResourceID, AccumulatorReport> taskManagerHeartbeatConsumer) { this.taskManagerHeartbeatConsumer = taskManagerHeartbeatConsumer; return this; }
Example #11
Source File: JobMaster.java From flink with Apache License 2.0 | 4 votes |
@Override public void heartbeatFromTaskManager(final ResourceID resourceID, AccumulatorReport accumulatorReport) { taskManagerHeartbeatManager.receiveHeartbeat(resourceID, accumulatorReport); }
Example #12
Source File: TestingJobMasterGateway.java From flink with Apache License 2.0 | 4 votes |
@Override public void heartbeatFromTaskManager(ResourceID resourceID, AccumulatorReport accumulatorReport) { taskManagerHeartbeatConsumer.accept(resourceID, accumulatorReport); }
Example #13
Source File: TestingJobMasterGateway.java From flink with Apache License 2.0 | 4 votes |
public TestingJobMasterGateway( @Nonnull String address, @Nonnull String hostname, @Nonnull Supplier<CompletableFuture<Acknowledge>> cancelFunction, @Nonnull Function<TaskExecutionState, CompletableFuture<Acknowledge>> updateTaskExecutionStateFunction, @Nonnull BiFunction<JobVertexID, ExecutionAttemptID, CompletableFuture<SerializedInputSplit>> requestNextInputSplitFunction, @Nonnull BiFunction<IntermediateDataSetID, ResultPartitionID, CompletableFuture<ExecutionState>> requestPartitionStateFunction, @Nonnull Function<ResultPartitionID, CompletableFuture<Acknowledge>> scheduleOrUpdateConsumersFunction, @Nonnull Function<ResourceID, CompletableFuture<Acknowledge>> disconnectTaskManagerFunction, @Nonnull Consumer<ResourceManagerId> disconnectResourceManagerConsumer, @Nonnull Supplier<CompletableFuture<ClassloadingProps>> classloadingPropsSupplier, @Nonnull BiFunction<ResourceID, Collection<SlotOffer>, CompletableFuture<Collection<SlotOffer>>> offerSlotsFunction, @Nonnull TriConsumer<ResourceID, AllocationID, Throwable> failSlotConsumer, @Nonnull BiFunction<String, TaskManagerLocation, CompletableFuture<RegistrationResponse>> registerTaskManagerFunction, @Nonnull BiConsumer<ResourceID, AccumulatorReport> taskManagerHeartbeatConsumer, @Nonnull Consumer<ResourceID> resourceManagerHeartbeatConsumer, @Nonnull Supplier<CompletableFuture<JobDetails>> requestJobDetailsSupplier, @Nonnull Supplier<CompletableFuture<ArchivedExecutionGraph>> requestJobSupplier, @Nonnull BiFunction<String, Boolean, CompletableFuture<String>> triggerSavepointFunction, @Nonnull BiFunction<String, Boolean, CompletableFuture<String>> stopWithSavepointFunction, @Nonnull Function<JobVertexID, CompletableFuture<OperatorBackPressureStatsResponse>> requestOperatorBackPressureStatsFunction, @Nonnull BiConsumer<AllocationID, Throwable> notifyAllocationFailureConsumer, @Nonnull Consumer<Tuple5<JobID, ExecutionAttemptID, Long, CheckpointMetrics, TaskStateSnapshot>> acknowledgeCheckpointConsumer, @Nonnull Consumer<DeclineCheckpoint> declineCheckpointConsumer, @Nonnull Supplier<JobMasterId> fencingTokenSupplier, @Nonnull BiFunction<JobID, String, CompletableFuture<KvStateLocation>> requestKvStateLocationFunction, @Nonnull Function<Tuple6<JobID, JobVertexID, KeyGroupRange, String, KvStateID, InetSocketAddress>, CompletableFuture<Acknowledge>> notifyKvStateRegisteredFunction, @Nonnull Function<Tuple4<JobID, JobVertexID, KeyGroupRange, String>, CompletableFuture<Acknowledge>> notifyKvStateUnregisteredFunction, @Nonnull TriFunction<String, Object, byte[], CompletableFuture<Object>> updateAggregateFunction) { this.address = address; this.hostname = hostname; this.cancelFunction = cancelFunction; this.updateTaskExecutionStateFunction = updateTaskExecutionStateFunction; this.requestNextInputSplitFunction = requestNextInputSplitFunction; this.requestPartitionStateFunction = requestPartitionStateFunction; this.scheduleOrUpdateConsumersFunction = scheduleOrUpdateConsumersFunction; this.disconnectTaskManagerFunction = disconnectTaskManagerFunction; this.disconnectResourceManagerConsumer = disconnectResourceManagerConsumer; this.classloadingPropsSupplier = classloadingPropsSupplier; this.offerSlotsFunction = offerSlotsFunction; this.failSlotConsumer = failSlotConsumer; this.registerTaskManagerFunction = registerTaskManagerFunction; this.taskManagerHeartbeatConsumer = taskManagerHeartbeatConsumer; this.resourceManagerHeartbeatConsumer = resourceManagerHeartbeatConsumer; this.requestJobDetailsSupplier = requestJobDetailsSupplier; this.requestJobSupplier = requestJobSupplier; this.triggerSavepointFunction = triggerSavepointFunction; this.stopWithSavepointFunction = stopWithSavepointFunction; this.requestOperatorBackPressureStatsFunction = requestOperatorBackPressureStatsFunction; this.notifyAllocationFailureConsumer = notifyAllocationFailureConsumer; this.acknowledgeCheckpointConsumer = acknowledgeCheckpointConsumer; this.declineCheckpointConsumer = declineCheckpointConsumer; this.fencingTokenSupplier = fencingTokenSupplier; this.requestKvStateLocationFunction = requestKvStateLocationFunction; this.notifyKvStateRegisteredFunction = notifyKvStateRegisteredFunction; this.notifyKvStateUnregisteredFunction = notifyKvStateUnregisteredFunction; this.updateAggregateFunction = updateAggregateFunction; }
Example #14
Source File: JobMaster.java From flink with Apache License 2.0 | 4 votes |
@Override public void heartbeatFromTaskManager(final ResourceID resourceID, AccumulatorReport accumulatorReport) { taskManagerHeartbeatManager.receiveHeartbeat(resourceID, accumulatorReport); }
Example #15
Source File: TestingJobMasterGateway.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
@Override public void heartbeatFromTaskManager(ResourceID resourceID, AccumulatorReport accumulatorReport) { taskManagerHeartbeatConsumer.accept(resourceID, accumulatorReport); }
Example #16
Source File: TestingJobMasterGateway.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
public TestingJobMasterGateway( @Nonnull String address, @Nonnull String hostname, @Nonnull Supplier<CompletableFuture<Acknowledge>> cancelFunction, @Nonnull Supplier<CompletableFuture<Acknowledge>> stopFunction, @Nonnull BiFunction<Integer, RescalingBehaviour, CompletableFuture<Acknowledge>> rescalingJobFunction, @Nonnull TriFunction<Collection<JobVertexID>, Integer, RescalingBehaviour, CompletableFuture<Acknowledge>> rescalingOperatorsFunction, @Nonnull Function<TaskExecutionState, CompletableFuture<Acknowledge>> updateTaskExecutionStateFunction, @Nonnull BiFunction<JobVertexID, ExecutionAttemptID, CompletableFuture<SerializedInputSplit>> requestNextInputSplitFunction, @Nonnull BiFunction<IntermediateDataSetID, ResultPartitionID, CompletableFuture<ExecutionState>> requestPartitionStateFunction, @Nonnull Function<ResultPartitionID, CompletableFuture<Acknowledge>> scheduleOrUpdateConsumersFunction, @Nonnull Function<ResourceID, CompletableFuture<Acknowledge>> disconnectTaskManagerFunction, @Nonnull Consumer<ResourceManagerId> disconnectResourceManagerConsumer, @Nonnull Supplier<CompletableFuture<ClassloadingProps>> classloadingPropsSupplier, @Nonnull BiFunction<ResourceID, Collection<SlotOffer>, CompletableFuture<Collection<SlotOffer>>> offerSlotsFunction, @Nonnull TriConsumer<ResourceID, AllocationID, Throwable> failSlotConsumer, @Nonnull BiFunction<String, TaskManagerLocation, CompletableFuture<RegistrationResponse>> registerTaskManagerFunction, @Nonnull BiConsumer<ResourceID, AccumulatorReport> taskManagerHeartbeatConsumer, @Nonnull Consumer<ResourceID> resourceManagerHeartbeatConsumer, @Nonnull Supplier<CompletableFuture<JobDetails>> requestJobDetailsSupplier, @Nonnull Supplier<CompletableFuture<ArchivedExecutionGraph>> requestJobSupplier, @Nonnull BiFunction<String, Boolean, CompletableFuture<String>> triggerSavepointFunction, @Nonnull Function<JobVertexID, CompletableFuture<OperatorBackPressureStatsResponse>> requestOperatorBackPressureStatsFunction, @Nonnull BiConsumer<AllocationID, Throwable> notifyAllocationFailureConsumer, @Nonnull Consumer<Tuple5<JobID, ExecutionAttemptID, Long, CheckpointMetrics, TaskStateSnapshot>> acknowledgeCheckpointConsumer, @Nonnull Consumer<DeclineCheckpoint> declineCheckpointConsumer, @Nonnull Supplier<JobMasterId> fencingTokenSupplier, @Nonnull BiFunction<JobID, String, CompletableFuture<KvStateLocation>> requestKvStateLocationFunction, @Nonnull Function<Tuple6<JobID, JobVertexID, KeyGroupRange, String, KvStateID, InetSocketAddress>, CompletableFuture<Acknowledge>> notifyKvStateRegisteredFunction, @Nonnull Function<Tuple4<JobID, JobVertexID, KeyGroupRange, String>, CompletableFuture<Acknowledge>> notifyKvStateUnregisteredFunction, @Nonnull TriFunction<String, Object, byte[], CompletableFuture<Object>> updateAggregateFunction) { this.address = address; this.hostname = hostname; this.cancelFunction = cancelFunction; this.stopFunction = stopFunction; this.rescalingJobFunction = rescalingJobFunction; this.rescalingOperatorsFunction = rescalingOperatorsFunction; this.updateTaskExecutionStateFunction = updateTaskExecutionStateFunction; this.requestNextInputSplitFunction = requestNextInputSplitFunction; this.requestPartitionStateFunction = requestPartitionStateFunction; this.scheduleOrUpdateConsumersFunction = scheduleOrUpdateConsumersFunction; this.disconnectTaskManagerFunction = disconnectTaskManagerFunction; this.disconnectResourceManagerConsumer = disconnectResourceManagerConsumer; this.classloadingPropsSupplier = classloadingPropsSupplier; this.offerSlotsFunction = offerSlotsFunction; this.failSlotConsumer = failSlotConsumer; this.registerTaskManagerFunction = registerTaskManagerFunction; this.taskManagerHeartbeatConsumer = taskManagerHeartbeatConsumer; this.resourceManagerHeartbeatConsumer = resourceManagerHeartbeatConsumer; this.requestJobDetailsSupplier = requestJobDetailsSupplier; this.requestJobSupplier = requestJobSupplier; this.triggerSavepointFunction = triggerSavepointFunction; this.requestOperatorBackPressureStatsFunction = requestOperatorBackPressureStatsFunction; this.notifyAllocationFailureConsumer = notifyAllocationFailureConsumer; this.acknowledgeCheckpointConsumer = acknowledgeCheckpointConsumer; this.declineCheckpointConsumer = declineCheckpointConsumer; this.fencingTokenSupplier = fencingTokenSupplier; this.requestKvStateLocationFunction = requestKvStateLocationFunction; this.notifyKvStateRegisteredFunction = notifyKvStateRegisteredFunction; this.notifyKvStateUnregisteredFunction = notifyKvStateUnregisteredFunction; this.updateAggregateFunction = updateAggregateFunction; }
Example #17
Source File: TestingJobMasterGatewayBuilder.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
public TestingJobMasterGatewayBuilder setTaskManagerHeartbeatConsumer(BiConsumer<ResourceID, AccumulatorReport> taskManagerHeartbeatConsumer) { this.taskManagerHeartbeatConsumer = taskManagerHeartbeatConsumer; return this; }
Example #18
Source File: JobMaster.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
@Override public void heartbeatFromTaskManager(final ResourceID resourceID, AccumulatorReport accumulatorReport) { taskManagerHeartbeatManager.receiveHeartbeat(resourceID, accumulatorReport); }
Example #19
Source File: JobMasterGateway.java From Flink-CEPplus with Apache License 2.0 | 2 votes |
/** * Sends the heartbeat to job manager from task manager. * * @param resourceID unique id of the task manager * @param accumulatorReport report containing accumulator updates */ void heartbeatFromTaskManager( final ResourceID resourceID, final AccumulatorReport accumulatorReport);
Example #20
Source File: JobMasterGateway.java From flink with Apache License 2.0 | 2 votes |
/** * Sends the heartbeat to job manager from task manager. * * @param resourceID unique id of the task manager * @param accumulatorReport report containing accumulator updates */ void heartbeatFromTaskManager( final ResourceID resourceID, final AccumulatorReport accumulatorReport);
Example #21
Source File: JobMasterGateway.java From flink with Apache License 2.0 | 2 votes |
/** * Sends the heartbeat to job manager from task manager. * * @param resourceID unique id of the task manager * @param accumulatorReport report containing accumulator updates */ void heartbeatFromTaskManager( final ResourceID resourceID, final AccumulatorReport accumulatorReport);