org.elasticsearch.action.support.PlainActionFuture Java Examples

@SuppressWarnings("unchecked")
public void testExecutionException() {
    TransportService transportService = new TransportService(
        Settings.EMPTY,
        mock(Transport.class),
        null,
        TransportService.NOOP_TRANSPORT_INTERCEPTOR,
        x -> null,
        null,
        Collections.emptySet()
    );

    ModelManager manager = mock(ModelManager.class);
    ThresholdResultTransportAction action = new ThresholdResultTransportAction(mock(ActionFilters.class), transportService, manager);
    doThrow(NullPointerException.class)
        .when(manager)
        .getThresholdingResult(any(String.class), any(String.class), anyDouble(), any(ActionListener.class));

    final PlainActionFuture<ThresholdResultResponse> future = new PlainActionFuture<>();
    ThresholdResultRequest request = new ThresholdResultRequest("123", "123-threshold", 2);
    action.doExecute(mock(Task.class), request, future);

    expectThrows(NullPointerException.class, () -> future.actionGet());
}
 

void assertChildren(BlobPath path, Collection<String> children) throws Exception {
    final PlainActionFuture<Set<String>> future = PlainActionFuture.newFuture();
    final BlobStoreRepository repository = getRepository();
    repository.threadPool().generic().execute(new ActionRunnable<>(future) {
        @Override
        protected void doRun() throws Exception {
            final BlobStore blobStore = repository.blobStore();
            future.onResponse(blobStore.blobContainer(path).children().keySet());
        }
    });
    Set<String> foundChildren = future.actionGet();
    if (children.isEmpty()) {
        assertThat(foundChildren, empty());
    } else {
        assertThat(foundChildren, containsInAnyOrder(children.toArray(Strings.EMPTY_ARRAY)));
    }
}
 

void assertBlobsByPrefix(BlobPath path, String prefix, Map<String, BlobMetaData> blobs) throws Exception {
    final PlainActionFuture<Map<String, BlobMetaData>> future = PlainActionFuture.newFuture();
    final BlobStoreRepository repository = getRepository();
    repository.threadPool().generic().execute(new ActionRunnable<>(future) {
        @Override
        protected void doRun() throws Exception {
            final BlobStore blobStore = repository.blobStore();
            future.onResponse(blobStore.blobContainer(path).listBlobsByPrefix(prefix));
        }
    });
    Map<String, BlobMetaData> foundBlobs = future.actionGet();
    if (blobs.isEmpty()) {
        assertThat(foundBlobs.keySet(), empty());
    } else {
        assertThat(foundBlobs.keySet(), containsInAnyOrder(blobs.keySet().toArray(Strings.EMPTY_ARRAY)));
        for (Map.Entry<String, BlobMetaData> entry : foundBlobs.entrySet()) {
            assertEquals(entry.getValue().length(), blobs.get(entry.getKey()).length());
        }
    }
}
 

protected void publish(ClusterChangedEvent clusterChangedEvent, TaskOutputs taskOutputs, long startTimeNS) {
    final PlainActionFuture<Void> fut = new PlainActionFuture<Void>() {
        @Override
        protected boolean blockingAllowed() {
            return isMasterUpdateThread() || super.blockingAllowed();
        }
    };
    clusterStatePublisher.publish(clusterChangedEvent, fut, taskOutputs.createAckListener(threadPool, clusterChangedEvent.state()));

    // indefinitely wait for publication to complete
    try {
        FutureUtils.get(fut);
        onPublicationSuccess(clusterChangedEvent, taskOutputs, startTimeNS);
    } catch (Exception e) {
        onPublicationFailed(clusterChangedEvent, taskOutputs, startTimeNS, e);
    }
}
 

/**
 * Closes the channels.
 *
 * @param channels to close
 * @param blocking indicates if we should block on channel close
 */
static <C extends TcpChannel> void closeChannels(List<C> channels, boolean blocking) {
    if (blocking) {
        ArrayList<ActionFuture<Void>> futures = new ArrayList<>(channels.size());
        for (final C channel : channels) {
            if (channel.isOpen()) {
                PlainActionFuture<Void> closeFuture = PlainActionFuture.newFuture();
                channel.addCloseListener(closeFuture);
                channel.close();
                futures.add(closeFuture);
            }
        }
        blockOnFutures(futures);
    } else {
        Releasables.close(channels);
    }
}
 

/**
 * @return an array with the rowCounts
 */
private long[] executeBulk(String stmt, Object[][] bulkArgs, TimeValue timeout) {
    try {
        AdapterActionFuture<long[], long[]> actionFuture = new PlainActionFuture<>();
        execute(stmt, bulkArgs, actionFuture);
        return actionFuture.actionGet(timeout);
    } catch (ElasticsearchTimeoutException e) {
        LOGGER.error("Timeout on SQL statement: {}", e, stmt);
        throw e;
    }
}
 

@SuppressWarnings("unchecked")
public void testNormal() {
    TransportService transportService = new TransportService(
        Settings.EMPTY,
        mock(Transport.class),
        null,
        TransportService.NOOP_TRANSPORT_INTERCEPTOR,
        x -> null,
        null,
        Collections.emptySet()
    );

    ModelManager manager = mock(ModelManager.class);
    ThresholdResultTransportAction action = new ThresholdResultTransportAction(mock(ActionFilters.class), transportService, manager);
    doAnswer(invocation -> {
        ActionListener<ThresholdingResult> listener = invocation.getArgument(3);
        listener.onResponse(new ThresholdingResult(0, 1.0d));
        return null;
    }).when(manager).getThresholdingResult(any(String.class), any(String.class), anyDouble(), any(ActionListener.class));

    final PlainActionFuture<ThresholdResultResponse> future = new PlainActionFuture<>();
    ThresholdResultRequest request = new ThresholdResultRequest("123", "123-threshold", 2);
    action.doExecute(mock(Task.class), request, future);

    ThresholdResultResponse response = future.actionGet();
    assertEquals(0, response.getAnomalyGrade(), 0.001);
    assertEquals(1, response.getConfidence(), 0.001);
}
 

private <T> void assertListenerThrows(String msg, PlainActionFuture<T> listener, Class<?> klass) throws InterruptedException {
    try {
        listener.get();
        fail(msg);
    } catch (ExecutionException ex) {
        assertThat(ex.getCause(), instanceOf(klass));
    }
}
 

@Test
public void testListChildren() throws Exception {
    final BlobStoreRepository repo = getRepository();
    final PlainActionFuture<Void> future = PlainActionFuture.newFuture();
    final Executor genericExec = repo.threadPool().generic();
    final int testBlobLen = randomIntBetween(1, 100);
    genericExec.execute(new ActionRunnable<>(future) {
        @Override
        protected void doRun() throws Exception {
            final BlobStore blobStore = repo.blobStore();
            blobStore.blobContainer(repo.basePath().add("foo"))
                .writeBlob("nested-blob", new ByteArrayInputStream(randomByteArrayOfLength(testBlobLen)), testBlobLen, false);
            blobStore.blobContainer(repo.basePath().add("foo").add("nested"))
                .writeBlob("bar", new ByteArrayInputStream(randomByteArrayOfLength(testBlobLen)), testBlobLen, false);
            blobStore.blobContainer(repo.basePath().add("foo").add("nested2"))
                .writeBlob("blub", new ByteArrayInputStream(randomByteArrayOfLength(testBlobLen)), testBlobLen, false);
            future.onResponse(null);
        }
    });
    future.actionGet();
    assertChildren(repo.basePath(), Collections.singleton("foo"));
    assertBlobsByPrefix(repo.basePath(), "fo", Collections.emptyMap());
    assertChildren(repo.basePath().add("foo"), List.of("nested", "nested2"));
    assertBlobsByPrefix(repo.basePath().add("foo"), "nest",
                        Collections.singletonMap("nested-blob", new PlainBlobMetaData("nested-blob", testBlobLen)));
    assertChildren(repo.basePath().add("foo").add("nested"), Collections.emptyList());
}
 

@Test
public void testSendSnapshotStopOnError() throws Exception {
    final int fileChunkSizeInBytes = between(1, 10 * 1024);
    final StartRecoveryRequest request = getStartRecoveryRequest();
    final IndexShard shard = mock(IndexShard.class);
    when(shard.state()).thenReturn(IndexShardState.STARTED);
    final List<Translog.Operation> ops = new ArrayList<>();
    for (int numOps = between(1, 256), i = 0; i < numOps; i++) {
        final Engine.Index index = getIndex(Integer.toString(i));
        ops.add(new Translog.Index(index, new Engine.IndexResult(1, 1, i, true)));
    }
    final AtomicBoolean wasFailed = new AtomicBoolean();
    RecoveryTargetHandler recoveryTarget = new TestRecoveryTargetHandler() {
        @Override
        public void indexTranslogOperations(List<Translog.Operation> operations, int totalTranslogOps, long timestamp,
                                            long msu, ActionListener<Long> listener) {
            if (randomBoolean()) {
                maybeExecuteAsync(() -> listener.onResponse(SequenceNumbers.NO_OPS_PERFORMED));
            } else {
                maybeExecuteAsync(() -> listener.onFailure(new RuntimeException("test - failed to index")));
                wasFailed.set(true);
            }
        }
    };
    RecoverySourceHandler handler = new RecoverySourceHandler(shard, recoveryTarget, request, fileChunkSizeInBytes, between(1, 10));
    PlainActionFuture<RecoverySourceHandler.SendSnapshotResult> future = new PlainActionFuture<>();
    final long startingSeqNo = randomLongBetween(0, ops.size() - 1L);
    final long endingSeqNo = randomLongBetween(startingSeqNo, ops.size() - 1L);
    handler.phase2(startingSeqNo, startingSeqNo, endingSeqNo, newTranslogSnapshot(ops, Collections.emptyList()),
                   randomNonNegativeLong(), randomNonNegativeLong(), future);
    if (wasFailed.get()) {
        assertThat(expectThrows(RuntimeException.class, future::actionGet).getMessage(), equalTo("test - failed to index"));
    }
}
 

@Override
public final <Request extends TransportRequest, Response extends TransportResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> ActionFuture<Response> execute(
        Action<Request, Response, RequestBuilder> action, Request request) {
    PlainActionFuture<Response> actionFuture = PlainActionFuture.newFuture();
    execute(action, request, actionFuture);
    return actionFuture;
}
 

public void testCircuitBreaker() {

        ADCircuitBreakerService breakerService = mock(ADCircuitBreakerService.class);
        when(breakerService.isOpen()).thenReturn(true);

        // These constructors register handler in transport service
        new RCFResultTransportAction(new ActionFilters(Collections.emptySet()), transportService, normalModelManager, breakerService);
        new ThresholdResultTransportAction(new ActionFilters(Collections.emptySet()), transportService, normalModelManager);

        AnomalyResultTransportAction action = new AnomalyResultTransportAction(
            new ActionFilters(Collections.emptySet()),
            transportService,
            settings,
            stateManager,
            runner,
            featureQuery,
            normalModelManager,
            hashRing,
            clusterService,
            indexNameResolver,
            breakerService,
            adStats
        );

        AnomalyResultRequest request = new AnomalyResultRequest(adID, 100, 200);
        PlainActionFuture<AnomalyResultResponse> listener = new PlainActionFuture<>();
        action.doExecute(null, request, listener);

        assertException(listener, LimitExceededException.class);
    }
 

@SuppressWarnings("unchecked")
public void testNormal() {
    TransportService transportService = new TransportService(
        Settings.EMPTY,
        mock(Transport.class),
        null,
        TransportService.NOOP_TRANSPORT_INTERCEPTOR,
        x -> null,
        null,
        Collections.emptySet()
    );

    ModelManager manager = mock(ModelManager.class);
    ADCircuitBreakerService adCircuitBreakerService = mock(ADCircuitBreakerService.class);
    RCFResultTransportAction action = new RCFResultTransportAction(
        mock(ActionFilters.class),
        transportService,
        manager,
        adCircuitBreakerService
    );
    doAnswer(invocation -> {
        ActionListener<RcfResult> listener = invocation.getArgument(3);
        listener.onResponse(new RcfResult(0, 0, 25));
        return null;
    }).when(manager).getRcfResult(any(String.class), any(String.class), any(double[].class), any(ActionListener.class));

    when(adCircuitBreakerService.isOpen()).thenReturn(false);

    final PlainActionFuture<RCFResultResponse> future = new PlainActionFuture<>();
    RCFResultRequest request = new RCFResultRequest("123", "123-rcf-1", new double[] { 0 });
    action.doExecute(mock(Task.class), request, future);

    RCFResultResponse response = future.actionGet();
    assertEquals(0, response.getRCFScore(), 0.001);
    assertEquals(25, response.getForestSize(), 0.001);
}
 

@SuppressWarnings("unchecked")
public void testExecutionException() {
    TransportService transportService = new TransportService(
        Settings.EMPTY,
        mock(Transport.class),
        null,
        TransportService.NOOP_TRANSPORT_INTERCEPTOR,
        x -> null,
        null,
        Collections.emptySet()
    );

    ModelManager manager = mock(ModelManager.class);
    ADCircuitBreakerService adCircuitBreakerService = mock(ADCircuitBreakerService.class);
    RCFResultTransportAction action = new RCFResultTransportAction(
        mock(ActionFilters.class),
        transportService,
        manager,
        adCircuitBreakerService
    );
    doThrow(NullPointerException.class)
        .when(manager)
        .getRcfResult(any(String.class), any(String.class), any(double[].class), any(ActionListener.class));
    when(adCircuitBreakerService.isOpen()).thenReturn(false);

    final PlainActionFuture<RCFResultResponse> future = new PlainActionFuture<>();
    RCFResultRequest request = new RCFResultRequest("123", "123-rcf-1", new double[] { 0 });
    action.doExecute(mock(Task.class), request, future);

    expectThrows(NullPointerException.class, () -> future.actionGet());
}
 

@SuppressWarnings("unchecked")
public void testCircuitBreaker() {
    TransportService transportService = new TransportService(
        Settings.EMPTY,
        mock(Transport.class),
        null,
        TransportService.NOOP_TRANSPORT_INTERCEPTOR,
        x -> null,
        null,
        Collections.emptySet()
    );

    ModelManager manager = mock(ModelManager.class);
    ADCircuitBreakerService breakerService = mock(ADCircuitBreakerService.class);
    RCFResultTransportAction action = new RCFResultTransportAction(
        mock(ActionFilters.class),
        transportService,
        manager,
        breakerService
    );
    doAnswer(invocation -> {
        ActionListener<RcfResult> listener = invocation.getArgument(3);
        listener.onResponse(new RcfResult(0, 0, 25));
        return null;
    }).when(manager).getRcfResult(any(String.class), any(String.class), any(double[].class), any(ActionListener.class));
    when(breakerService.isOpen()).thenReturn(true);

    final PlainActionFuture<RCFResultResponse> future = new PlainActionFuture<>();
    RCFResultRequest request = new RCFResultRequest("123", "123-rcf-1", new double[] { 0 });
    action.doExecute(mock(Task.class), request, future);

    expectThrows(LimitExceededException.class, () -> future.actionGet());
}
 

@SuppressWarnings("unchecked")
public void testInsufficientCapacityExceptionDuringRestoringModel() {

    ModelManager rcfManager = mock(ModelManager.class);
    doThrow(new NotSerializableExceptionWrapper(new LimitExceededException(adID, CommonErrorMessages.MEMORY_LIMIT_EXCEEDED_ERR_MSG)))
        .when(rcfManager)
        .getRcfResult(any(String.class), any(String.class), any(double[].class), any(ActionListener.class));

    // These constructors register handler in transport service
    new RCFResultTransportAction(new ActionFilters(Collections.emptySet()), transportService, rcfManager, adCircuitBreakerService);
    new ThresholdResultTransportAction(new ActionFilters(Collections.emptySet()), transportService, normalModelManager);

    AnomalyResultTransportAction action = new AnomalyResultTransportAction(
        new ActionFilters(Collections.emptySet()),
        transportService,
        settings,
        stateManager,
        runner,
        featureQuery,
        normalModelManager,
        hashRing,
        clusterService,
        indexNameResolver,
        adCircuitBreakerService,
        adStats
    );

    AnomalyResultRequest request = new AnomalyResultRequest(adID, 100, 200);
    PlainActionFuture<AnomalyResultResponse> listener = new PlainActionFuture<>();
    action.doExecute(null, request, listener);

    assertException(listener, LimitExceededException.class);
}
 

@SuppressWarnings("unchecked")
public void testInsufficientCapacityExceptionDuringColdStart() {

    ModelManager rcfManager = mock(ModelManager.class);
    doThrow(ResourceNotFoundException.class)
        .when(rcfManager)
        .getRcfResult(any(String.class), any(String.class), any(double[].class), any(ActionListener.class));
    when(rcfManager.getRcfModelId(any(String.class), anyInt())).thenReturn(rcfModelID);

    ColdStartRunner mockRunner = mock(ColdStartRunner.class);
    when(mockRunner.fetchException(any(String.class)))
        .thenReturn(Optional.of(new LimitExceededException(adID, CommonErrorMessages.MEMORY_LIMIT_EXCEEDED_ERR_MSG)));

    // These constructors register handler in transport service
    new RCFResultTransportAction(new ActionFilters(Collections.emptySet()), transportService, rcfManager, adCircuitBreakerService);
    new ThresholdResultTransportAction(new ActionFilters(Collections.emptySet()), transportService, normalModelManager);

    AnomalyResultTransportAction action = new AnomalyResultTransportAction(
        new ActionFilters(Collections.emptySet()),
        transportService,
        settings,
        stateManager,
        mockRunner,
        featureQuery,
        normalModelManager,
        hashRing,
        clusterService,
        indexNameResolver,
        adCircuitBreakerService,
        adStats
    );

    AnomalyResultRequest request = new AnomalyResultRequest(adID, 100, 200);
    PlainActionFuture<AnomalyResultResponse> listener = new PlainActionFuture<>();
    action.doExecute(null, request, listener);

    assertException(listener, LimitExceededException.class);
}
 

public void testThresholdException() {

        ModelManager exceptionThreadholdfManager = mock(ModelManager.class);
        doThrow(NullPointerException.class)
            .when(exceptionThreadholdfManager)
            .getThresholdingResult(any(String.class), any(String.class), anyDouble());

        // These constructors register handler in transport service
        new RCFResultTransportAction(
            new ActionFilters(Collections.emptySet()),
            transportService,
            normalModelManager,
            adCircuitBreakerService
        );
        new ThresholdResultTransportAction(new ActionFilters(Collections.emptySet()), transportService, exceptionThreadholdfManager);

        AnomalyResultTransportAction action = new AnomalyResultTransportAction(
            new ActionFilters(Collections.emptySet()),
            transportService,
            settings,
            stateManager,
            runner,
            featureQuery,
            normalModelManager,
            hashRing,
            clusterService,
            indexNameResolver,
            adCircuitBreakerService,
            adStats
        );

        AnomalyResultRequest request = new AnomalyResultRequest(adID, 100, 200);
        PlainActionFuture<AnomalyResultResponse> listener = new PlainActionFuture<>();
        action.doExecute(null, request, listener);

        assertException(listener, AnomalyDetectionException.class);
    }
 

public Throwable noModelExceptionTemplate(
    Exception thrownException,
    ColdStartRunner globalRunner,
    String adID,
    Class<? extends Exception> expectedExceptionType,
    String error
) {

    ModelManager rcfManager = mock(ModelManager.class);
    doThrow(thrownException).when(rcfManager).getRcfResult(any(String.class), any(String.class), any(double[].class));
    when(rcfManager.getRcfModelId(any(String.class), anyInt())).thenReturn(rcfModelID);

    doNothing().when(normalModelManager).trainModel(any(AnomalyDetector.class), any(double[][].class));
    when(featureQuery.getColdStartData(any(AnomalyDetector.class))).thenReturn(Optional.of(new double[][] { { 0 } }));

    // These constructors register handler in transport service
    new RCFResultTransportAction(new ActionFilters(Collections.emptySet()), transportService, rcfManager, adCircuitBreakerService);
    new ThresholdResultTransportAction(new ActionFilters(Collections.emptySet()), transportService, normalModelManager);

    AnomalyResultTransportAction action = new AnomalyResultTransportAction(
        new ActionFilters(Collections.emptySet()),
        transportService,
        settings,
        stateManager,
        globalRunner,
        featureQuery,
        normalModelManager,
        hashRing,
        clusterService,
        indexNameResolver,
        adCircuitBreakerService,
        adStats
    );

    AnomalyResultRequest request = new AnomalyResultRequest(adID, 100, 200);
    PlainActionFuture<AnomalyResultResponse> listener = new PlainActionFuture<>();
    action.doExecute(null, request, listener);

    return assertException(listener, expectedExceptionType);
}
 

public void testNormal() throws IOException {

        // These constructors register handler in transport service
        new RCFResultTransportAction(
            new ActionFilters(Collections.emptySet()),
            transportService,
            normalModelManager,
            adCircuitBreakerService
        );
        new ThresholdResultTransportAction(new ActionFilters(Collections.emptySet()), transportService, normalModelManager);

        AnomalyResultTransportAction action = new AnomalyResultTransportAction(
            new ActionFilters(Collections.emptySet()),
            transportService,
            settings,
            stateManager,
            runner,
            featureQuery,
            normalModelManager,
            hashRing,
            clusterService,
            indexNameResolver,
            adCircuitBreakerService,
            adStats
        );

        AnomalyResultRequest request = new AnomalyResultRequest(adID, 100, 200);
        PlainActionFuture<AnomalyResultResponse> listener = new PlainActionFuture<>();
        action.doExecute(null, request, listener);

        AnomalyResultResponse response = listener.actionGet(10000L);
        assertAnomalyResultResponse(response, 0, 1, 0d);
    }
 

public ActionFuture<NodesGatewayMetaState> list(DiscoveryNode[] discoveryNodes, @Nullable TimeValue timeout) {
    PlainActionFuture<NodesGatewayMetaState> future = PlainActionFuture.newFuture();
    execute(new Request(discoveryNodes).timeout(timeout), future);
    return future;
}
 

@Test
public void testWaitForActiveShards() throws Exception {
    final String index = "test";
    final ShardId shardId = new ShardId(index, "_na_", 0);
    final int assignedReplicas = randomInt(2);
    final int unassignedReplicas = randomInt(2);
    final int totalShards = 1 + assignedReplicas + unassignedReplicas;
    final int activeShardCount = randomIntBetween(0, totalShards);
    Request request = new Request(shardId).waitForActiveShards(
        activeShardCount == totalShards ? ActiveShardCount.ALL : ActiveShardCount.from(activeShardCount));
    final boolean passesActiveShardCheck = activeShardCount <= assignedReplicas + 1;

    ShardRoutingState[] replicaStates = new ShardRoutingState[assignedReplicas + unassignedReplicas];
    for (int i = 0; i < assignedReplicas; i++) {
        replicaStates[i] = randomFrom(ShardRoutingState.STARTED, ShardRoutingState.RELOCATING);
    }
    for (int i = assignedReplicas; i < replicaStates.length; i++) {
        replicaStates[i] = ShardRoutingState.UNASSIGNED;
    }

    final ClusterState state = state(index, true, ShardRoutingState.STARTED, replicaStates);
    logger.debug("using active shard count of [{}], assigned shards [{}], total shards [{}]." +
                 " expecting op to [{}]. using state: \n{}",
                 request.waitForActiveShards(), 1 + assignedReplicas, 1 + assignedReplicas + unassignedReplicas,
                 passesActiveShardCheck ? "succeed" : "retry", state);
    final long primaryTerm = state.metaData().index(index).primaryTerm(shardId.id());
    final IndexShardRoutingTable shardRoutingTable = state.routingTable().index(index).shard(shardId.id());

    final Set<String> inSyncAllocationIds = state.metaData().index(index).inSyncAllocationIds(0);
    Set<String> trackedShards = new HashSet<>();
    addTrackingInfo(shardRoutingTable, null, trackedShards, new HashSet<>());
    final ReplicationGroup initialReplicationGroup = new ReplicationGroup(shardRoutingTable, inSyncAllocationIds, trackedShards);

    PlainActionFuture<TestPrimary.Result> listener = new PlainActionFuture<>();
    final ShardRouting primaryShard = shardRoutingTable.primaryShard();
    final TestReplicationOperation op = new TestReplicationOperation(request,
                                                                     new TestPrimary(primaryShard, () -> initialReplicationGroup),
                                                                     listener, new TestReplicaProxy(primaryTerm), logger, "test");

    if (passesActiveShardCheck) {
        assertThat(op.checkActiveShardCount(), nullValue());
        op.execute();
        assertTrue("operations should have been performed, active shard count is met",
                   request.processedOnPrimary.get());
    } else {
        assertThat(op.checkActiveShardCount(), notNullValue());
        op.execute();
        assertFalse("operations should not have been perform, active shard count is *NOT* met",
                    request.processedOnPrimary.get());
        assertListenerThrows("should throw exception to trigger retry", listener, UnavailableShardsException.class);
    }
}
 

/**
 * Recovers a replica from the give primary, allow the user to supply a custom recovery target. A typical usage of a custom recovery
 * target is to assert things in the various stages of recovery.
 *
 * Note: this method keeps the shard in {@link IndexShardState#POST_RECOVERY} and doesn't start it.
 *
 * @param replica                the recovery target shard
 * @param primary                the recovery source shard
 * @param targetSupplier         supplies an instance of {@link RecoveryTarget}
 * @param markAsRecovering       set to {@code false} if the replica is marked as recovering
 */
protected final void recoverUnstartedReplica(final IndexShard replica,
                                             final IndexShard primary,
                                             final BiFunction<IndexShard, DiscoveryNode, RecoveryTarget> targetSupplier,
                                             final boolean markAsRecovering,
                                             final Set<String> inSyncIds,
                                             final IndexShardRoutingTable routingTable) throws IOException {
    final DiscoveryNode pNode = getFakeDiscoNode(primary.routingEntry().currentNodeId());
    final DiscoveryNode rNode = getFakeDiscoNode(replica.routingEntry().currentNodeId());
    if (markAsRecovering) {
        replica.markAsRecovering("remote", new RecoveryState(replica.routingEntry(), pNode, rNode));
    } else {
        assertEquals(replica.state(), IndexShardState.RECOVERING);
    }
    replica.prepareForIndexRecovery();
    final RecoveryTarget recoveryTarget = targetSupplier.apply(replica, pNode);
    final String targetAllocationId = recoveryTarget.indexShard().routingEntry().allocationId().getId();

    final Store.MetadataSnapshot snapshot = getMetadataSnapshotOrEmpty(replica);
    final long startingSeqNo;
    if (snapshot.size() > 0) {
        startingSeqNo = PeerRecoveryTargetService.getStartingSeqNo(logger, recoveryTarget);
    } else {
        startingSeqNo = SequenceNumbers.UNASSIGNED_SEQ_NO;
    }

    final StartRecoveryRequest request = new StartRecoveryRequest(replica.shardId(), targetAllocationId,
        pNode, rNode, snapshot, replica.routingEntry().primary(), 0, startingSeqNo);
    final RecoverySourceHandler recovery = new RecoverySourceHandler(
        primary,
        recoveryTarget,
        request,
        Math.toIntExact(ByteSizeUnit.MB.toBytes(1)),
        between(1, 8));
    primary.updateShardState(
        primary.routingEntry(),
        primary.getPendingPrimaryTerm(),
        null,
        currentClusterStateVersion.incrementAndGet(),
        inSyncIds,
        routingTable
    );
    PlainActionFuture<RecoveryResponse> future = new PlainActionFuture<>();
    recovery.recoverToTarget(future);
    future.actionGet();
    recoveryTarget.markAsDone();
}
 

private Throwable assertException(PlainActionFuture<AnomalyResultResponse> listener, Class<? extends Exception> exceptionType) {
    return expectThrows(exceptionType, () -> listener.actionGet());
}
 

private void assertException(PlainActionFuture<AnomalyResultResponse> listener, Class<? extends Exception> exceptionType, String msg) {
    Exception e = expectThrows(exceptionType, () -> listener.actionGet());
    assertThat(e.getMessage(), containsString(msg));
}
 

public final ActionFuture<Response> execute(HttpInvocationContext<Request,Response> httpInvocationContext, Request request) {
    PlainActionFuture<Response> future = newFuture();
    execute(httpInvocationContext, future);
    return future;
}
 

public SyncRetryHandler(Class<? extends Throwable> retryOnThrowable, BackoffPolicy backoffPolicy, Client client, PlainActionFuture<BulkResponse> actionFuture) {
    super(retryOnThrowable, backoffPolicy, client, actionFuture);
    this.actionFuture = actionFuture;
}
 

public static SyncRetryHandler create(Class<? extends Throwable> retryOnThrowable, BackoffPolicy backoffPolicy, Client client) {
    PlainActionFuture<BulkResponse> actionFuture = PlainActionFuture.newFuture();
    return new SyncRetryHandler(retryOnThrowable, backoffPolicy, client, actionFuture);
}
 

@Override
public final <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> ActionFuture<Response> execute(Action<Request, Response, RequestBuilder> action, Request request) {
    PlainActionFuture<Response> actionFuture = PlainActionFuture.newFuture();
    execute(action, request, actionFuture);
    return actionFuture;
}
 

private void globalBlockTemplate(BlockType type, String errLogMsg, Settings indexSettings, String indexName) {
    ClusterState blockedClusterState = null;

    switch (type) {
        case GLOBAL_BLOCK_WRITE:
            blockedClusterState = ClusterState
                .builder(new ClusterName("test cluster"))
                .blocks(ClusterBlocks.builder().addGlobalBlock(IndexMetadata.INDEX_WRITE_BLOCK))
                .build();
            break;
        case GLOBAL_BLOCK_READ:
            blockedClusterState = ClusterState
                .builder(new ClusterName("test cluster"))
                .blocks(ClusterBlocks.builder().addGlobalBlock(IndexMetadata.INDEX_READ_BLOCK))
                .build();
            break;
        case INDEX_BLOCK:
            blockedClusterState = createIndexBlockedState(indexName, indexSettings, null);
            break;
        default:
            break;
    }

    ClusterService hackedClusterService = spy(clusterService);
    when(hackedClusterService.state()).thenReturn(blockedClusterState);

    // These constructors register handler in transport service
    new RCFResultTransportAction(
        new ActionFilters(Collections.emptySet()),
        transportService,
        normalModelManager,
        adCircuitBreakerService
    );
    new ThresholdResultTransportAction(new ActionFilters(Collections.emptySet()), transportService, normalModelManager);

    AnomalyResultTransportAction action = new AnomalyResultTransportAction(
        new ActionFilters(Collections.emptySet()),
        transportService,
        settings,
        stateManager,
        runner,
        featureQuery,
        normalModelManager,
        hashRing,
        hackedClusterService,
        indexNameResolver,
        adCircuitBreakerService,
        adStats
    );

    AnomalyResultRequest request = new AnomalyResultRequest(adID, 100, 200);
    PlainActionFuture<AnomalyResultResponse> listener = new PlainActionFuture<>();
    action.doExecute(null, request, listener);

    assertException(listener, AnomalyDetectionException.class, errLogMsg);
}