org.elasticsearch.ElasticsearchTimeoutException Java Examples

@Override
protected void doRun() throws Exception {
    RecoveryTarget status = onGoingRecoveries.get(recoveryId);
    if (status == null) {
        logger.trace("[monitor] no status found for [{}], shutting down", recoveryId);
        return;
    }
    long accessTime = status.lastAccessTime();
    if (accessTime == lastSeenAccessTime) {
        String message = "no activity after [" + checkInterval + "]";
        failRecovery(recoveryId,
                new RecoveryFailedException(status.state(), message, new ElasticsearchTimeoutException(message)),
                true // to be safe, we don't know what go stuck
        );
        return;
    }
    lastSeenAccessTime = accessTime;
    logger.trace("[monitor] rescheduling check for [{}]. last access time is [{}]", recoveryId, lastSeenAccessTime);
    threadPool.schedule(this, checkInterval, ThreadPool.Names.GENERIC);
}
 

@Before
public void startNodes() {
    try {
        logger.info("starting");
        setClusterName();
        startNode("1");
        findNodeAddress();
        try {
            ClusterHealthResponse healthResponse = client("1").execute(ClusterHealthAction.INSTANCE,
                            new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN).timeout(TimeValue.timeValueSeconds(30))).actionGet();
            if (healthResponse != null && healthResponse.isTimedOut()) {
                throw new IOException("cluster state is " + healthResponse.getStatus().name()
                        + ", from here on, everything will fail!");
            }
        } catch (ElasticsearchTimeoutException e) {
            throw new IOException("timeout, cluster does not respond to health request, cowardly refusing to continue with operations");
        }
    } catch (Throwable t) {
        logger.error("startNodes failed", t);
    }
}
 

public static void waitForCluster(Client client, ClusterHealthStatus statusThreshold,
    String timeout) throws UnableToStartException {
  try {
    ClusterHealthResponse healthResponse = (ClusterHealthResponse) client
        .execute(ClusterHealthAction.INSTANCE,
            new ClusterHealthRequest().waitForStatus(statusThreshold).timeout(timeout))
        .actionGet();
    if (healthResponse != null && healthResponse.isTimedOut()) {
      throw new UnableToStartException("cluster state is " + healthResponse.getStatus().name()
          + " and not " + statusThreshold.name()
          + ", from here on, everything will fail!");
    }
  } catch (ElasticsearchTimeoutException e) {
    throw new UnableToStartException(
        "timeout, cluster does not respond to health request, cowardly refusing to continue with operations");
  }
}
 

public void waitForCluster(String statusString, TimeValue timeout) throws IOException {
    if (client() == null) {
        return;
    }
    try {
        ClusterHealthStatus status = ClusterHealthStatus.fromString(statusString);
        ClusterHealthResponse healthResponse =
                client().execute(ClusterHealthAction.INSTANCE, new ClusterHealthRequest().waitForStatus(status).timeout(timeout)).actionGet();
        if (healthResponse != null && healthResponse.isTimedOut()) {
            throw new IOException("cluster state is " + healthResponse.getStatus().name()
                    + " and not " + status.name()
                    + ", from here on, everything will fail!");
        }
    } catch (ElasticsearchTimeoutException e) {
        throw new IOException("timeout, cluster does not respond to health request, cowardly refusing to continue with operations");
    }
}
 

@Before
public void startNodes() {
    try {
        logger.info("starting");
        setClusterName();
        startNode("1");
        findNodeAddress();
        try {
            ClusterHealthResponse healthResponse = client("1").execute(ClusterHealthAction.INSTANCE,
                            new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN).timeout(TimeValue.timeValueSeconds(30))).actionGet();
            if (healthResponse != null && healthResponse.isTimedOut()) {
                throw new IOException("cluster state is " + healthResponse.getStatus().name()
                        + ", from here on, everything will fail!");
            }
        } catch (ElasticsearchTimeoutException e) {
            throw new IOException("timeout, cluster does not respond to health request, cowardly refusing to continue with operations");
        }
    } catch (Throwable t) {
        logger.error("startNodes failed", t);
    }
}
 

@Override
protected void doRun() throws Exception {
    RecoveryStatus status = onGoingRecoveries.get(recoveryId);
    if (status == null) {
        logger.trace("[monitor] no status found for [{}], shutting down", recoveryId);
        return;
    }
    long accessTime = status.lastAccessTime();
    if (accessTime == lastSeenAccessTime) {
        String message = "no activity after [" + checkInterval + "]";
        failRecovery(recoveryId,
                new RecoveryFailedException(status.state(), message, new ElasticsearchTimeoutException(message)),
                true // to be safe, we don't know what go stuck
        );
        return;
    }
    lastSeenAccessTime = accessTime;
    logger.trace("[monitor] rescheduling check for [{}]. last access time is [{}]", recoveryId, lastSeenAccessTime);
    threadPool.schedule(checkInterval, ThreadPool.Names.GENERIC, this);
}
 

@Before
public void startNodes() {
    try {
        logger.info("starting");
        setClusterName();
        startNode("1");
        findNodeAddress();
        try {
            ClusterHealthResponse healthResponse = client("1").execute(ClusterHealthAction.INSTANCE,
                            new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN).timeout(TimeValue.timeValueSeconds(30))).actionGet();
            if (healthResponse != null && healthResponse.isTimedOut()) {
                throw new IOException("cluster state is " + healthResponse.getStatus().name()
                        + ", from here on, everything will fail!");
            }
        } catch (ElasticsearchTimeoutException e) {
            throw new IOException("timeout, cluster does not respond to health request, cowardly refusing to continue with operations");
        }
    } catch (Throwable t) {
        logger.error("startNodes failed", t);
    }
}
 

public void testTimesOut() throws InterruptedException {
    final CountDownLatch countDownLatch = new CountDownLatch(1);
    final SetOnce<TransportException> exceptionHolder = new SetOnce<>();

    transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME,
        new AddVotingConfigExclusionsRequest(new String[]{"other1"}, TimeValue.timeValueMillis(100)),
        expectError(e -> {
            exceptionHolder.set(e);
            countDownLatch.countDown();
        })
    );

    assertTrue(countDownLatch.await(30, TimeUnit.SECONDS));
    final Throwable rootCause = exceptionHolder.get().getRootCause();
    assertThat(rootCause,instanceOf(ElasticsearchTimeoutException.class));
    assertThat(rootCause.getMessage(), startsWith("timed out waiting for voting config exclusions [{other1}"));
}
 

public void testTimesOutIfWaitingForNodesThatAreNotRemoved() throws InterruptedException {
    final CountDownLatch countDownLatch = new CountDownLatch(1);
    final SetOnce<TransportException> responseHolder = new SetOnce<>();

    final ClearVotingConfigExclusionsRequest clearVotingConfigExclusionsRequest = new ClearVotingConfigExclusionsRequest();
    clearVotingConfigExclusionsRequest.setTimeout(TimeValue.timeValueMillis(100));
    transportService.sendRequest(localNode, ClearVotingConfigExclusionsAction.NAME,
        clearVotingConfigExclusionsRequest,
        expectError(e -> {
            responseHolder.set(e);
            countDownLatch.countDown();
        })
    );

    assertTrue(countDownLatch.await(30, TimeUnit.SECONDS));
    assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(),
            containsInAnyOrder(otherNode1Exclusion, otherNode2Exclusion));
    final Throwable rootCause = responseHolder.get().getRootCause();
    assertThat(rootCause, instanceOf(ElasticsearchTimeoutException.class));
    assertThat(rootCause.getMessage(),
        startsWith("timed out waiting for removal of nodes; if nodes should not be removed, set waitForRemoval to false. ["));
}
 

public void testColdStartTimeoutPutCheckpoint() throws Exception {
    when(featureQuery.getColdStartData(any(AnomalyDetector.class))).thenReturn(Optional.of(new double[][] { { 1.0 } }));
    doThrow(new ElasticsearchTimeoutException(""))
        .when(normalModelManager)
        .trainModel(any(AnomalyDetector.class), any(double[][].class));

    AnomalyResultTransportAction action = new AnomalyResultTransportAction(
        new ActionFilters(Collections.emptySet()),
        transportService,
        settings,
        stateManager,
        runner,
        featureQuery,
        normalModelManager,
        hashRing,
        clusterService,
        indexNameResolver,
        adCircuitBreakerService,
        adStats
    );

    AnomalyResultTransportAction.ColdStartJob job = action.new ColdStartJob(detector);
    expectThrows(InternalFailure.class, () -> job.call());
}
 

private void findException(Throwable cause, String adID, AtomicReference<AnomalyDetectionException> failure) {
    if (cause instanceof Error) {
        // we cannot do anything with Error.
        LOG.error(new ParameterizedMessage("Error during prediction for {}: ", adID), cause);
        return;
    }

    Exception causeException = (Exception) cause;
    if (isException(causeException, ResourceNotFoundException.class, RESOURCE_NOT_FOUND_EXCEPTION_NAME_UNDERSCORE)
        || (causeException instanceof IndexNotFoundException
            && causeException.getMessage().contains(CommonName.CHECKPOINT_INDEX_NAME))) {
        failure.set(new ResourceNotFoundException(adID, causeException.getMessage()));
    } else if (isException(causeException, LimitExceededException.class, LIMIT_EXCEEDED_EXCEPTION_NAME_UNDERSCORE)) {
        failure.set(new LimitExceededException(adID, causeException.getMessage()));
    } else if (causeException instanceof ElasticsearchTimeoutException) {
        // we can have ElasticsearchTimeoutException when a node tries to load RCF or
        // threshold model
        failure.set(new InternalFailure(adID, causeException));
    } else {
        // some unexpected bugs occur while predicting anomaly
        failure.set(new EndRunException(adID, "We might have bugs.", causeException, false));
    }
}
 

@Override
public Boolean call() {
    try {
        Optional<double[][]> traingData = featureManager.getColdStartData(detector);
        if (traingData.isPresent()) {
            modelManager.trainModel(detector, traingData.get());
            return true;
        } else {
            throw new EndRunException(detector.getDetectorId(), "Cannot get training data", false);
        }

    } catch (ElasticsearchTimeoutException timeoutEx) {
        throw new InternalFailure(
            detector.getDetectorId(),
            "Time out while indexing cold start checkpoint or get training data",
            timeoutEx
        );
    } catch (EndRunException endRunEx) {
        throw endRunEx;
    } catch (Exception ex) {
        throw new EndRunException(detector.getDetectorId(), "Error while cold start", ex, false);
    }
}
 

/**
 * Blocks the calling thread, waiting for the task to vanish from the TaskManager.
 */
public void waitForTaskCompletion(Task task, long untilInNanos) {
    while (System.nanoTime() - untilInNanos < 0) {
        if (getTask(task.getId()) == null) {
            return;
        }
        try {
            Thread.sleep(WAIT_FOR_COMPLETION_POLL.millis());
        } catch (InterruptedException e) {
            throw new ElasticsearchException("Interrupted waiting for completion of [{}]", e, task);
        }
    }
    throw new ElasticsearchTimeoutException("Timed out waiting for completion of [{}]", task);
}
 

/**
 * @return an array with the rowCounts
 */
private long[] executeBulk(String stmt, Object[][] bulkArgs, TimeValue timeout) {
    try {
        AdapterActionFuture<long[], long[]> actionFuture = new PlainActionFuture<>();
        execute(stmt, bulkArgs, actionFuture);
        return actionFuture.actionGet(timeout);
    } catch (ElasticsearchTimeoutException e) {
        LOGGER.error("Timeout on SQL statement: {}", e, stmt);
        throw e;
    }
}
 

public void blockingUpdateOnMaster(Index index, Mapping mappingUpdate) {
    TimeValue timeout = this.dynamicMappingUpdateTimeout;
    var response = schemaUpdateAction.execute(new SchemaUpdateRequest(index, mappingUpdate.toString())).actionGet();
    if (!response.isAcknowledged()) {
        throw new ElasticsearchTimeoutException("Failed to acknowledge mapping update within [" + timeout + "]");
    }
}
 

protected void waitForCluster(final ClusterHealthStatus status, final TimeValue timeout, final Client client) throws IOException {
    try {
        logger.debug("waiting for cluster state {}", status.name());
        final ClusterHealthResponse healthResponse = client.admin().cluster().prepareHealth().setWaitForStatus(status)
                .setTimeout(timeout).execute().actionGet();
        if (healthResponse.isTimedOut()) {
            throw new IOException("cluster state is " + healthResponse.getStatus().name() + " and not " + status.name()
                    + ", cowardly refusing to continue with operations");
        } else {
            logger.debug("... cluster state ok");
        }
    } catch (final ElasticsearchTimeoutException e) {
        throw new IOException("timeout, cluster does not respond to health request, cowardly refusing to continue with operations");
    }
}
 

private void waitForCluster(final ClusterHealthStatus status, final TimeValue timeout) throws IOException {
    try {
        logger.debug("waiting for cluster state {}", status.name());
        final ClusterHealthResponse healthResponse = client.admin().cluster().prepareHealth().setWaitForStatus(status)
                .setTimeout(timeout).execute().actionGet();
        if (healthResponse.isTimedOut()) {
            throw new IOException("cluster state is " + healthResponse.getStatus().name() + " and not " + status.name()
                    + ", cowardly refusing to continue with operations");
        } else {
            logger.debug("... cluster state ok");
        }
    } catch (final ElasticsearchTimeoutException e) {
        throw new IOException("timeout, cluster does not respond to health request, cowardly refusing to continue with operations");
    }
}
 

@Override
protected void processTasks(ListTasksRequest request, final Consumer<Task> operation) {
    if (false == request.getWaitForCompletion()) {
        super.processTasks(request, operation);
        return;
    }
    // If we should wait for completion then we have to intercept every found task and wait for it to leave the manager.
    TimeValue timeout = request.getTimeout();
    if (timeout == null) {
        timeout = DEFAULT_WAIT_FOR_COMPLETION_TIMEOUT;
    }
    final long timeoutTime = System.nanoTime() + timeout.nanos();
    super.processTasks(request, new Consumer<Task>() {
        @Override
        public void accept(Task t) {
            operation.accept(t);
            while (System.nanoTime() - timeoutTime < 0) {
                Task task = taskManager.getTask(t.getId());
                if (task == null) {
                    return;
                }
                if (task.getAction().startsWith(ListTasksAction.NAME)) {
                    // It doesn't make sense to wait for List Tasks and it can cause an infinite loop of the task waiting
                    // for itself of one of its child tasks
                    return;
                }
                try {
                    Thread.sleep(WAIT_FOR_COMPLETION_POLL.millis());
                } catch (InterruptedException e) {
                    throw new ElasticsearchException("Interrupted waiting for completion of [{}]", e, t);
                }
            }
            throw new ElasticsearchTimeoutException("Timed out waiting for completion of [{}]", t);
        }
    });
}
 

/**
 * Send a nonblocking request with a timeout and return response. Blocking is not allowed in a
 * transport call context. See BaseFuture.blockingAllowed
 * @param request request like index/search/get
 * @param LOG log
 * @param consumer functional interface to operate as a client request like client::get
 * @param <Request> ActionRequest
 * @param <Response> ActionResponse
 * @return the response
 * @throws ElasticsearchTimeoutException when we cannot get response within time.
 * @throws IllegalStateException when the waiting thread is interrupted
 */
public <Request extends ActionRequest, Response extends ActionResponse> Optional<Response> timedRequest(
    Request request,
    Logger LOG,
    BiConsumer<Request, ActionListener<Response>> consumer
) {
    try {
        AtomicReference<Response> respReference = new AtomicReference<>();
        final CountDownLatch latch = new CountDownLatch(1);

        consumer
            .accept(
                request,
                new LatchedActionListener<Response>(
                    ActionListener
                        .wrap(
                            response -> { respReference.set(response); },
                            exception -> { LOG.error("Cannot get response for request {}, error: {}", request, exception); }
                        ),
                    latch
                )
            );

        if (!latch.await(requestTimeout.getSeconds(), TimeUnit.SECONDS)) {
            throw new ElasticsearchTimeoutException("Cannot get response within time limit: " + request.toString());
        }
        return Optional.ofNullable(respReference.get());
    } catch (InterruptedException e1) {
        LOG.error(CommonErrorMessages.WAIT_ERR_MSG);
        throw new IllegalStateException(e1);
    }
}
 

public void joinClusterAndWaitForInitialState() {
    try {
        discovery.startInitialJoin();
        if (!initialStateListener.waitForInitialState(initialStateTimeout)) {
            logger.warn("waited for {} and no initial state was set by the discovery", initialStateTimeout);
        }
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        throw new ElasticsearchTimeoutException("Interrupted while waiting for initial discovery state");
    }
}
 

public boolean download(URL source, Path dest, @Nullable DownloadProgress progress, TimeValue timeout) throws Exception {
    if (Files.exists(dest) && skipExisting) {
        return true;
    }

    //don't do any progress, unless asked
    if (progress == null) {
        progress = new NullProgress();
    }

    //set the timestamp to the file date.
    long timestamp = 0;

    boolean hasTimestamp = false;
    if (useTimestamp && Files.exists(dest) ) {
        timestamp = Files.getLastModifiedTime(dest).toMillis();
        hasTimestamp = true;
    }

    GetThread getThread = new GetThread(source, dest, hasTimestamp, timestamp, progress);

    try {
        getThread.setDaemon(true);
        getThread.start();
        getThread.join(timeout.millis());

        if (getThread.isAlive()) {
            throw new ElasticsearchTimeoutException("The GET operation took longer than " + timeout + ", stopping it.");
        }
    }
    catch (InterruptedException ie) {
        return false;
    } finally {
        getThread.closeStreams();
    }

    return getThread.wasSuccessful();
}
 

protected void waitForCluster(final ClusterHealthStatus status, final TimeValue timeout) throws IOException {
    try {
        log.debug("waiting for cluster state {}", status.name());
        final ClusterHealthResponse healthResponse = client.admin().cluster().prepareHealth().setWaitForStatus(status)
                .setWaitForNodes(">2").setTimeout(timeout).execute().actionGet();
        if (healthResponse.isTimedOut()) {
            throw new IOException("cluster state is " + healthResponse.getStatus().name() + " and not " + status.name()
                    + ", cowardly refusing to continue with operations");
        } else {
            log.debug("... cluster state ok");
        }
    } catch (final ElasticsearchTimeoutException e) {
        throw new IOException("timeout, cluster does not respond to health request, cowardly refusing to continue with operations");
    }
}
 

@Test
public void testPutElasticsearch5OnTriggerWithExceptions() throws IOException {
    PutElasticsearch5TestProcessor processor = new PutElasticsearch5TestProcessor(false);
    runner = TestRunners.newTestRunner(processor);
    runner.setProperty(AbstractElasticsearch5TransportClientProcessor.CLUSTER_NAME, "elasticsearch");
    runner.setProperty(AbstractElasticsearch5TransportClientProcessor.HOSTS, "127.0.0.1:9300");
    runner.setProperty(AbstractElasticsearch5TransportClientProcessor.PING_TIMEOUT, "5s");
    runner.setProperty(AbstractElasticsearch5TransportClientProcessor.SAMPLER_INTERVAL, "5s");
    runner.setProperty(PutElasticsearch5.INDEX, "doc");
    runner.setProperty(PutElasticsearch5.TYPE, "status");
    runner.setValidateExpressionUsage(true);
    runner.setProperty(PutElasticsearch5.ID_ATTRIBUTE, "doc_id");

    // No Node Available exception
    processor.setExceptionToThrow(new NoNodeAvailableException("test"));
    runner.enqueue(docExample, new HashMap<String, String>() {{
        put("doc_id", "28039652140");
    }});
    runner.run(1, true, true);

    runner.assertAllFlowFilesTransferred(FetchElasticsearch5.REL_RETRY, 1);
    runner.clearTransferState();

    // Elasticsearch5 Timeout exception
    processor.setExceptionToThrow(new ElasticsearchTimeoutException("test"));
    runner.enqueue(docExample, new HashMap<String, String>() {{
        put("doc_id", "28039652141");
    }});
    runner.run(1, true, true);

    runner.assertAllFlowFilesTransferred(FetchElasticsearch5.REL_RETRY, 1);
    runner.clearTransferState();

    // Receive Timeout Transport exception
    processor.setExceptionToThrow(new ReceiveTimeoutTransportException(mock(StreamInput.class)));
    runner.enqueue(docExample, new HashMap<String, String>() {{
        put("doc_id", "28039652142");
    }});
    runner.run(1, true, true);

    runner.assertAllFlowFilesTransferred(FetchElasticsearch5.REL_RETRY, 1);
    runner.clearTransferState();

    // Node Closed exception
    processor.setExceptionToThrow(new NodeClosedException(mock(StreamInput.class)));
    runner.enqueue(docExample, new HashMap<String, String>() {{
        put("doc_id", "28039652143");
    }});
    runner.run(1, true, true);

    runner.assertAllFlowFilesTransferred(FetchElasticsearch5.REL_RETRY, 1);
    runner.clearTransferState();

    // Elasticsearch5 Parse exception
    processor.setExceptionToThrow(new ElasticsearchParseException("test"));
    runner.enqueue(docExample, new HashMap<String, String>() {{
        put("doc_id", "28039652144");
    }});
    runner.run(1, true, true);

    // This test generates an exception on execute(),routes to failure
    runner.assertTransferCount(PutElasticsearch5.REL_FAILURE, 1);
}
 

private SQLResponse executeTransportOrJdbc(TestExecutionConfig config,
                                           String stmt,
                                           @Nullable Object[] args,
                                           TimeValue timeout) {
    String pgUrl = clientProvider.pgUrl();
    Random random = RandomizedContext.current().getRandom();

    List<String> sessionList = new ArrayList<>();
    sessionList.add("set search_path to "
                    + StreamSupport.stream(searchPath.spliterator(), false)
                        // explicitly setting the pg catalog schema will make it the current schema so attempts to
                        // create un-fully-qualified relations will fail. we filter it out and will implicitly
                        // remain the first in the search path.
                        .filter(s -> !s.equals(PgCatalogSchemaInfo.NAME))
                        .collect(Collectors.joining(", "))
    );

    if (!config.isHashJoinEnabled()) {
        sessionList.add("set enable_hashjoin=false");
        LOGGER.trace("Executing with enable_hashjoin=false: {}", stmt);
    }

    if (pgUrl != null && config.isJdbcEnabled()) {
        LOGGER.trace("Executing with pgJDBC: {}", stmt);
        return executeWithPg(
            stmt,
            args,
            pgUrl,
            random,
            sessionList);
    }
    try {
        if (!sessionList.isEmpty()) {
            try (Session session = newSession()) {
                sessionList.forEach((setting) -> exec(setting, session));
                return execute(stmt, args, session).actionGet(timeout);
            }
        }
        return execute(stmt, args).actionGet(timeout);
    } catch (ElasticsearchTimeoutException e) {
        LOGGER.error("Timeout on SQL statement: {} {}", stmt, e);
        throw e;
    }
}
 

@Test
public void testPrimaryTermIsIncreasedOnReplicaPromotion() throws Throwable {
    logger.info("starting 3 nodes");
    String masterNodeName = internalCluster().startMasterOnlyNode(DEFAULT_SETTINGS);
    String firstDataNodeName = internalCluster().startDataOnlyNode(DEFAULT_SETTINGS);
    String secondDataNodeName = internalCluster().startDataOnlyNode(DEFAULT_SETTINGS);

    logger.info("wait for all nodes to join the cluster");
    ensureGreen();

    execute("create table registers (id int primary key, value string) CLUSTERED INTO 1 shards " +
            "with (number_of_replicas = 1, \"unassigned.node_left.delayed_timeout\" = '5s')");
    execute("insert into registers values (1, 'initial value')");
    refresh();

    // not setting this when creating the table because we want the replica to be initialized before we start disrupting
    // otherwise the replica might not be ready to be promoted to primary due to "primary failed while replica initializing"
    execute("alter table registers set (\"write.wait_for_active_shards\" = 1)");

    ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
    String firstDataNodeId = clusterState.getNodes().resolveNode(firstDataNodeName).getId();
    String secondDataNodeId = clusterState.getNodes().resolveNode(secondDataNodeName).getId();

    boolean firstDataNodeHasPrimary = clusterState.getRoutingNodes().node(firstDataNodeId).copyShards().get(0).primary();

    // isolate node with the primary
    String isolatedNode = firstDataNodeHasPrimary ? firstDataNodeName : secondDataNodeName;
    Set<String> otherNodes = firstDataNodeHasPrimary ? Set.of(masterNodeName, secondDataNodeName) :
        Set.of(masterNodeName, firstDataNodeName);

    NetworkDisruption partition = new NetworkDisruption(
        new NetworkDisruption.TwoPartitions(otherNodes, Set.of(isolatedNode)),
        new NetworkDisruption.NetworkDisconnect()
    );
    internalCluster().setDisruptionScheme(partition);

    logger.info("start disrupting network");
    partition.startDisrupting();

    String schema = sqlExecutor.getCurrentSchema();
    try {
        execute("update registers set value = 'value whilst disrupted' where id = 1", null, isolatedNode);
        fail("expected timeout exception");
    } catch (ElasticsearchTimeoutException elasticsearchTimeoutException) {
    }

    String nonIsolatedDataNodeId = firstDataNodeHasPrimary ? secondDataNodeId : firstDataNodeId;
    logger.info("wait for replica on the partition with the master to be promoted to primary");
    assertBusy(() -> {
        String index = IndexParts.toIndexName(schema, "registers", null);
        ShardRouting primaryShard = client(masterNodeName).admin().cluster().prepareState().get().getState().getRoutingTable()
            .index(index).shard(0).primaryShard();
        // the node that's part of the same partition as master is now the primary for the table shard
        assertThat(primaryShard.currentNodeId(), equalTo(nonIsolatedDataNodeId));
        assertTrue(primaryShard.active());
    }, 30, TimeUnit.SECONDS);

    execute("update registers set value = 'value set on master' where id = 1", null, masterNodeName);
    execute("update registers set value = 'value set on master the second time' where id = 1",
            null,
            masterNodeName);

    logger.info("heal disruption");
    partition.stopDisrupting();

    logger.info("wait for cluster to get into a green state");
    ensureGreen();

    execute("select value, _seq_no, _primary_term from registers where id = 1", null, masterNodeName);
    String finalValue = (String) response.rows()[0][0];
    long finalSequenceNumber = (long) response.rows()[0][1];
    long finalPrimaryTerm = (long) response.rows()[0][2];

    assertThat("We executed 2 updates on the new primary", finalSequenceNumber, is(2L));
    assertThat("Primary promotion should've triggered a bump in primary term", finalPrimaryTerm, equalTo(2L));
    assertThat(finalValue, equalTo("value set on master the second time"));
}
 

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {

    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }

    final String index = context.getProperty(INDEX).evaluateAttributeExpressions(flowFile).getValue();
    final String docId = context.getProperty(DOC_ID).evaluateAttributeExpressions(flowFile).getValue();
    final String docType = context.getProperty(TYPE).evaluateAttributeExpressions(flowFile).getValue();
    final Charset charset = Charset.forName(context.getProperty(CHARSET).evaluateAttributeExpressions(flowFile).getValue());

    final ComponentLog logger = getLogger();
    try {

        logger.debug("Fetching {}/{}/{} from Elasticsearch", new Object[]{index, docType, docId});
        final long startNanos = System.nanoTime();

        GetRequestBuilder getRequestBuilder = esClient.get().prepareGet(index, docType, docId);
        if (authToken != null) {
            getRequestBuilder.putHeader("Authorization", authToken);
        }
        final GetResponse getResponse = getRequestBuilder.execute().actionGet();

        if (getResponse == null || !getResponse.isExists()) {
            logger.warn("Failed to read {}/{}/{} from Elasticsearch: Document not found",
                    new Object[]{index, docType, docId});

            // We couldn't find the document, so penalize it and send it to "not found"
            flowFile = session.penalize(flowFile);
            session.transfer(flowFile, REL_NOT_FOUND);
        } else {
            flowFile = session.putAttribute(flowFile, "filename", docId);
            flowFile = session.putAttribute(flowFile, "es.index", index);
            flowFile = session.putAttribute(flowFile, "es.type", docType);
            flowFile = session.write(flowFile, new OutputStreamCallback() {
                @Override
                public void process(OutputStream out) throws IOException {
                    out.write(getResponse.getSourceAsString().getBytes(charset));
                }
            });
            logger.debug("Elasticsearch document " + docId + " fetched, routing to success");

            final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
            final String uri = context.getProperty(HOSTS).evaluateAttributeExpressions().getValue() + "/" + index + "/" + docType + "/" + docId;
            session.getProvenanceReporter().fetch(flowFile, uri, millis);
            session.transfer(flowFile, REL_SUCCESS);
        }
    } catch (NoNodeAvailableException
            | ElasticsearchTimeoutException
            | ReceiveTimeoutTransportException
            | NodeClosedException exceptionToRetry) {
        logger.error("Failed to read into Elasticsearch due to {}, this may indicate an error in configuration "
                        + "(hosts, username/password, etc.). Routing to retry",
                new Object[]{exceptionToRetry.getLocalizedMessage()}, exceptionToRetry);
        session.transfer(flowFile, REL_RETRY);
        context.yield();

    } catch (Exception e) {
        logger.error("Failed to read {} from Elasticsearch due to {}", new Object[]{flowFile, e.getLocalizedMessage()}, e);
        session.transfer(flowFile, REL_FAILURE);
        context.yield();
    }
}
 

@Test
public void testPutElasticsearchOnTriggerWithExceptions() throws IOException {
    PutElasticsearchTestProcessor processor = new PutElasticsearchTestProcessor(false);
    runner = TestRunners.newTestRunner(processor);
    runner.setProperty(AbstractElasticsearchTransportClientProcessor.CLUSTER_NAME, "elasticsearch");
    runner.setProperty(AbstractElasticsearchTransportClientProcessor.HOSTS, "127.0.0.1:9300");
    runner.setProperty(AbstractElasticsearchTransportClientProcessor.PING_TIMEOUT, "5s");
    runner.setProperty(AbstractElasticsearchTransportClientProcessor.SAMPLER_INTERVAL, "5s");
    runner.setProperty(PutElasticsearch.INDEX, "doc");
    runner.setProperty(PutElasticsearch.TYPE, "status");
    runner.setValidateExpressionUsage(true);
    runner.setProperty(PutElasticsearch.ID_ATTRIBUTE, "doc_id");

    // No Node Available exception
    processor.setExceptionToThrow(new NoNodeAvailableException("test"));
    runner.enqueue(docExample, new HashMap<String, String>() {{
        put("doc_id", "28039652140");
    }});
    runner.run(1, true, true);

    runner.assertAllFlowFilesTransferred(FetchElasticsearch.REL_RETRY, 1);
    runner.clearTransferState();

    // Elasticsearch Timeout exception
    processor.setExceptionToThrow(new ElasticsearchTimeoutException("test"));
    runner.enqueue(docExample, new HashMap<String, String>() {{
        put("doc_id", "28039652141");
    }});
    runner.run(1, true, true);

    runner.assertAllFlowFilesTransferred(FetchElasticsearch.REL_RETRY, 1);
    runner.clearTransferState();

    // Receive Timeout Transport exception
    processor.setExceptionToThrow(new ReceiveTimeoutTransportException(mock(StreamInput.class)));
    runner.enqueue(docExample, new HashMap<String, String>() {{
        put("doc_id", "28039652142");
    }});
    runner.run(1, true, true);

    runner.assertAllFlowFilesTransferred(FetchElasticsearch.REL_RETRY, 1);
    runner.clearTransferState();

    // Node Closed exception
    processor.setExceptionToThrow(new NodeClosedException(mock(StreamInput.class)));
    runner.enqueue(docExample, new HashMap<String, String>() {{
        put("doc_id", "28039652143");
    }});
    runner.run(1, true, true);

    runner.assertAllFlowFilesTransferred(FetchElasticsearch.REL_RETRY, 1);
    runner.clearTransferState();

    // Elasticsearch Parse exception
    processor.setExceptionToThrow(new ElasticsearchParseException("test"));
    runner.enqueue(docExample, new HashMap<String, String>() {{
        put("doc_id", "28039652144");
    }});
    runner.run(1, true, true);

    // This test generates an exception on execute(),routes to failure
    runner.assertTransferCount(PutElasticsearch.REL_FAILURE, 1);
}
 

@Test
public void testFetchElasticsearchOnTriggerWithExceptions() throws IOException {
    FetchElasticsearchTestProcessor processor = new FetchElasticsearchTestProcessor(true);
    runner = TestRunners.newTestRunner(processor);
    runner.setProperty(AbstractElasticsearchTransportClientProcessor.CLUSTER_NAME, "elasticsearch");
    runner.setProperty(AbstractElasticsearchTransportClientProcessor.HOSTS, "127.0.0.1:9300");
    runner.setProperty(AbstractElasticsearchTransportClientProcessor.PING_TIMEOUT, "5s");
    runner.setProperty(AbstractElasticsearchTransportClientProcessor.SAMPLER_INTERVAL, "5s");
    runner.setProperty(FetchElasticsearch.INDEX, "doc");
    runner.setProperty(FetchElasticsearch.TYPE, "status");
    runner.setValidateExpressionUsage(true);
    runner.setProperty(FetchElasticsearch.DOC_ID, "${doc_id}");

    // No Node Available exception
    processor.setExceptionToThrow(new NoNodeAvailableException("test"));
    runner.enqueue(docExample, new HashMap<String, String>() {{
        put("doc_id", "28039652140");
    }});
    runner.run(1, true, true);

    runner.assertAllFlowFilesTransferred(FetchElasticsearch.REL_RETRY, 1);
    runner.clearTransferState();

    // Elasticsearch Timeout exception
    processor.setExceptionToThrow(new ElasticsearchTimeoutException("test"));
    runner.enqueue(docExample, new HashMap<String, String>() {{
        put("doc_id", "28039652141");
    }});
    runner.run(1, true, true);

    runner.assertAllFlowFilesTransferred(FetchElasticsearch.REL_RETRY, 1);
    runner.clearTransferState();

    // Receive Timeout Transport exception
    processor.setExceptionToThrow(new ReceiveTimeoutTransportException(mock(StreamInput.class)));
    runner.enqueue(docExample, new HashMap<String, String>() {{
        put("doc_id", "28039652141");
    }});
    runner.run(1, true, true);

    runner.assertAllFlowFilesTransferred(FetchElasticsearch.REL_RETRY, 1);
    runner.clearTransferState();

    // Node Closed exception
    processor.setExceptionToThrow(new NodeClosedException(mock(StreamInput.class)));
    runner.enqueue(docExample, new HashMap<String, String>() {{
        put("doc_id", "28039652141");
    }});
    runner.run(1, true, true);

    runner.assertAllFlowFilesTransferred(FetchElasticsearch.REL_RETRY, 1);
    runner.clearTransferState();

    // Elasticsearch Parse exception
    processor.setExceptionToThrow(new ElasticsearchParseException("test"));
    runner.enqueue(docExample, new HashMap<String, String>() {{
        put("doc_id", "28039652141");
    }});
    runner.run(1, true, true);

    // This test generates an exception on execute(),routes to failure
    runner.assertTransferCount(FetchElasticsearch.REL_FAILURE, 1);
}
 

@Override
protected void masterOperation(ClearVotingConfigExclusionsRequest request, ClusterState initialState,
                               ActionListener<ClearVotingConfigExclusionsResponse> listener) throws Exception {

    final long startTimeMillis = threadPool.relativeTimeInMillis();

    final Predicate<ClusterState> allExclusionsRemoved = newState -> {
        for (VotingConfigExclusion tombstone : initialState.getVotingConfigExclusions()) {
            // NB checking for the existence of any node with this persistent ID, because persistent IDs are how votes are counted.
            if (newState.nodes().nodeExists(tombstone.getNodeId())) {
                return false;
            }
        }
        return true;
    };

    if (request.getWaitForRemoval() && allExclusionsRemoved.test(initialState) == false) {
        final ClusterStateObserver clusterStateObserver = new ClusterStateObserver(
            initialState,
            clusterService,
            request.getTimeout(),
            logger);

        clusterStateObserver.waitForNextChange(new Listener() {
            @Override
            public void onNewClusterState(ClusterState state) {
                submitClearVotingConfigExclusionsTask(request, startTimeMillis, listener);
            }

            @Override
            public void onClusterServiceClose() {
                listener.onFailure(new ElasticsearchException("cluster service closed while waiting for removal of nodes "
                    + initialState.getVotingConfigExclusions()));
            }

            @Override
            public void onTimeout(TimeValue timeout) {
                listener.onFailure(new ElasticsearchTimeoutException(
                    "timed out waiting for removal of nodes; if nodes should not be removed, set waitForRemoval to false. "
                    + initialState.getVotingConfigExclusions()));
            }
        }, allExclusionsRemoved);
    } else {
        submitClearVotingConfigExclusionsTask(request, startTimeMillis, listener);
    }
}
 

/**
 * Update mappings synchronously on the master node, waiting for at most
 * {@code timeout}. When this method returns successfully mappings have
 * been applied to the master node and propagated to data nodes.
 */
public void updateMappingOnMaster(Index index, String type, Mapping mappingUpdate, TimeValue timeout) {
    if (updateMappingRequest(index, type, mappingUpdate, timeout).get().isAcknowledged() == false) {
        throw new ElasticsearchTimeoutException("Failed to acknowledge mapping update within [" + timeout + "]");
    }
}