org.elasticsearch.action.bulk.BackoffPolicy Java Examples
The following examples show how to use
org.elasticsearch.action.bulk.BackoffPolicy.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: Elasticsearch5ApiCallBridge.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Override public void configureBulkProcessorBackoff( BulkProcessor.Builder builder, @Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) { BackoffPolicy backoffPolicy; if (flushBackoffPolicy != null) { switch (flushBackoffPolicy.getBackoffType()) { case CONSTANT: backoffPolicy = BackoffPolicy.constantBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); break; case EXPONENTIAL: default: backoffPolicy = BackoffPolicy.exponentialBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); } } else { backoffPolicy = BackoffPolicy.noBackoff(); } builder.setBackoffPolicy(backoffPolicy); }
Example #2
Source File: BulkProcessingOptionsBuilderTest.java From ElasticUtils with MIT License | 6 votes |
@Test public void custom_values_set_when_custom_values_set_for_builder() { // Build: BulkProcessingOptions options = new BulkProcessingOptionsBuilder() .setName("Test") .setConcurrentRequests(3) .setBulkActions(10) .setBulkSize(new ByteSizeValue(2, ByteSizeUnit.MB)) .setFlushInterval(new TimeValue(321)) .setBackoffPolicy(BackoffPolicy.noBackoff()) .build(); // Check Values: Assert.assertEquals("Test", options.getName()); Assert.assertEquals(3, options.getConcurrentRequests()); Assert.assertEquals(10, options.getBulkActions()); Assert.assertEquals(new ByteSizeValue(2, ByteSizeUnit.MB).bytes(), options.getBulkSize().bytes()); Assert.assertEquals(new TimeValue(321).getMillis(), options.getFlushInterval().getMillis()); Assert.assertEquals(BackoffPolicy.noBackoff().getClass(), options.getBackoffPolicy().getClass()); }
Example #3
Source File: ElasticSearchClient.java From skywalking with Apache License 2.0 | 6 votes |
public BulkProcessor createBulkProcessor(int bulkActions, int flushInterval, int concurrentRequests) { BulkProcessor.Listener listener = createBulkListener(); return BulkProcessor.builder(client::bulkAsync, listener) .setBulkActions(bulkActions) .setFlushInterval(TimeValue.timeValueSeconds(flushInterval)) .setConcurrentRequests(concurrentRequests) .setBackoffPolicy(BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(100), 3)) .build(); }
Example #4
Source File: Elasticsearch7ApiCallBridge.java From flink with Apache License 2.0 | 6 votes |
@Override public void configureBulkProcessorBackoff( BulkProcessor.Builder builder, @Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) { BackoffPolicy backoffPolicy; if (flushBackoffPolicy != null) { switch (flushBackoffPolicy.getBackoffType()) { case CONSTANT: backoffPolicy = BackoffPolicy.constantBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); break; case EXPONENTIAL: default: backoffPolicy = BackoffPolicy.exponentialBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); } } else { backoffPolicy = BackoffPolicy.noBackoff(); } builder.setBackoffPolicy(backoffPolicy); }
Example #5
Source File: Elasticsearch5ApiCallBridge.java From flink with Apache License 2.0 | 6 votes |
@Override public void configureBulkProcessorBackoff( BulkProcessor.Builder builder, @Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) { BackoffPolicy backoffPolicy; if (flushBackoffPolicy != null) { switch (flushBackoffPolicy.getBackoffType()) { case CONSTANT: backoffPolicy = BackoffPolicy.constantBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); break; case EXPONENTIAL: default: backoffPolicy = BackoffPolicy.exponentialBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); } } else { backoffPolicy = BackoffPolicy.noBackoff(); } builder.setBackoffPolicy(backoffPolicy); }
Example #6
Source File: Elasticsearch6ApiCallBridge.java From flink with Apache License 2.0 | 6 votes |
@Override public void configureBulkProcessorBackoff( BulkProcessor.Builder builder, @Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) { BackoffPolicy backoffPolicy; if (flushBackoffPolicy != null) { switch (flushBackoffPolicy.getBackoffType()) { case CONSTANT: backoffPolicy = BackoffPolicy.constantBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); break; case EXPONENTIAL: default: backoffPolicy = BackoffPolicy.exponentialBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); } } else { backoffPolicy = BackoffPolicy.noBackoff(); } builder.setBackoffPolicy(backoffPolicy); }
Example #7
Source File: TransportDistributedResultAction.java From crate with Apache License 2.0 | 6 votes |
@Inject public TransportDistributedResultAction(Transports transports, TasksService tasksService, ThreadPool threadPool, TransportService transportService, ClusterService clusterService, TransportKillJobsNodeAction killJobsAction, Settings settings) { this(transports, tasksService, threadPool, transportService, clusterService, killJobsAction, BackoffPolicy.exponentialBackoff()); }
Example #8
Source File: TransportDistributedResultAction.java From crate with Apache License 2.0 | 6 votes |
@VisibleForTesting TransportDistributedResultAction(Transports transports, TasksService tasksService, ThreadPool threadPool, TransportService transportService, ClusterService clusterService, TransportKillJobsNodeAction killJobsAction, BackoffPolicy backoffPolicy) { this.transports = transports; this.tasksService = tasksService; scheduler = threadPool.scheduler(); this.clusterService = clusterService; this.killJobsAction = killJobsAction; this.backoffPolicy = backoffPolicy; transportService.registerRequestHandler(DISTRIBUTED_RESULT_ACTION, DistributedResultRequest::new, ThreadPool.Names.SAME, // <- we will dispatch later at the nodeOperation on non failures new NodeActionRequestHandler<>(this)); }
Example #9
Source File: BatchIteratorBackpressureExecutor.java From crate with Apache License 2.0 | 6 votes |
/** * @param batchIterator provides the items for {@code execute} * @param execute async function which is called for each item from the batchIterator * @param combiner used to combine partial results returned by the {@code execute} function * @param identity default value (this is the result, if the batchIterator contains no items) * @param pauseConsumption predicate used to check if the consumption should be paused */ public BatchIteratorBackpressureExecutor(ScheduledExecutorService scheduler, Executor executor, BatchIterator<T> batchIterator, Function<T, CompletableFuture<R>> execute, BinaryOperator<R> combiner, R identity, Predicate<T> pauseConsumption, BackoffPolicy backoffPolicy) { this.executor = executor; this.batchIterator = batchIterator; this.scheduler = scheduler; this.execute = execute; this.combiner = combiner; this.pauseConsumption = pauseConsumption; this.throttleDelay = backoffPolicy.iterator(); this.resultRef = new AtomicReference<>(identity); this.continueConsumptionOrFinish = this::continueConsumptionOrFinish; }
Example #10
Source File: BulkProcessorBulider.java From flume-elasticsearch-sink with Apache License 2.0 | 6 votes |
private BulkProcessor build(final RestHighLevelClient client) { logger.trace("Bulk processor name: [{}] bulkActions: [{}], bulkSize: [{}], flush interval time: [{}]," + " concurrent Request: [{}], backoffPolicyTimeInterval: [{}], backoffPolicyRetries: [{}] ", new Object[]{bulkProcessorName, bulkActions, bulkSize, flushIntervalTime, concurrentRequest, backoffPolicyTimeInterval, backoffPolicyRetries}); BiConsumer<BulkRequest, ActionListener<BulkResponse>> bulkConsumer = (request, bulkListener) -> client .bulkAsync(request, RequestOptions.DEFAULT, bulkListener); return BulkProcessor.builder(bulkConsumer, getListener()) .setBulkActions(bulkActions) .setBulkSize(bulkSize) .setFlushInterval(flushIntervalTime) .setConcurrentRequests(concurrentRequest) .setBackoffPolicy(BackoffPolicy.exponentialBackoff( Util.getTimeValue(backoffPolicyTimeInterval, DEFAULT_ES_BACKOFF_POLICY_START_DELAY), backoffPolicyRetries)) .build(); }
Example #11
Source File: Elasticsearch2ApiCallBridge.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Override public void configureBulkProcessorBackoff( BulkProcessor.Builder builder, @Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) { BackoffPolicy backoffPolicy; if (flushBackoffPolicy != null) { switch (flushBackoffPolicy.getBackoffType()) { case CONSTANT: backoffPolicy = BackoffPolicy.constantBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); break; case EXPONENTIAL: default: backoffPolicy = BackoffPolicy.exponentialBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); } } else { backoffPolicy = BackoffPolicy.noBackoff(); } builder.setBackoffPolicy(backoffPolicy); }
Example #12
Source File: AnomalyResultHandler.java From anomaly-detection with Apache License 2.0 | 6 votes |
public AnomalyResultHandler( Client client, Settings settings, ClusterService clusterService, IndexNameExpressionResolver indexNameExpressionResolver, AnomalyDetectionIndices anomalyDetectionIndices, ThreadPool threadPool ) { this.client = client; this.clusterService = clusterService; this.indexNameExpressionResolver = indexNameExpressionResolver; this.anomalyDetectionIndices = anomalyDetectionIndices; this.threadPool = threadPool; this.resultSavingBackoffPolicy = BackoffPolicy .exponentialBackoff( AnomalyDetectorSettings.BACKOFF_INITIAL_DELAY.get(settings), AnomalyDetectorSettings.MAX_RETRY_FOR_BACKOFF.get(settings) ); }
Example #13
Source File: Elasticsearch6ApiCallBridge.java From flink with Apache License 2.0 | 6 votes |
@Override public void configureBulkProcessorBackoff( BulkProcessor.Builder builder, @Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) { BackoffPolicy backoffPolicy; if (flushBackoffPolicy != null) { switch (flushBackoffPolicy.getBackoffType()) { case CONSTANT: backoffPolicy = BackoffPolicy.constantBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); break; case EXPONENTIAL: default: backoffPolicy = BackoffPolicy.exponentialBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); } } else { backoffPolicy = BackoffPolicy.noBackoff(); } builder.setBackoffPolicy(backoffPolicy); }
Example #14
Source File: Elasticsearch5ApiCallBridge.java From flink with Apache License 2.0 | 6 votes |
@Override public void configureBulkProcessorBackoff( BulkProcessor.Builder builder, @Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) { BackoffPolicy backoffPolicy; if (flushBackoffPolicy != null) { switch (flushBackoffPolicy.getBackoffType()) { case CONSTANT: backoffPolicy = BackoffPolicy.constantBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); break; case EXPONENTIAL: default: backoffPolicy = BackoffPolicy.exponentialBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); } } else { backoffPolicy = BackoffPolicy.noBackoff(); } builder.setBackoffPolicy(backoffPolicy); }
Example #15
Source File: Elasticsearch2ApiCallBridge.java From flink with Apache License 2.0 | 6 votes |
@Override public void configureBulkProcessorBackoff( BulkProcessor.Builder builder, @Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) { BackoffPolicy backoffPolicy; if (flushBackoffPolicy != null) { switch (flushBackoffPolicy.getBackoffType()) { case CONSTANT: backoffPolicy = BackoffPolicy.constantBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); break; case EXPONENTIAL: default: backoffPolicy = BackoffPolicy.exponentialBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); } } else { backoffPolicy = BackoffPolicy.noBackoff(); } builder.setBackoffPolicy(backoffPolicy); }
Example #16
Source File: Elasticsearch6ApiCallBridge.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Override public void configureBulkProcessorBackoff( BulkProcessor.Builder builder, @Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) { BackoffPolicy backoffPolicy; if (flushBackoffPolicy != null) { switch (flushBackoffPolicy.getBackoffType()) { case CONSTANT: backoffPolicy = BackoffPolicy.constantBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); break; case EXPONENTIAL: default: backoffPolicy = BackoffPolicy.exponentialBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); } } else { backoffPolicy = BackoffPolicy.noBackoff(); } builder.setBackoffPolicy(backoffPolicy); }
Example #17
Source File: TransportDistributedResultActionTest.java From crate with Apache License 2.0 | 5 votes |
@Test public void testKillIsInvokedIfContextIsNotFound() throws Exception { TasksService tasksService = new TasksService(clusterService, new JobsLogs(() -> false)); AtomicInteger numBroadcasts = new AtomicInteger(0); TransportKillJobsNodeAction killJobsAction = new TransportKillJobsNodeAction( tasksService, clusterService, mock(TransportService.class) ) { @Override public void broadcast(KillJobsRequest request, ActionListener<Long> listener, Collection<String> excludedNodeIds) { numBroadcasts.incrementAndGet(); } }; TransportDistributedResultAction transportDistributedResultAction = new TransportDistributedResultAction( mock(Transports.class), tasksService, THREAD_POOL, mock(TransportService.class), clusterService, killJobsAction, BackoffPolicy.exponentialBackoff(TimeValue.ZERO, 0) ); StreamBucket.Builder builder = new StreamBucket.Builder( new Streamer[0], RamAccounting.NO_ACCOUNTING); try { transportDistributedResultAction.nodeOperation( new DistributedResultRequest(UUID.randomUUID(), 0, (byte) 0, 0, builder.build(), true) ).get(5, TimeUnit.SECONDS); fail("nodeOperation call should fail with TaskMissing"); } catch (ExecutionException e) { assertThat(e.getCause(), Matchers.instanceOf(TaskMissing.class)); } assertThat(numBroadcasts.get(), is(1)); }
Example #18
Source File: ElasticSearch7Client.java From skywalking with Apache License 2.0 | 5 votes |
public BulkProcessor createBulkProcessor(int bulkActions, int flushInterval, int concurrentRequests) { BulkProcessor.Listener listener = createBulkListener(); return BulkProcessor.builder( (bulkRequest, bulkResponseActionListener) -> client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener ), listener) .setBulkActions(bulkActions) .setFlushInterval(TimeValue.timeValueSeconds(flushInterval)) .setConcurrentRequests(concurrentRequests) .setBackoffPolicy(BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(100), 3)) .build(); }
Example #19
Source File: BulkProcessingOptionsTest.java From ElasticUtils with MIT License | 5 votes |
@Test public void get_methods_return_correct_values_when_initialized() { BulkProcessingOptions options = new BulkProcessingOptions("Name", 1, 2, new ByteSizeValue(10), new TimeValue(10), BackoffPolicy.exponentialBackoff()); Assert.assertEquals("Name", options.getName()); Assert.assertEquals(1, options.getConcurrentRequests()); Assert.assertEquals(2, options.getBulkActions()); Assert.assertEquals(10, options.getBulkSize().bytes()); Assert.assertEquals(10, options.getFlushInterval().getMillis()); Assert.assertEquals(BackoffPolicy.exponentialBackoff().getClass(), options.getBackoffPolicy().getClass()); }
Example #20
Source File: LimitedBackoffPolicyTest.java From crate with Apache License 2.0 | 5 votes |
@Test public void testNoNext() throws Exception { BackoffPolicy policy = new LimitedExponentialBackoff(0, 1, Integer.MAX_VALUE); Iterator<TimeValue> it = policy.iterator(); it.next(); expectedException.expect(NoSuchElementException.class); expectedException.expectMessage("Reached maximum amount of backoff iterations. Only 1 iterations allowed."); it.next(); }
Example #21
Source File: BulkProcessingOptionsBuilderTest.java From ElasticUtils with MIT License | 5 votes |
@Test public void default_values_set_when_initializing_builder() { // Build: BulkProcessingOptions options = new BulkProcessingOptionsBuilder().build(); // Check Values: Assert.assertEquals(null, options.getName()); Assert.assertEquals(1, options.getConcurrentRequests()); Assert.assertEquals(1000, options.getBulkActions()); Assert.assertEquals(new ByteSizeValue(5, ByteSizeUnit.MB).bytes(), options.getBulkSize().bytes()); Assert.assertEquals(null, options.getFlushInterval()); Assert.assertEquals(BackoffPolicy.exponentialBackoff().getClass(), options.getBackoffPolicy().getClass()); }
Example #22
Source File: BulkProcessingOptions.java From ElasticUtils with MIT License | 5 votes |
public BulkProcessingOptions(String name, int concurrentRequests, int bulkActions, ByteSizeValue bulkSize, TimeValue flushInterval, BackoffPolicy backoffPolicy) { this.name = name; this.concurrentRequests = concurrentRequests; this.bulkActions = bulkActions; this.bulkSize = bulkSize; this.flushInterval = flushInterval; this.backoffPolicy = backoffPolicy; }
Example #23
Source File: BulkProcessingOptions.java From ElasticUtils with MIT License | 5 votes |
public BulkProcessingOptions(String name, int concurrentRequests, int bulkActions, ByteSizeValue bulkSize, TimeValue flushInterval, BackoffPolicy backoffPolicy) { this.name = name; this.concurrentRequests = concurrentRequests; this.bulkActions = bulkActions; this.bulkSize = bulkSize; this.flushInterval = flushInterval; this.backoffPolicy = backoffPolicy; }
Example #24
Source File: BulkProcessingOptions.java From ElasticUtils with MIT License | 5 votes |
public BulkProcessingOptions(String name, int concurrentRequests, int bulkActions, ByteSizeValue bulkSize, TimeValue flushInterval, BackoffPolicy backoffPolicy) { this.name = name; this.concurrentRequests = concurrentRequests; this.bulkActions = bulkActions; this.bulkSize = bulkSize; this.flushInterval = flushInterval; this.backoffPolicy = backoffPolicy; }
Example #25
Source File: BulkProcessingOptions.java From ElasticUtils with MIT License | 5 votes |
public BulkProcessingOptions(String name, int concurrentRequests, int bulkActions, ByteSizeValue bulkSize, TimeValue flushInterval, BackoffPolicy backoffPolicy) { this.name = name; this.concurrentRequests = concurrentRequests; this.bulkActions = bulkActions; this.bulkSize = bulkSize; this.flushInterval = flushInterval; this.backoffPolicy = backoffPolicy; }
Example #26
Source File: ESBufferConfiguration.java From SkaETL with Apache License 2.0 | 5 votes |
public BackoffPolicy toBackOffPolicy() { switch (backoffPolicy) { case NO_BACKOFF: return BackoffPolicy.noBackoff(); case CONSTANT_BACKOFF: return BackoffPolicy.constantBackoff(TimeValue.timeValueMillis(delayBackoffTimeUnit.toMillis(delayBackoff)), backoffMaxRetry); case EXPONENTIAL_BACKOFF: return BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(delayBackoffTimeUnit.toMillis(delayBackoff)), backoffMaxRetry); default: throw new IllegalArgumentException(backoffPolicy + " not supported"); } }
Example #27
Source File: BulkProcessingOptions.java From ElasticUtils with MIT License | 4 votes |
public BackoffPolicy getBackoffPolicy() { return backoffPolicy; }
Example #28
Source File: BulkProcessingOptionsBuilder.java From ElasticUtils with MIT License | 4 votes |
public BulkProcessingOptionsBuilder setBackoffPolicy(BackoffPolicy backoffPolicy) { this.backoffPolicy = backoffPolicy; return this; }
Example #29
Source File: BulkProcessingOptions.java From ElasticUtils with MIT License | 4 votes |
public BackoffPolicy getBackoffPolicy() { return backoffPolicy; }
Example #30
Source File: BulkProcessingOptionsBuilder.java From ElasticUtils with MIT License | 4 votes |
public BulkProcessingOptionsBuilder setBackoffPolicy(BackoffPolicy backoffPolicy) { this.backoffPolicy = backoffPolicy; return this; }