org.elasticsearch.action.bulk.BulkProcessor Java Examples

The following examples show how to use org.elasticsearch.action.bulk.BulkProcessor. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BulkProcessorConfiguration.java    From ElasticUtils with MIT License 7 votes vote down vote up 
public BulkProcessor build(final RestHighLevelClient client) {

        // Taken from:
        //
        //      * https://www.elastic.co/guide/en/elasticsearch/client/java-rest/master/java-rest-high-document-bulk.html#java-rest-high-document-bulk-processor
        //
        return BulkProcessor.builder((request, bulkListener) -> {
            client.bulkAsync(request, RequestOptions.DEFAULT, bulkListener);
        }, listener)
                .setConcurrentRequests(options.getConcurrentRequests())
                .setBulkActions(options.getBulkActions())
                .setBulkSize(options.getBulkSize())
                .setFlushInterval(options.getFlushInterval())
                .setBackoffPolicy(options.getBackoffPolicy())
                .build();
    }
Example #2
Source File: ElasticsearchSystemProducerTest.java    From samza with Apache License 2.0 6 votes vote down vote up 
@Before
public void setUp() throws Exception {
  metrics = new ElasticsearchSystemProducerMetrics("es", new MetricsRegistryMap());
  producer = new ElasticsearchSystemProducer(SYSTEM_NAME,
                                             BULK_PROCESSOR_FACTORY,
                                             CLIENT,
                                             INDEX_REQUEST_FACTORY,
                                             metrics);

  processorOne = mock(BulkProcessor.class);
  processorTwo = mock(BulkProcessor.class);

  when(BULK_PROCESSOR_FACTORY.getBulkProcessor(eq(CLIENT), any(BulkProcessor.Listener.class)))
      .thenReturn(processorOne);
  producer.register(SOURCE_ONE);

  when(BULK_PROCESSOR_FACTORY.getBulkProcessor(eq(CLIENT), any(BulkProcessor.Listener.class)))
      .thenReturn(processorTwo);
  producer.register(SOURCE_TWO);
}
Example #3
Source File: ElasticSearchClient.java    From skywalking with Apache License 2.0 6 votes vote down vote up 
public BulkProcessor createBulkProcessor(int bulkActions, int flushInterval, int concurrentRequests) {
    BulkProcessor.Listener listener = createBulkListener();

    return BulkProcessor.builder(client::bulkAsync, listener)
                        .setBulkActions(bulkActions)
                        .setFlushInterval(TimeValue.timeValueSeconds(flushInterval))
                        .setConcurrentRequests(concurrentRequests)
                        .setBackoffPolicy(BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(100), 3))
                        .build();
}
Example #4
Source File: Elasticsearch2ApiCallBridge.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public void configureBulkProcessorBackoff(
	BulkProcessor.Builder builder,
	@Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) {

	BackoffPolicy backoffPolicy;
	if (flushBackoffPolicy != null) {
		switch (flushBackoffPolicy.getBackoffType()) {
			case CONSTANT:
				backoffPolicy = BackoffPolicy.constantBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
				break;
			case EXPONENTIAL:
			default:
				backoffPolicy = BackoffPolicy.exponentialBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
		}
	} else {
		backoffPolicy = BackoffPolicy.noBackoff();
	}

	builder.setBackoffPolicy(backoffPolicy);
}
Example #5
Source File: ITElasticSearchClient.java    From skywalking with Apache License 2.0 6 votes vote down vote up 
@Test
public void bulk() throws InterruptedException {
    BulkProcessor bulkProcessor = client.createBulkProcessor(2000, 10, 2);

    Map<String, String> source = new HashMap<>();
    source.put("column1", "value1");
    source.put("column2", "value2");

    for (int i = 0; i < 100; i++) {
        IndexRequest indexRequest = new IndexRequest("bulk_insert_test", "type", String.valueOf(i));
        indexRequest.source(source);
        bulkProcessor.add(indexRequest);
    }

    bulkProcessor.flush();
    bulkProcessor.awaitClose(2, TimeUnit.SECONDS);
}
Example #6
Source File: Elasticsearch7ApiCallBridge.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public void configureBulkProcessorBackoff(
	BulkProcessor.Builder builder,
	@Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) {

	BackoffPolicy backoffPolicy;
	if (flushBackoffPolicy != null) {
		switch (flushBackoffPolicy.getBackoffType()) {
			case CONSTANT:
				backoffPolicy = BackoffPolicy.constantBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
				break;
			case EXPONENTIAL:
			default:
				backoffPolicy = BackoffPolicy.exponentialBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
		}
	} else {
		backoffPolicy = BackoffPolicy.noBackoff();
	}

	builder.setBackoffPolicy(backoffPolicy);
}
Example #7
Source File: Elasticsearch6ApiCallBridge.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public void configureBulkProcessorBackoff(
	BulkProcessor.Builder builder,
	@Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) {

	BackoffPolicy backoffPolicy;
	if (flushBackoffPolicy != null) {
		switch (flushBackoffPolicy.getBackoffType()) {
			case CONSTANT:
				backoffPolicy = BackoffPolicy.constantBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
				break;
			case EXPONENTIAL:
			default:
				backoffPolicy = BackoffPolicy.exponentialBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
		}
	} else {
		backoffPolicy = BackoffPolicy.noBackoff();
	}

	builder.setBackoffPolicy(backoffPolicy);
}
Example #8
Source File: ElasticStatisticsPublisher.java    From product-ei with Apache License 2.0 6 votes vote down vote up 
/**
 * Publishes the array list of simplified jsons to Elasticsearch using the Java High Level REST Client.
 *
 * @param jsonsToSend array list of json strings to be published to Elasticsearch
 * @param client      elasticsearch RestHighLevelClient
 */
public static void publish(List<String> jsonsToSend, RestHighLevelClient client) {

    // Prepare and Send the bulk request
    IndexRequest indexRequest = new IndexRequest("eidata");
    if (jsonsToSend != null) {
        for (String jsonString : jsonsToSend) {
            BulkProcessor.Builder bulkProcessorBuilder = BulkProcessor.builder(
                    (request, bulkListener) ->
                            client.bulkAsync(request, RequestOptions.DEFAULT, bulkListener),
                    new BulkProcessorListener());
            bulkProcessor = bulkProcessorBuilder.build();
            indexRequest.source(jsonString, XContentType.JSON);
            bulkProcessor.add(indexRequest);
            bulkProcessor.close();
        }
    }
}
Example #9
Source File: Elasticsearch5ApiCallBridge.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public void configureBulkProcessorBackoff(
	BulkProcessor.Builder builder,
	@Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) {

	BackoffPolicy backoffPolicy;
	if (flushBackoffPolicy != null) {
		switch (flushBackoffPolicy.getBackoffType()) {
			case CONSTANT:
				backoffPolicy = BackoffPolicy.constantBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
				break;
			case EXPONENTIAL:
			default:
				backoffPolicy = BackoffPolicy.exponentialBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
		}
	} else {
		backoffPolicy = BackoffPolicy.noBackoff();
	}

	builder.setBackoffPolicy(backoffPolicy);
}
Example #10
Source File: BulkProcessorBulider.java    From flume-elasticsearch-sink with Apache License 2.0 6 votes vote down vote up 
private BulkProcessor build(final RestHighLevelClient client) {
    logger.trace("Bulk processor name: [{}]  bulkActions: [{}], bulkSize: [{}], flush interval time: [{}]," +
                    " concurrent Request: [{}], backoffPolicyTimeInterval: [{}], backoffPolicyRetries: [{}] ",
            new Object[]{bulkProcessorName, bulkActions, bulkSize, flushIntervalTime,
                    concurrentRequest, backoffPolicyTimeInterval, backoffPolicyRetries});
    BiConsumer<BulkRequest, ActionListener<BulkResponse>> bulkConsumer =
            (request, bulkListener) -> client
                    .bulkAsync(request, RequestOptions.DEFAULT, bulkListener);
    return BulkProcessor.builder(bulkConsumer, getListener())
            .setBulkActions(bulkActions)
            .setBulkSize(bulkSize)
            .setFlushInterval(flushIntervalTime)
            .setConcurrentRequests(concurrentRequest)
            .setBackoffPolicy(BackoffPolicy.exponentialBackoff(
                    Util.getTimeValue(backoffPolicyTimeInterval,
                            DEFAULT_ES_BACKOFF_POLICY_START_DELAY),
                    backoffPolicyRetries))
            .build();
}
Example #11
Source File: Elasticsearch6ApiCallBridge.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Override
public void configureBulkProcessorBackoff(
	BulkProcessor.Builder builder,
	@Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) {

	BackoffPolicy backoffPolicy;
	if (flushBackoffPolicy != null) {
		switch (flushBackoffPolicy.getBackoffType()) {
			case CONSTANT:
				backoffPolicy = BackoffPolicy.constantBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
				break;
			case EXPONENTIAL:
			default:
				backoffPolicy = BackoffPolicy.exponentialBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
		}
	} else {
		backoffPolicy = BackoffPolicy.noBackoff();
	}

	builder.setBackoffPolicy(backoffPolicy);
}
Example #12
Source File: Elasticsearch5ApiCallBridge.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Override
public void configureBulkProcessorBackoff(
	BulkProcessor.Builder builder,
	@Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) {

	BackoffPolicy backoffPolicy;
	if (flushBackoffPolicy != null) {
		switch (flushBackoffPolicy.getBackoffType()) {
			case CONSTANT:
				backoffPolicy = BackoffPolicy.constantBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
				break;
			case EXPONENTIAL:
			default:
				backoffPolicy = BackoffPolicy.exponentialBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
		}
	} else {
		backoffPolicy = BackoffPolicy.noBackoff();
	}

	builder.setBackoffPolicy(backoffPolicy);
}
Example #13
Source File: BulkProcessorFactory.java    From samza with Apache License 2.0 6 votes vote down vote up 
public BulkProcessor getBulkProcessor(Client client, BulkProcessor.Listener listener) {
  BulkProcessor.Builder builder = BulkProcessor.builder(client, listener);

  // Concurrent requests set to 0 to ensure ordering of documents is maintained in batches.
  // This also means BulkProcessor#flush() is blocking as is also required.
  builder.setConcurrentRequests(0);

  config.getBulkFlushMaxActions().ifPresent(builder::setBulkActions);
  config.getBulkFlushMaxSizeMB().ifPresent(size ->
    builder.setBulkSize(new ByteSizeValue(size, ByteSizeUnit.MB))
  );
  config.getBulkFlushIntervalMS().ifPresent(interval ->
    builder.setFlushInterval(TimeValue.timeValueMillis(interval))
  );

  return builder.build();
}
Example #14
Source File: Elasticsearch2ApiCallBridge.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Override
public void configureBulkProcessorBackoff(
	BulkProcessor.Builder builder,
	@Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) {

	BackoffPolicy backoffPolicy;
	if (flushBackoffPolicy != null) {
		switch (flushBackoffPolicy.getBackoffType()) {
			case CONSTANT:
				backoffPolicy = BackoffPolicy.constantBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
				break;
			case EXPONENTIAL:
			default:
				backoffPolicy = BackoffPolicy.exponentialBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
		}
	} else {
		backoffPolicy = BackoffPolicy.noBackoff();
	}

	builder.setBackoffPolicy(backoffPolicy);
}
Example #15
Source File: ElasticsearchSystemProducerTest.java    From samza with Apache License 2.0 6 votes vote down vote up 
@Test
public void testIgnoreVersionConficts() throws Exception {
  ArgumentCaptor<BulkProcessor.Listener> listenerCaptor =
          ArgumentCaptor.forClass(BulkProcessor.Listener.class);

  when(BULK_PROCESSOR_FACTORY.getBulkProcessor(eq(CLIENT), listenerCaptor.capture()))
          .thenReturn(processorOne);
  producer.register(SOURCE_ONE);

  BulkResponse response = getRespWithFailedDocument(RestStatus.CONFLICT);

  listenerCaptor.getValue().afterBulk(0, null, response);
  assertEquals(1, metrics.conflicts.getCount());

  producer.flush(SOURCE_ONE);
}
Example #16
Source File: BulkRequestExecutor.java    From soundwave with Apache License 2.0 5 votes vote down vote up 
public BulkRequestExecutor(Client client, int bulkActions, long maxSizeMegaBytes,
                           int concurrentRequest,
                           TimeValue flushInterval) {
  processor =
      BulkProcessor.builder(client, this).setBulkActions(bulkActions)
          .setBulkSize(new ByteSizeValue(maxSizeMegaBytes, ByteSizeUnit.MB))
          .setConcurrentRequests(concurrentRequest).setFlushInterval(flushInterval).build();
}
Example #17
Source File: ElasticSearchClientTest.java    From ElasticUtils with MIT License 5 votes vote down vote up 
@Test
public void no_value_inserted_when_not_enough_requests() {

    // Create Mocks:
    Client mockedTransportClient = mock(Client.class);
    BulkProcessor.Listener mockedBulkProcessorListener = mock(BulkProcessor.Listener.class);

    // Configure the BulkProcessor to use:
    BulkProcessorConfiguration configuration = new BulkProcessorConfiguration(new BulkProcessingOptionsBuilder().build(), mockedBulkProcessorListener);

    // And create a fake index builder:
    IndexRequestBuilder indexRequestBuilder = new IndexRequestBuilder(mockedTransportClient, IndexAction.INSTANCE);

    // The mapping to use:
    IElasticSearchMapping localWeatherDataMapper = new LocalWeatherDataMapper();

    // Index to insert to:
    String indexName = "weather_data";

    // Initialize it with the default settings:
    when(mockedTransportClient.settings())
            .thenReturn(Settings.builder().build());

    when(mockedTransportClient.prepareIndex())
            .thenReturn(indexRequestBuilder);

    // Create the Test subject:
    ElasticSearchClient<LocalWeatherData> elasticSearchClient = new ElasticSearchClient<>(mockedTransportClient, indexName, localWeatherDataMapper, configuration);

    // Create more entities, than Bulk insertion threshold:
    Stream<LocalWeatherData> entitiesStream = getData(configuration.getBulkProcessingOptions().getBulkActions() - 1).stream();

    // Index the Data:
    elasticSearchClient.index(entitiesStream);

    // Verify, that the TransportClient bulk insert has been called:
    verify(mockedTransportClient, times(0)).bulk(anyObject(), anyObject());
    verify(mockedBulkProcessorListener, times(0)).beforeBulk(anyLong(), anyObject());
}
Example #18
Source File: ElasticsearchClientRest.java    From c2mon with GNU Lesser General Public License v3.0 5 votes vote down vote up 
@Override
public BulkProcessor getBulkProcessor(BulkProcessor.Listener listener) {
  BiConsumer<BulkRequest, ActionListener<BulkResponse>> bulkConsumer =
      (request, bulkListener) -> client.bulkAsync(request, RequestOptions.DEFAULT, bulkListener);

  return BulkProcessor.builder(bulkConsumer, listener)
      .setBulkActions(properties.getBulkActions())
      .setBulkSize(new ByteSizeValue(properties.getBulkSize(), ByteSizeUnit.MB))
      .setFlushInterval(TimeValue.timeValueSeconds(properties.getBulkFlushInterval()))
      .setConcurrentRequests(properties.getConcurrentRequests())
      .build();
}
Example #19
Source File: SearchIndexServiceImpl.java    From nexus-public with Eclipse Public License 1.0 5 votes vote down vote up 
@Override
public <T> List<Future<Void>> bulkPut(final Repository repository,
                                      final Iterable<T> components,
                                      final Function<T, String> identifierProducer,
                                      final Function<T, String> jsonDocumentProducer)
{
  checkNotNull(repository);
  checkNotNull(components);
  String indexName = repositoryIndexNames.get(repository.getName());
  if (indexName == null) {
    return emptyList();
  }

  final Entry<BulkProcessor, ExecutorService> bulkProcessorToExecutorPair = pickABulkProcessor();
  final BulkProcessor bulkProcessor = bulkProcessorToExecutorPair.getKey();
  final ExecutorService executorService = bulkProcessorToExecutorPair.getValue();
  final List<Future<Void>> futures = new ArrayList<>();

  components.forEach(component -> {
    checkCancellation();
    String identifier = identifierProducer.apply(component);
    String json = jsonDocumentProducer.apply(component);
    if (json != null) {
      updateCount.getAndIncrement();

      log.debug("Bulk adding to index document {} from {}: {}", identifier, repository, json);
      futures.add(executorService.submit(
          new BulkProcessorUpdater<>(bulkProcessor, createIndexRequest(indexName, identifier, json))));
    }
  });

  if (!periodicFlush) {
    futures.add(executorService.submit(new BulkProcessorFlusher(bulkProcessor)));
  }
  return futures;
}
Example #20
Source File: SearchIndexServiceImpl.java    From nexus-public with Eclipse Public License 1.0 5 votes vote down vote up 
private Entry<BulkProcessor, ExecutorService> pickABulkProcessor() {
  final int numberOfBulkProcessors = bulkProcessorToExecutors.size();
  if (numberOfBulkProcessors > 1) {
    final int index = ThreadLocalRandom.current().nextInt(numberOfBulkProcessors);
    return bulkProcessorToExecutors.get(index);
  }
  return bulkProcessorToExecutors.get(0);
}
Example #21
Source File: AbstractGatherer.java    From elasticsearch-gatherer with Apache License 2.0 5 votes vote down vote up 
public Gatherer start() throws ElasticSearchException {
    //this.logger = Loggers.getLogger(getClass(), settings.globalSettings(), riverName);
    this.bulkActions = settings.getAsInt("bulk_actions", 1000);
    this.bulkSize = settings.getAsBytesSize("bulk_size", new ByteSizeValue(5, ByteSizeUnit.MB));
    this.flushInterval = settings.getAsTime("flush_interval", TimeValue.timeValueSeconds(5));
    this.concurrentRequests = settings.getAsInt("concurrent_requests", 4);
    bulkProcessor = BulkProcessor.builder(client, new BulkListener())
            .setBulkActions(bulkActions)
            .setBulkSize(bulkSize)
            .setFlushInterval(flushInterval)
            .setConcurrentRequests(concurrentRequests)
            .build();
    return this;
}
Example #22
Source File: BulkWriterCollector.java    From elasticsearch-inout-plugin with Apache License 2.0 5 votes vote down vote up 
@Override
public void open() throws WriterException {

    bulkListener = new BulkListener();
    bulkProcessor = BulkProcessor.builder(getClient(),
            bulkListener).setBulkActions(1000).setBulkSize(
            new ByteSizeValue(5, ByteSizeUnit.MB)).setFlushInterval(
            TimeValue.timeValueSeconds(5)).setConcurrentRequests(
            1).build();
}
Example #23
Source File: ElasticsearchSystemProducer.java    From samza with Apache License 2.0 5 votes vote down vote up 
@Override
public void stop() {
  for (Map.Entry<String, BulkProcessor> e : sourceBulkProcessor.entrySet()) {
    flush(e.getKey());
    e.getValue().close();
  }

  client.close();
}
Example #24
Source File: ClientFacade.java    From molgenis with GNU Lesser General Public License v3.0 5 votes vote down vote up 
public void processDocumentActions(Stream<DocumentAction> documentActions) {
  LOG.trace("Processing document actions ...");
  BulkProcessor bulkProcessor = bulkProcessorFactory.create(client);
  try {
    documentActions.forEachOrdered(
        documentAction -> {
          DocWriteRequest docWriteRequest = toDocWriteRequest(documentAction);
          bulkProcessor.add(docWriteRequest);
        });
  } finally {
    waitForCompletion(bulkProcessor);
    LOG.debug("Processed document actions.");
  }
}
Example #25
Source File: ElasticSearchReader.java    From garmadon with Apache License 2.0 5 votes vote down vote up 
ElasticSearchReader(GarmadonReader.Builder builderReader,
                    BulkProcessor bulkProcessorMain,
                    String esIndexPrefix,
                    PrometheusHttpConsumerMetrics prometheusHttpConsumerMetrics,
                    ElasticSearchCacheManager elasticSearchCacheManager) {
    this.reader = builderReader
        .intercept(GarmadonMessageFilter.ANY.INSTANCE, this::writeToES)
        .build();

    this.bulkProcessor = bulkProcessorMain;

    this.esIndexPrefix = esIndexPrefix;
    this.prometheusHttpConsumerMetrics = prometheusHttpConsumerMetrics;
    this.elasticSearchCacheManager = elasticSearchCacheManager;
}
Example #26
Source File: ElasticSearchConnection.java    From storm-crawler with Apache License 2.0 5 votes vote down vote up 
public static ElasticSearchConnection getConnection(Map stormConf,
        String boltType, BulkProcessor.Listener listener) {

    String flushIntervalString = ConfUtils.getString(stormConf,
            "es." + boltType + ".flushInterval", "5s");

    TimeValue flushInterval = TimeValue.parseTimeValue(flushIntervalString,
            TimeValue.timeValueSeconds(5), "flushInterval");

    int bulkActions = ConfUtils.getInt(stormConf,
            "es." + boltType + ".bulkActions", 50);

    int concurrentRequests = ConfUtils.getInt(stormConf,
            "es." + boltType + ".concurrentRequests", 1);

    RestHighLevelClient client = getClient(stormConf, boltType);

    boolean sniff = ConfUtils.getBoolean(stormConf,
            "es." + boltType + ".sniff", true);
    Sniffer sniffer = null;
    if (sniff) {
        sniffer = Sniffer.builder(client.getLowLevelClient()).build();
    }

    BulkProcessor bulkProcessor = BulkProcessor
            .builder((request, bulkListener) -> client.bulkAsync(request,
                    RequestOptions.DEFAULT, bulkListener), listener)
            .setFlushInterval(flushInterval).setBulkActions(bulkActions)
            .setConcurrentRequests(concurrentRequests).build();

    return new ElasticSearchConnection(client, bulkProcessor, sniffer);
}
Example #27
Source File: BulkProcessorConfiguration.java    From ElasticUtils with MIT License 5 votes vote down vote up 
public BulkProcessor build(final Client client) {
    return BulkProcessor.builder(client, listener)
            .setConcurrentRequests(options.getConcurrentRequests())
            .setBulkActions(options.getBulkActions())
            .setBulkSize(options.getBulkSize())
            .setFlushInterval(options.getFlushInterval())
            .setBackoffPolicy(options.getBackoffPolicy())
            .build();
}
Example #28
Source File: BulkProcessorFactory.java    From log4j2-elasticsearch with Apache License 2.0 5 votes vote down vote up 
@Override
public BatchEmitter createInstance(int batchSize, int deliveryInterval, ClientObjectFactory clientObjectFactory, FailoverPolicy failoverPolicy) {

    Function<BulkRequest, Boolean> failureHandler = clientObjectFactory.createFailureHandler(failoverPolicy);

    BulkProcessor.Listener listener = new BulkExecutionListener(failureHandler);

    BulkProcessor.Builder builder = BulkProcessor.builder((Client) clientObjectFactory.createClient(), listener)
            .setBulkActions(batchSize)
            .setFlushInterval(TimeValue.timeValueMillis(deliveryInterval));
    return new BulkProcessorDelegate(builder.build());
}
Example #29
Source File: ElasticSearchReader.java    From garmadon with Apache License 2.0 5 votes vote down vote up 
public static void main(String[] args) throws IOException, GarmadonEsException {
    EsReaderConfiguration config = ReaderConfiguration.loadConfig(EsReaderConfiguration.class);

    GarmadonReader.Builder builderReader = setUpKafkaReader(config.getKafka());
    BulkProcessor bulkProcessorMain = setUpBulkProcessor(config.getElasticsearch());

    ElasticSearchReader reader = new ElasticSearchReader(builderReader, bulkProcessorMain,
        config.getElasticsearch().getIndexPrefix(), new PrometheusHttpConsumerMetrics(config.getPrometheus().getPort()),
        new ElasticSearchCacheManager());

    Runtime.getRuntime().addShutdownHook(new Thread(() -> reader.stop().join()));

    reader.startReading().join();
}
Example #30
Source File: BulkProcessorFactory.java    From log4j2-elasticsearch with Apache License 2.0 5 votes vote down vote up 
@Override
public BatchEmitter createInstance(int batchSize, int deliveryInterval, ClientObjectFactory clientObjectFactory, FailoverPolicy failoverPolicy) {

    Function<BulkRequest, Boolean> failureHandler = clientObjectFactory.createFailureHandler(failoverPolicy);

    BulkProcessor.Listener listener = new BulkExecutionListener(failureHandler);

    BulkProcessor.Builder builder = BulkProcessor.builder((Client) clientObjectFactory.createClient(), listener)
            .setBulkActions(batchSize)
            .setFlushInterval(TimeValue.timeValueMillis(deliveryInterval));
    return new BulkProcessorDelegate(builder.build());
}