Java Code Examples for org.elasticsearch.action.bulk.BulkProcessor#Builder

The following examples show how to use org.elasticsearch.action.bulk.BulkProcessor#Builder . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: Elasticsearch6ApiCallBridge.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public void configureBulkProcessorBackoff(
	BulkProcessor.Builder builder,
	@Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) {

	BackoffPolicy backoffPolicy;
	if (flushBackoffPolicy != null) {
		switch (flushBackoffPolicy.getBackoffType()) {
			case CONSTANT:
				backoffPolicy = BackoffPolicy.constantBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
				break;
			case EXPONENTIAL:
			default:
				backoffPolicy = BackoffPolicy.exponentialBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
		}
	} else {
		backoffPolicy = BackoffPolicy.noBackoff();
	}

	builder.setBackoffPolicy(backoffPolicy);
}
Example 2
Source File: Elasticsearch2ApiCallBridge.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Override
public void configureBulkProcessorBackoff(
	BulkProcessor.Builder builder,
	@Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) {

	BackoffPolicy backoffPolicy;
	if (flushBackoffPolicy != null) {
		switch (flushBackoffPolicy.getBackoffType()) {
			case CONSTANT:
				backoffPolicy = BackoffPolicy.constantBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
				break;
			case EXPONENTIAL:
			default:
				backoffPolicy = BackoffPolicy.exponentialBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
		}
	} else {
		backoffPolicy = BackoffPolicy.noBackoff();
	}

	builder.setBackoffPolicy(backoffPolicy);
}
Example 3
Source File: Elasticsearch5ApiCallBridge.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Override
public void configureBulkProcessorBackoff(
	BulkProcessor.Builder builder,
	@Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) {

	BackoffPolicy backoffPolicy;
	if (flushBackoffPolicy != null) {
		switch (flushBackoffPolicy.getBackoffType()) {
			case CONSTANT:
				backoffPolicy = BackoffPolicy.constantBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
				break;
			case EXPONENTIAL:
			default:
				backoffPolicy = BackoffPolicy.exponentialBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
		}
	} else {
		backoffPolicy = BackoffPolicy.noBackoff();
	}

	builder.setBackoffPolicy(backoffPolicy);
}
Example 4
Source File: Elasticsearch6ApiCallBridge.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Override
public void configureBulkProcessorBackoff(
	BulkProcessor.Builder builder,
	@Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) {

	BackoffPolicy backoffPolicy;
	if (flushBackoffPolicy != null) {
		switch (flushBackoffPolicy.getBackoffType()) {
			case CONSTANT:
				backoffPolicy = BackoffPolicy.constantBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
				break;
			case EXPONENTIAL:
			default:
				backoffPolicy = BackoffPolicy.exponentialBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
		}
	} else {
		backoffPolicy = BackoffPolicy.noBackoff();
	}

	builder.setBackoffPolicy(backoffPolicy);
}
Example 5
Source File: Elasticsearch5ApiCallBridge.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public void configureBulkProcessorBackoff(
	BulkProcessor.Builder builder,
	@Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) {

	BackoffPolicy backoffPolicy;
	if (flushBackoffPolicy != null) {
		switch (flushBackoffPolicy.getBackoffType()) {
			case CONSTANT:
				backoffPolicy = BackoffPolicy.constantBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
				break;
			case EXPONENTIAL:
			default:
				backoffPolicy = BackoffPolicy.exponentialBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
		}
	} else {
		backoffPolicy = BackoffPolicy.noBackoff();
	}

	builder.setBackoffPolicy(backoffPolicy);
}
Example 6
Source File: ElasticsearchSinkBase.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Build the {@link BulkProcessor}.
 *
 * <p>Note: this is exposed for testing purposes.
 */
@VisibleForTesting
protected BulkProcessor buildBulkProcessor(BulkProcessor.Listener listener) {
	checkNotNull(listener);

	BulkProcessor.Builder bulkProcessorBuilder = callBridge.createBulkProcessorBuilder(client, listener);

	// This makes flush() blocking
	bulkProcessorBuilder.setConcurrentRequests(0);

	if (bulkProcessorFlushMaxActions != null) {
		bulkProcessorBuilder.setBulkActions(bulkProcessorFlushMaxActions);
	}

	if (bulkProcessorFlushMaxSizeMb != null) {
		bulkProcessorBuilder.setBulkSize(new ByteSizeValue(bulkProcessorFlushMaxSizeMb, ByteSizeUnit.MB));
	}

	if (bulkProcessorFlushIntervalMillis != null) {
		bulkProcessorBuilder.setFlushInterval(TimeValue.timeValueMillis(bulkProcessorFlushIntervalMillis));
	}

	// if backoff retrying is disabled, bulkProcessorFlushBackoffPolicy will be null
	callBridge.configureBulkProcessorBackoff(bulkProcessorBuilder, bulkProcessorFlushBackoffPolicy);

	return bulkProcessorBuilder.build();
}
Example 7
Source File: BulkProcessorFactory.java    From log4j2-elasticsearch with Apache License 2.0 5 votes vote down vote up 
@Override
public BatchEmitter createInstance(int batchSize, int deliveryInterval, ClientObjectFactory clientObjectFactory, FailoverPolicy failoverPolicy) {

    Function<BulkRequest, Boolean> failureHandler = clientObjectFactory.createFailureHandler(failoverPolicy);

    BulkProcessor.Listener listener = new BulkExecutionListener(failureHandler);

    BulkProcessor.Builder builder = BulkProcessor.builder((Client) clientObjectFactory.createClient(), listener)
            .setBulkActions(batchSize)
            .setFlushInterval(TimeValue.timeValueMillis(deliveryInterval));
    return new BulkProcessorDelegate(builder.build());
}
Example 8
Source File: BulkProcessorFactory.java    From log4j2-elasticsearch with Apache License 2.0 5 votes vote down vote up 
@Override
public BatchEmitter createInstance(int batchSize, int deliveryInterval, ClientObjectFactory clientObjectFactory, FailoverPolicy failoverPolicy) {

    Function<BulkRequest, Boolean> failureHandler = clientObjectFactory.createFailureHandler(failoverPolicy);

    BulkProcessor.Listener listener = new BulkExecutionListener(failureHandler);

    BulkProcessor.Builder builder = BulkProcessor.builder((Client) clientObjectFactory.createClient(), listener)
            .setBulkActions(batchSize)
            .setFlushInterval(TimeValue.timeValueMillis(deliveryInterval));
    return new BulkProcessorDelegate(builder.build());
}
Example 9
Source File: BulkProcessorFactory.java    From log4j2-elasticsearch with Apache License 2.0 5 votes vote down vote up 
@Override
public BatchEmitter createInstance(int batchSize, int deliveryInterval, ClientObjectFactory clientObjectFactory, FailoverPolicy failoverPolicy) {

    Function<BulkRequest, Boolean> failureHandler = clientObjectFactory.createFailureHandler(failoverPolicy);

    BulkProcessor.Listener listener = new BulkExecutionListener(failureHandler);

    BulkProcessor.Builder builder = BulkProcessor.builder((Client) clientObjectFactory.createClient(), listener)
            .setBulkActions(batchSize)
            .setFlushInterval(TimeValue.timeValueMillis(deliveryInterval));
    return new BulkProcessorDelegate(builder.build());
}
Example 10
Source File: Elasticsearch6ApiCallBridge.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public BulkProcessor.Builder createBulkProcessorBuilder(RestHighLevelClient client, BulkProcessor.Listener listener) {
	return BulkProcessor.builder(client::bulkAsync, listener);
}
Example 11
Source File: Elasticsearch6ApiCallBridge.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public BulkProcessor.Builder createBulkProcessorBuilder(RestHighLevelClient client, BulkProcessor.Listener listener) {
	return BulkProcessor.builder(client::bulkAsync, listener);
}
Example 12
Source File: Elasticsearch5ApiCallBridge.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public BulkProcessor.Builder createBulkProcessorBuilder(TransportClient client, BulkProcessor.Listener listener) {
	return BulkProcessor.builder(client, listener);
}
Example 13
Source File: ElasticsearchSinkBaseTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Override
public BulkProcessor.Builder createBulkProcessorBuilder(Client client, BulkProcessor.Listener listener) {
	return null;
}
Example 14
Source File: Elasticsearch1ApiCallBridge.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Override
public BulkProcessor.Builder createBulkProcessorBuilder(Client client, BulkProcessor.Listener listener) {
	return BulkProcessor.builder(client, listener);
}
Example 15
Source File: ElasticsearchSinkBaseTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public BulkProcessor.Builder createBulkProcessorBuilder(Client client, BulkProcessor.Listener listener) {
	return null;
}
Example 16
Source File: Elasticsearch5ApiCallBridge.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Override
public BulkProcessor.Builder createBulkProcessorBuilder(TransportClient client, BulkProcessor.Listener listener) {
	return BulkProcessor.builder(client, listener);
}
Example 17
Source File: ElasticsearchSinkBaseTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public void configureBulkProcessorBackoff(BulkProcessor.Builder builder, @Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) {
	// no need for this in the test cases here
}
Example 18
Source File: Elasticsearch7ApiCallBridge.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public BulkProcessor.Builder createBulkProcessorBuilder(RestHighLevelClient client, BulkProcessor.Listener listener) {
	return BulkProcessor.builder((request, bulkListener) -> client.bulkAsync(request, RequestOptions.DEFAULT, bulkListener), listener);
}
Example 19
Source File: ElasticsearchApiCallBridge.java    From flink with Apache License 2.0 2 votes vote down vote up 
/**
 * Set backoff-related configurations on the provided {@link BulkProcessor.Builder}.
 * The builder will be later on used to instantiate the actual {@link BulkProcessor}.
 *
 * @param builder the {@link BulkProcessor.Builder} to configure.
 * @param flushBackoffPolicy user-provided backoff retry settings ({@code null} if the user disabled backoff retries).
 */
void configureBulkProcessorBackoff(
	BulkProcessor.Builder builder,
	@Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy);
Example 20
Source File: ElasticsearchApiCallBridge.java    From Flink-CEPplus with Apache License 2.0 2 votes vote down vote up 
/**
 * Creates a {@link BulkProcessor.Builder} for creating the bulk processor.
 *
 * @param client the Elasticsearch client.
 * @param listener the bulk processor listender.
 * @return the bulk processor builder.
 */
BulkProcessor.Builder createBulkProcessorBuilder(C client, BulkProcessor.Listener listener);