Java Code Examples for org.elasticsearch.action.bulk.BulkProcessor#Listener

public BulkProcessor getBulkProcessor(Client client, BulkProcessor.Listener listener) {
  BulkProcessor.Builder builder = BulkProcessor.builder(client, listener);

  // Concurrent requests set to 0 to ensure ordering of documents is maintained in batches.
  // This also means BulkProcessor#flush() is blocking as is also required.
  builder.setConcurrentRequests(0);

  config.getBulkFlushMaxActions().ifPresent(builder::setBulkActions);
  config.getBulkFlushMaxSizeMB().ifPresent(size ->
    builder.setBulkSize(new ByteSizeValue(size, ByteSizeUnit.MB))
  );
  config.getBulkFlushIntervalMS().ifPresent(interval ->
    builder.setFlushInterval(TimeValue.timeValueMillis(interval))
  );

  return builder.build();
}
 

@Test(expected = SamzaException.class)
public void testFlushFailedSendFromFailedDocument() throws Exception {
  ArgumentCaptor<BulkProcessor.Listener> listenerCaptor =
      ArgumentCaptor.forClass(BulkProcessor.Listener.class);

  when(BULK_PROCESSOR_FACTORY.getBulkProcessor(eq(CLIENT), listenerCaptor.capture()))
      .thenReturn(processorOne);
  producer.register(SOURCE_ONE);

  BulkResponse response = getRespWithFailedDocument(RestStatus.BAD_REQUEST);

  listenerCaptor.getValue().afterBulk(0, null, response);

  producer.flush(SOURCE_ONE);
}
 

@Override
public BulkProcessor getBulkProcessor(BulkProcessor.Listener listener) {
  return BulkProcessor.builder(client, listener)
      .setBulkActions(properties.getBulkActions())
      .setBulkSize(new ByteSizeValue(properties.getBulkSize(), ByteSizeUnit.MB))
      .setFlushInterval(TimeValue.timeValueSeconds(properties.getBulkFlushInterval()))
      .setConcurrentRequests(properties.getConcurrentRequests())
      .build();
}
 

@Test
public void no_value_inserted_when_not_enough_requests() {

    // Create Mocks:
    Client mockedTransportClient = mock(Client.class);
    BulkProcessor.Listener mockedBulkProcessorListener = mock(BulkProcessor.Listener.class);

    // Configure the BulkProcessor to use:
    BulkProcessorConfiguration configuration = new BulkProcessorConfiguration(new BulkProcessingOptionsBuilder().build(), mockedBulkProcessorListener);

    // And create a fake index builder:
    IndexRequestBuilder indexRequestBuilder = new IndexRequestBuilder(mockedTransportClient, IndexAction.INSTANCE);

    // The mapping to use:
    IElasticSearchMapping localWeatherDataMapper = new LocalWeatherDataMapper();

    // Index to insert to:
    String indexName = "weather_data";

    // Initialize it with the default settings:
    when(mockedTransportClient.settings())
            .thenReturn(Settings.builder().build());

    when(mockedTransportClient.prepareIndex())
            .thenReturn(indexRequestBuilder);

    // Create the Test subject:
    ElasticSearchClient<LocalWeatherData> elasticSearchClient = new ElasticSearchClient<>(mockedTransportClient, indexName, localWeatherDataMapper, configuration);

    // Create more entities, than Bulk insertion threshold:
    Stream<LocalWeatherData> entitiesStream = getData(configuration.getBulkProcessingOptions().getBulkActions() - 1).stream();

    // Index the Data:
    elasticSearchClient.index(entitiesStream);

    // Verify, that the TransportClient bulk insert has been called:
    verify(mockedTransportClient, times(0)).bulk(anyObject(), anyObject());
    verify(mockedBulkProcessorListener, times(0)).beforeBulk(anyLong(), anyObject());
}
 

@Override
public BatchEmitter createInstance(int batchSize, int deliveryInterval, ClientObjectFactory clientObjectFactory, FailoverPolicy failoverPolicy) {

    Function<BulkRequest, Boolean> failureHandler = clientObjectFactory.createFailureHandler(failoverPolicy);

    BulkProcessor.Listener listener = new BulkExecutionListener(failureHandler);

    BulkProcessor.Builder builder = BulkProcessor.builder((Client) clientObjectFactory.createClient(), listener)
            .setBulkActions(batchSize)
            .setFlushInterval(TimeValue.timeValueMillis(deliveryInterval));
    return new BulkProcessorDelegate(builder.build());
}
 

/**
 * Build the {@link BulkProcessor}.
 *
 * <p>Note: this is exposed for testing purposes.
 */
@VisibleForTesting
protected BulkProcessor buildBulkProcessor(BulkProcessor.Listener listener) {
	checkNotNull(listener);

	BulkProcessor.Builder bulkProcessorBuilder = callBridge.createBulkProcessorBuilder(client, listener);

	// This makes flush() blocking
	bulkProcessorBuilder.setConcurrentRequests(0);

	if (bulkProcessorFlushMaxActions != null) {
		bulkProcessorBuilder.setBulkActions(bulkProcessorFlushMaxActions);
	}

	if (bulkProcessorFlushMaxSizeMb != null) {
		bulkProcessorBuilder.setBulkSize(new ByteSizeValue(bulkProcessorFlushMaxSizeMb, ByteSizeUnit.MB));
	}

	if (bulkProcessorFlushIntervalMillis != null) {
		bulkProcessorBuilder.setFlushInterval(TimeValue.timeValueMillis(bulkProcessorFlushIntervalMillis));
	}

	// if backoff retrying is disabled, bulkProcessorFlushBackoffPolicy will be null
	callBridge.configureBulkProcessorBackoff(bulkProcessorBuilder, bulkProcessorFlushBackoffPolicy);

	return bulkProcessorBuilder.build();
}
 

@Test(expected = SamzaException.class)
public void testFlushFailedSendFromException() throws Exception {
  ArgumentCaptor<BulkProcessor.Listener> listenerCaptor =
      ArgumentCaptor.forClass(BulkProcessor.Listener.class);

  when(BULK_PROCESSOR_FACTORY.getBulkProcessor(eq(CLIENT), listenerCaptor.capture()))
      .thenReturn(processorOne);
  producer.register(SOURCE_ONE);

  listenerCaptor.getValue().afterBulk(0, null, new Throwable());

  producer.flush(SOURCE_ONE);
}
 

/**
 * Build the {@link BulkProcessor}.
 *
 * <p>Note: this is exposed for testing purposes.
 */
@VisibleForTesting
protected BulkProcessor buildBulkProcessor(BulkProcessor.Listener listener) {
	checkNotNull(listener);

	BulkProcessor.Builder bulkProcessorBuilder = callBridge.createBulkProcessorBuilder(client, listener);

	// This makes flush() blocking
	bulkProcessorBuilder.setConcurrentRequests(0);

	if (bulkProcessorFlushMaxActions != null) {
		bulkProcessorBuilder.setBulkActions(bulkProcessorFlushMaxActions);
	}

	if (bulkProcessorFlushMaxSizeMb != null) {
		bulkProcessorBuilder.setBulkSize(new ByteSizeValue(bulkProcessorFlushMaxSizeMb, ByteSizeUnit.MB));
	}

	if (bulkProcessorFlushIntervalMillis != null) {
		bulkProcessorBuilder.setFlushInterval(TimeValue.timeValueMillis(bulkProcessorFlushIntervalMillis));
	}

	// if backoff retrying is disabled, bulkProcessorFlushBackoffPolicy will be null
	callBridge.configureBulkProcessorBackoff(bulkProcessorBuilder, bulkProcessorFlushBackoffPolicy);

	return bulkProcessorBuilder.build();
}
 

@Override
public BulkProcessor.Builder createBulkProcessorBuilder(TransportClient client, BulkProcessor.Listener listener) {
	return BulkProcessor.builder(client, listener);
}
 

@Override
public BulkProcessor.Builder createBulkProcessorBuilder(Client client, BulkProcessor.Listener listener) {
	return null;
}
 

@Override
public BulkProcessor.Builder createBulkProcessorBuilder(RestHighLevelClient client, BulkProcessor.Listener listener) {
	return BulkProcessor.builder(client::bulkAsync, listener);
}
 

public BulkProcessorConfiguration(BulkProcessingOptions options, BulkProcessor.Listener listener) {
    this.options = options;
    this.listener = listener;
}
 

@Override
public BulkProcessor.Builder createBulkProcessorBuilder(Client client, BulkProcessor.Listener listener) {
	return null;
}
 

@Override
public BulkProcessor.Builder createBulkProcessorBuilder(TransportClient client, BulkProcessor.Listener listener) {
	return BulkProcessor.builder(client, listener);
}
 

@Override
public BulkProcessor.Builder createBulkProcessorBuilder(Client client, BulkProcessor.Listener listener) {
	return null;
}
 

public BulkProcessorConfiguration(BulkProcessingOptions options, BulkProcessor.Listener listener) {
    this.options = options;
    this.listener = listener;
}
 

@Override
public BulkProcessor.Builder createBulkProcessorBuilder(RestHighLevelClient client, BulkProcessor.Listener listener) {
	return BulkProcessor.builder(client::bulkAsync, listener);
}
 

@Override
public BulkProcessor.Builder createBulkProcessorBuilder(TransportClient client, BulkProcessor.Listener listener) {
	return BulkProcessor.builder(client, listener);
}
 

@Override
public BulkProcessor.Builder createBulkProcessorBuilder(RestHighLevelClient client, BulkProcessor.Listener listener) {
	return BulkProcessor.builder((request, bulkListener) -> client.bulkAsync(request, RequestOptions.DEFAULT, bulkListener), listener);
}
 

/**
 * Creates a {@link BulkProcessor.Builder} for creating the bulk processor.
 *
 * @param client the Elasticsearch client.
 * @param listener the bulk processor listender.
 * @return the bulk processor builder.
 */
BulkProcessor.Builder createBulkProcessorBuilder(C client, BulkProcessor.Listener listener);