Java Code Examples for org.elasticsearch.action.bulk.BulkResponse#getItems()
The following examples show how to use
org.elasticsearch.action.bulk.BulkResponse#getItems() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: ElasticsearchIndexer.java From datashare with GNU Affero General Public License v3.0 | 6 votes |
|
@Override
public <T extends Entity> boolean bulkUpdate(String indexName, List<? extends Entity> entities) throws IOException {
BulkRequest bulkRequest = new BulkRequest();
entities.stream().map(e -> createUpdateRequest(indexName, getType(e), e.getId(), getJson(e), getParent(e), getRoot(e))).
forEach(bulkRequest::add);
bulkRequest.setRefreshPolicy(esCfg.refreshPolicy);
BulkResponse bulkResponse = client.bulk(bulkRequest);
if (bulkResponse.hasFailures()) {
for (BulkItemResponse resp : bulkResponse.getItems()) {
if (resp.isFailed()) {
LOGGER.error("bulk update failed : {}", resp.getFailureMessage());
}
}
return false;
}
return true;
}
Example 2
| Source File: ESTemplate.java From canal-1.1.3 with Apache License 2.0 | 6 votes |
|
/**
* 提交批次
*/
public void commit() {
if (getBulk().numberOfActions() > 0) {
BulkResponse response = getBulk().execute().actionGet();
if (response.hasFailures()) {
for (BulkItemResponse itemResponse : response.getItems()) {
if (!itemResponse.isFailed()) {
continue;
}
if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) {
logger.error(itemResponse.getFailureMessage());
} else {
throw new RuntimeException("ES sync commit error" + itemResponse.getFailureMessage());
}
}
}
}
}
Example 3
| Source File: ElasticSearchRepository.java From elastic-crud with Apache License 2.0 | 6 votes |
|
@Override
public List<String> deleteAllIds(final Collection<String> ids) {
if (ids.isEmpty()) {
return ImmutableList.of();
}
final BulkRequestBuilder bulk = client
.prepareBulk()
.setRefreshPolicy(policy.get());
for (final String id : ids) {
bulk.add(client.prepareDelete(index, type, id));
}
final BulkResponse response = bulk.execute().actionGet();
final ImmutableList.Builder<String> builder = ImmutableList.builder();
for (final BulkItemResponse item : response.getItems()) {
builder.add(item.getId());
}
return builder.build();
}
Example 4
| Source File: ElasticsearchDocumentWriter.java From CogStack-Pipeline with Apache License 2.0 | 6 votes |
|
private void getResponses(BulkResponse response) {
if (response.hasFailures()) {
for (int i = 0; i < response.getItems().length; i++) {
//in bulk processing, retry all docs one by one. if one fails, log it. If the entire chunk fails,
// raise an exception towards skip limit
if (response.getItems().length == 1) {
LOG.warn("failed to index document: " + response.getItems()[i].getId() + " failure is: \n"
+ response.getItems()[i].getFailureMessage());
} else {
LOG.error("failed to index document: " + response.getItems()[i].getFailureMessage());
throw new ElasticsearchException("Bulk indexing request failed");
}
}
}
LOG.info("{} documents indexed into ElasticSearch in {} ms", response.getItems().length,
response.getIngestTookInMillis());
}
Example 5
| Source File: ESTest.java From canal with Apache License 2.0 | 6 votes |
|
private void commit(BulkRequestBuilder bulkRequestBuilder) {
if (bulkRequestBuilder.numberOfActions() > 0) {
BulkResponse response = bulkRequestBuilder.execute().actionGet();
if (response.hasFailures()) {
for (BulkItemResponse itemResponse : response.getItems()) {
if (!itemResponse.isFailed()) {
continue;
}
if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) {
System.out.println(itemResponse.getFailureMessage());
} else {
System.out.println("ES bulk commit error" + itemResponse.getFailureMessage());
}
}
}
}
}
Example 6
| Source File: ImportBulkListener.java From elasticsearch-inout-plugin with Apache License 2.0 | 6 votes |
|
@Override
public void afterBulk(long executionId, BulkRequest request,
BulkResponse response) {
bulksInProgress.decrementAndGet();
if (response.hasFailures()) {
for (BulkItemResponse item : response.getItems()) {
if (item.isFailed()) {
counts.failures++;
} else {
counts.successes++;
}
}
} else {
counts.successes += response.getItems().length;
}
checkRelease();
}
Example 7
| Source File: PercolateTagProcessor.java From streams with Apache License 2.0 | 5 votes |
|
/**
*
* @return returns true if all rules were addded. False indicates one or more rules have failed.
*/
public boolean writePercolateRules() {
if (this.numOfPercolateRules() < 0) {
throw new RuntimeException("No Rules Have been added!");
}
BulkResponse response = this.bulkBuilder.execute().actionGet();
for (BulkItemResponse r : response.getItems()) {
if (r.isFailed()) {
LOGGER.error(r.getId() + "\t" + r.getFailureMessage());
}
}
return !response.hasFailures();
}
Example 8
| Source File: ElasticsearchSinkBase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Override
public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
if (response.hasFailures()) {
BulkItemResponse itemResponse;
Throwable failure;
RestStatus restStatus;
try {
for (int i = 0; i < response.getItems().length; i++) {
itemResponse = response.getItems()[i];
failure = callBridge.extractFailureCauseFromBulkItemResponse(itemResponse);
if (failure != null) {
LOG.error("Failed Elasticsearch item request: {}", itemResponse.getFailureMessage(), failure);
restStatus = itemResponse.getFailure().getStatus();
if (restStatus == null) {
failureHandler.onFailure(request.requests().get(i), failure, -1, failureRequestIndexer);
} else {
failureHandler.onFailure(request.requests().get(i), failure, restStatus.getStatus(), failureRequestIndexer);
}
}
}
} catch (Throwable t) {
// fail the sink and skip the rest of the items
// if the failure handler decides to throw an exception
failureThrowable.compareAndSet(null, t);
}
}
if (flushOnCheckpoint) {
numPendingRequests.getAndAdd(-request.numberOfActions());
}
}
Example 9
| Source File: IndexBatchBolt.java From storm-trident-elasticsearch with Apache License 2.0 | 5 votes |
|
protected void bulkUpdateIndexes( ) {
List<Tuple> inputs = new ArrayList<>(queue.size());
queue.drainTo(inputs);
BulkRequestBuilder bulkRequest = client.prepareBulk();
for (Tuple input : inputs) {
Document<T> doc = mapper.map(input);
IndexRequestBuilder request = client.prepareIndex(doc.getName(), doc.getType(), doc.getId()).setSource((String)doc.getSource());
if(doc.getParentId() != null) {
request.setParent(doc.getParentId());
}
bulkRequest.add(request);
}
try {
if (bulkRequest.numberOfActions() > 0) {
BulkResponse bulkItemResponses = bulkRequest.execute().actionGet();
if (bulkItemResponses.hasFailures()) {
BulkItemResponse[] items = bulkItemResponses.getItems();
for (int i = 0; i < items.length; i++) {
ackOrFail(items[i], inputs.get(i));
}
} else {
ackAll(inputs);
}
}
} catch (ElasticsearchException e) {
LOGGER.error("Unable to process bulk request, " + inputs.size() + " tuples are in failure", e);
outputCollector.reportError(e.getRootCause());
failAll(inputs);
}
}
Example 10
| Source File: BaseElasticSearchIndexBuilder.java From sakai with Educational Community License v2.0 | 5 votes |
|
protected void executeBulkRequest(BulkRequestBuilder bulkRequest) {
BulkResponse bulkResponse = bulkRequest.execute().actionGet();
getLog().info("Bulk request of batch size: " + bulkRequest.numberOfActions() + " took "
+ bulkResponse.getTookInMillis() + " ms in index builder: " + getName());
for (BulkItemResponse response : bulkResponse.getItems()) {
if (response.getResponse() instanceof DeleteResponse) {
DeleteResponse deleteResponse = response.getResponse();
if (response.isFailed()) {
getLog().error("Problem deleting doc: " + response.getId() + " in index builder: " + getName()
+ " error: " + response.getFailureMessage());
} else if (!deleteResponse.isFound()) {
getLog().debug("ES could not find a doc with id: " + deleteResponse.getId()
+ " to delete in index builder: " + getName());
} else {
getLog().debug("ES deleted a doc with id: " + deleteResponse.getId() + " in index builder: "
+ getName());
}
} else if (response.getResponse() instanceof IndexResponse) {
IndexResponse indexResponse = response.getResponse();
if (response.isFailed()) {
getLog().error("Problem updating content for doc: " + response.getId() + " in index builder: "
+ getName() + " error: " + response.getFailureMessage());
} else {
getLog().debug("ES indexed content for doc with id: " + indexResponse.getId()
+ " in index builder: " + getName());
}
}
}
}
Example 11
| Source File: ConfService.java From SkaETL with Apache License 2.0 | 5 votes |
|
private void callAddES(ConfigurationLogstash cl) throws IOException {
BulkRequest bulk = new BulkRequest();
ObjectMapper objectMapper = new ObjectMapper();
String fluxJson = objectMapper.writeValueAsString(ConfEsSkalogs.builder().configurationLogstash(cl).pipeline(StringEscapeUtils.escapeJava(utilsConfig.generateConfig(cl))).build());
bulk.add(new IndexRequest(INDEX_STORAGE)
.type("doc")
.id(cl.getIdConfiguration())
.source(fluxJson, XContentType.JSON));
BulkResponse bulkResponse = restHighLevelClient.bulk(bulk);
if (bulkResponse.getItems().length == 1) {
cl.setIdEs(bulkResponse.getItems()[0].getId());
} else {
log.error("Problem with return ES {}", bulkResponse);
}
}
Example 12
| Source File: ElasticIndexer.java From bluima with Apache License 2.0 | 5 votes |
|
private void flushBulk() {
BulkResponse bulkResponse = bulkRequest.execute().actionGet(); // flush
if (bulkResponse.hasFailures()) { // log failures
for (BulkItemResponse r : bulkResponse.getItems()) {
LOG.error(r.getFailureMessage());
Failure failure = r.getFailure();
// e.g. when ES server is overloaded
throw new ElasticsearchException(failure.toString());
}
}
}
Example 13
| Source File: ElasticsearchPersistWriter.java From streams with Apache License 2.0 | 5 votes |
|
private void updateTotals(final BulkResponse bulkItemResponses, final Long sent, final Long sizeInBytes) {
long failed = 0;
long passed = 0;
long millis = bulkItemResponses.getTookInMillis();
// keep track of the number of totalFailed and items that we have totalOk.
for (BulkItemResponse resp : bulkItemResponses.getItems()) {
if (resp == null || resp.isFailed()) {
failed++;
LOGGER.debug("{} ({},{},{}) failed: {}", resp.getOpType(), resp.getIndex(), resp.getType(), resp.getId(), resp.getFailureMessage());
} else {
passed++;
}
}
if (failed > 0) {
LOGGER.warn("Bulk Uploading had {} failures of {}", failed, sent);
}
this.totalOk.addAndGet(passed);
this.totalFailed.addAndGet(failed);
this.totalSeconds.addAndGet(millis / 1000);
this.totalSizeInBytes.addAndGet(sizeInBytes);
if (sent != (passed + failed)) {
LOGGER.error("Count MisMatch: Sent[{}] Passed[{}] Failed[{}]", sent, passed, failed);
}
LOGGER.debug("Batch[{}mb {} items with {} failures in {}ms] - Total[{}mb {} items with {} failures in {}seconds] {} outstanding]",
MEGABYTE_FORMAT.format(sizeInBytes / (double) (1024 * 1024)), NUMBER_FORMAT.format(passed), NUMBER_FORMAT.format(failed), NUMBER_FORMAT.format(millis),
MEGABYTE_FORMAT.format((double) totalSizeInBytes.get() / (double) (1024 * 1024)), NUMBER_FORMAT.format(totalOk), NUMBER_FORMAT.format(totalFailed), NUMBER_FORMAT.format(totalSeconds), NUMBER_FORMAT.format(getTotalOutstanding()));
}
Example 14
| Source File: ESEtlService.java From canal-1.1.3 with Apache License 2.0 | 5 votes |
|
private void processFailBulkResponse(BulkResponse bulkResponse) {
for (BulkItemResponse response : bulkResponse.getItems()) {
if (!response.isFailed()) {
continue;
}
if (response.getFailure().getStatus() == RestStatus.NOT_FOUND) {
logger.warn(response.getFailureMessage());
} else {
logger.error("全量导入数据有误 {}", response.getFailureMessage());
throw new RuntimeException("全量数据 etl 异常: " + response.getFailureMessage());
}
}
}
Example 15
| Source File: TransportClient.java From elasticsearch-jest-example with MIT License | 5 votes |
|
/**
* bulkIndex
* @throws Exception
*/
private static void bulkIndex() throws Exception{
Client client = createTransportClient();
BulkRequestBuilder bulkRequest = client.prepareBulk();
bulkRequest.add(client.prepareIndex("book", "book", "3")
.setSource(jsonBuilder()
.startObject()
.field("name", "Docker开发实践")
.field("author", "曾金龙 肖新华 刘清")
.field("pubinfo", "人民邮电出版社")
.field("pubtime", "2015-07-01")
.field("desc", "《Docker开发实践》由浅入深地介绍了Docker的实践之道，首先讲解Docker的概念、容器和镜像的相关操作、容器的数据管理等内容，接着通过不同类型的应用说明Docker的实际应用，然后介绍了网络、安全、API、管理工具Fig、Kubernetes、shipyard以及Docker三件套（Machine+Swarm+Compose）等，最后列举了常见镜像、Docker API等内容。")
.endObject()
)
);
bulkRequest.add(client.prepareIndex("book", "book", "4")
.setSource(jsonBuilder()
.startObject()
.field("name", "图灵程序设计丛书：Hadoop基础教程")
.field("author", "张治起")
.field("pubinfo", "人民邮电出版社")
.field("pubtime", "2014-01-01")
.field("desc", "《图灵程序设计丛书：Hadoop基础教程》包括三个主要部分：第1~5章讲述了Hadoop的核心机制及Hadoop的工作模式；第6~7章涵盖了Hadoop更多可操作的内容；第8~11章介绍了Hadoop与其他产品和技术的组合使用。《图灵程序设计丛书：Hadoop基础教程》目的在于帮助读者了解什么是Hadoop，Hadoop是如何工作的，以及如何使用Hadoop从数据中提取有价值的信息，并用它解决大数据问题")
.endObject()
)
);
BulkResponse bulkResponse = bulkRequest.execute().actionGet();
if (bulkResponse.hasFailures()) {
BulkItemResponse[] bulkItemResponse = bulkResponse.getItems();
for (int i = 0; i <bulkItemResponse.length ; i++) {
System.out.println(bulkItemResponse[i].getItemId()+":"+bulkItemResponse[i].getIndex()+":"+bulkItemResponse[i].getFailureMessage());
}
}
}
Example 16
| Source File: TransportClient.java From elasticsearch-jest-example with MIT License | 5 votes |
|
/**
* 删除查询到的文档
* @param index
* @param name
* @param value
*/
private static void scrollSearchDelete(String index,String name,String value){
Client client = createTransportClient();
QueryBuilder qb = termQuery(name, value);
SearchResponse scrollResp = client.prepareSearch(index)
.setSearchType(SearchType.SCAN)
.setScroll(new TimeValue(60000))
.setQuery(qb)
.setSize(100).execute().actionGet(); //100 hits per shard will be returned for each scroll
BulkRequestBuilder bulkRequest = client.prepareBulk();
while (true) {
for (SearchHit hit : scrollResp.getHits().getHits()) {
bulkRequest.add(client.prepareDelete(hit.getIndex(),hit.getType(),hit.getId()));
}
scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet();
if (scrollResp.getHits().getHits().length == 0) {
break;
}
}
BulkResponse bulkResponse = bulkRequest.execute().actionGet();
if (bulkResponse.hasFailures()) {
BulkItemResponse[] bulkItemResponse = bulkResponse.getItems();
for (int i = 0; i <bulkItemResponse.length ; i++) {
System.out.println(bulkItemResponse[i].getItemId()+":"+bulkItemResponse[i].getIndex()+":"+bulkItemResponse[i].getFailureMessage());
}
}
}
Example 17
| Source File: ElasticsearchGis.java From Elasticsearch-Tutorial-zh-CN with GNU General Public License v3.0 | 4 votes |
|
/**
* 批量创建
* 资料：http://blog.csdn.net/loveisnull/article/details/45914115
* 纬度在前，经度在后
*
* @param transportClient
*/
private static void batchCreate(TransportClient transportClient) throws IOException {
BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();
IndexRequestBuilder indexRequestBuilder1 = transportClient.prepareIndex("shop_index", "shop", "1")
.setSource(XContentFactory.jsonBuilder()
.startObject()
.field("shop_name", "麻辣香锅1")
.latlon("location", 40.12, -71.34)
.field("created_date_time", new SimpleDateFormat("yyyyMMdd'T'HHmmss.SSSZ").format(new Date()))
.field("last_modified_date_time", new SimpleDateFormat("yyyyMMdd'T'HHmmss.SSSZ").format(new Date()))
.field("version", 1)
.endObject());
IndexRequestBuilder indexRequestBuilder2 = transportClient.prepareIndex("shop_index", "shop", "2")
.setSource(XContentFactory.jsonBuilder()
.startObject()
.field("shop_name", "麻辣香锅2")
.latlon("location", 40.12, -72.34)
.field("created_date_time", new SimpleDateFormat("yyyyMMdd'T'HHmmss.SSSZ").format(new Date()))
.field("last_modified_date_time", new SimpleDateFormat("yyyyMMdd'T'HHmmss.SSSZ").format(new Date()))
.field("version", 1)
.endObject());
IndexRequestBuilder indexRequestBuilder3 = transportClient.prepareIndex("shop_index", "shop", "3")
.setSource(XContentFactory.jsonBuilder()
.startObject()
.field("shop_name", "麻辣香锅3")
.latlon("location", 40.12, -73.34)
.field("created_date_time", new SimpleDateFormat("yyyyMMdd'T'HHmmss.SSSZ").format(new Date()))
.field("last_modified_date_time", new SimpleDateFormat("yyyyMMdd'T'HHmmss.SSSZ").format(new Date()))
.field("version", 1)
.endObject());
bulkRequestBuilder.add(indexRequestBuilder1);
bulkRequestBuilder.add(indexRequestBuilder2);
bulkRequestBuilder.add(indexRequestBuilder3);
BulkResponse bulkResponse = bulkRequestBuilder.get();
for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) {
logger.info("--------------------------------version= " + bulkItemResponse.getVersion());
}
}
Example 18
| Source File: BaseDemo.java From Elasticsearch-Tutorial-zh-CN with GNU General Public License v3.0 | 4 votes |
|
/**
* 批量创建
*
* @param transportClient
*/
private static void batchCreate(TransportClient transportClient) throws IOException {
BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();
IndexRequestBuilder indexRequestBuilder1 = transportClient.prepareIndex("product_index", "product", "1")
.setSource(XContentFactory.jsonBuilder()
.startObject()
.field("product_name", "飞利浦电动牙刷 HX6700-1")
.field("product_desc", "前 1000 名赠刷头，6 月 1 日 0 点火爆开抢，618 开门红巅峰 48 小时")
.field("price", 399.00)
.field("created_date_time", new SimpleDateFormat("yyyyMMdd'T'HHmmss.SSSZ").format(new Date()))
.field("last_modified_date_time", new SimpleDateFormat("yyyyMMdd'T'HHmmss.SSSZ").format(new Date()))
.field("version", 1)
.endObject());
IndexRequestBuilder indexRequestBuilder2 = transportClient.prepareIndex("product_index", "product", "2")
.setSource(XContentFactory.jsonBuilder()
.startObject()
.field("product_name", "飞利浦电动牙刷 HX6700-2")
.field("product_desc", "前 1000 名赠刷头，6 月 1 日 0 点火爆开抢，618 开门红巅峰 48 小时")
.field("price", 399.00)
.field("created_date_time", new SimpleDateFormat("yyyyMMdd'T'HHmmss.SSSZ").format(new Date()))
.field("last_modified_date_time", new SimpleDateFormat("yyyyMMdd'T'HHmmss.SSSZ").format(new Date()))
.field("version", 1)
.endObject());
IndexRequestBuilder indexRequestBuilder3 = transportClient.prepareIndex("product_index", "product", "3")
.setSource(XContentFactory.jsonBuilder()
.startObject()
.field("product_name", "飞利浦电动牙刷 HX6700-3")
.field("product_desc", "前 1000 名赠刷头，6 月 1 日 0 点火爆开抢，618 开门红巅峰 48 小时")
.field("price", 399.00)
.field("created_date_time", new SimpleDateFormat("yyyyMMdd'T'HHmmss.SSSZ").format(new Date()))
.field("last_modified_date_time", new SimpleDateFormat("yyyyMMdd'T'HHmmss.SSSZ").format(new Date()))
.field("version", 1)
.endObject());
bulkRequestBuilder.add(indexRequestBuilder1);
bulkRequestBuilder.add(indexRequestBuilder2);
bulkRequestBuilder.add(indexRequestBuilder3);
BulkResponse bulkResponse = bulkRequestBuilder.get();
for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) {
logger.info("--------------------------------version= " + bulkItemResponse.getVersion());
}
}
Example 19
| Source File: ElasticsearchSinkBase.java From flink with Apache License 2.0 | 4 votes |
|
@Override
public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
if (response.hasFailures()) {
BulkItemResponse itemResponse;
Throwable failure;
RestStatus restStatus;
DocWriteRequest actionRequest;
try {
for (int i = 0; i < response.getItems().length; i++) {
itemResponse = response.getItems()[i];
failure = callBridge.extractFailureCauseFromBulkItemResponse(itemResponse);
if (failure != null) {
restStatus = itemResponse.getFailure().getStatus();
actionRequest = request.requests().get(i);
if (restStatus == null) {
if (actionRequest instanceof ActionRequest) {
failureHandler.onFailure((ActionRequest) actionRequest, failure, -1, failureRequestIndexer);
} else {
throw new UnsupportedOperationException("The sink currently only supports ActionRequests");
}
} else {
if (actionRequest instanceof ActionRequest) {
failureHandler.onFailure((ActionRequest) actionRequest, failure, restStatus.getStatus(), failureRequestIndexer);
} else {
throw new UnsupportedOperationException("The sink currently only supports ActionRequests");
}
}
}
}
} catch (Throwable t) {
// fail the sink and skip the rest of the items
// if the failure handler decides to throw an exception
failureThrowable.compareAndSet(null, t);
}
}
if (flushOnCheckpoint) {
numPendingRequests.getAndAdd(-request.numberOfActions());
}
}
Example 20
| Source File: ElasticsearchEmitter.java From amazon-kinesis-connectors with Apache License 2.0 | 4 votes |
|
/**
* Emits records to elasticsearch.
* 1. Adds each record to a bulk index request, conditionally adding version, ttl or create if they were set in the
* transformer.
* 2. Executes the bulk request, returning any record specific failures to be retried by the connector library
* pipeline, unless
* outlined below.
*
* Record specific failures (noted in the failure.getMessage() string)
* - DocumentAlreadyExistsException means the record has create set to true, but a document already existed at the
* specific index/type/id.
* - VersionConflictEngineException means the record has a specific version number that did not match what existed
* in elasticsearch.
* To guarantee in order processing by the connector, when putting data use the same partition key for objects going
* to the same
* index/type/id and set sequence number for ordering.
* - In either case, the emitter will assume that the record would fail again in the future and thus will not return
* the record to
* be retried.
*
* Bulk request failures
* - NoNodeAvailableException means the TransportClient could not connect to the cluster.
* - A general Exception catches any other unexpected behavior.
* - In either case the emitter will continue making attempts until the issue has been resolved. This is to ensure
* that no data
* loss occurs and simplifies restarting the application once issues have been fixed.
*/
@Override
public List<ElasticsearchObject> emit(UnmodifiableBuffer<ElasticsearchObject> buffer) throws IOException {
List<ElasticsearchObject> records = buffer.getRecords();
if (records.isEmpty()) {
return Collections.emptyList();
}
BulkRequestBuilder bulkRequest = elasticsearchClient.prepareBulk();
for (ElasticsearchObject record : records) {
IndexRequestBuilder indexRequestBuilder =
elasticsearchClient.prepareIndex(record.getIndex(), record.getType(), record.getId());
indexRequestBuilder.setSource(record.getSource());
Long version = record.getVersion();
if (version != null) {
indexRequestBuilder.setVersion(version);
}
Long ttl = record.getTtl();
if (ttl != null) {
indexRequestBuilder.setTTL(ttl);
}
Boolean create = record.getCreate();
if (create != null) {
indexRequestBuilder.setCreate(create);
}
bulkRequest.add(indexRequestBuilder);
}
while (true) {
try {
BulkResponse bulkResponse = bulkRequest.execute().actionGet();
BulkItemResponse[] responses = bulkResponse.getItems();
List<ElasticsearchObject> failures = new ArrayList<ElasticsearchObject>();
int numberOfSkippedRecords = 0;
for (int i = 0; i < responses.length; i++) {
if (responses[i].isFailed()) {
LOG.error("Record failed with message: " + responses[i].getFailureMessage());
Failure failure = responses[i].getFailure();
if (failure.getMessage().contains("DocumentAlreadyExistsException")
|| failure.getMessage().contains("VersionConflictEngineException")) {
numberOfSkippedRecords++;
} else {
failures.add(records.get(i));
}
}
}
LOG.info("Emitted " + (records.size() - failures.size() - numberOfSkippedRecords)
+ " records to Elasticsearch");
if (!failures.isEmpty()) {
printClusterStatus();
LOG.warn("Returning " + failures.size() + " records as failed");
}
return failures;
} catch (NoNodeAvailableException nnae) {
LOG.error("No nodes found at " + elasticsearchEndpoint + ":" + elasticsearchPort + ". Retrying in "
+ BACKOFF_PERIOD + " milliseconds", nnae);
sleep(BACKOFF_PERIOD);
} catch (Exception e) {
LOG.error("ElasticsearchEmitter threw an unexpected exception ", e);
sleep(BACKOFF_PERIOD);
}
}
}
