org.elasticsearch.action.bulk.BulkItemResponse Java Examples
The following examples show how to use
org.elasticsearch.action.bulk.BulkItemResponse.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestPutElasticsearch.java From localization_nifi with Apache License 2.0 | 7 votes |
@Override public BulkResponse get() throws InterruptedException, ExecutionException { BulkResponse response = mock(BulkResponse.class); when(response.hasFailures()).thenReturn(responseHasFailures); BulkItemResponse item1 = mock(BulkItemResponse.class); BulkItemResponse item2 = mock(BulkItemResponse.class); when(item1.getItemId()).thenReturn(1); when(item1.isFailed()).thenReturn(true); BulkItemResponse.Failure failure = mock(BulkItemResponse.Failure.class); when(failure.getMessage()).thenReturn("Bad message"); when(item1.getFailure()).thenReturn(failure); when(item2.getItemId()).thenReturn(2); when(item2.isFailed()).thenReturn(false); when(response.getItems()).thenReturn(new BulkItemResponse[]{item1, item2}); return response; }
Example #2
Source File: BaseDemo.java From Elasticsearch-Tutorial-zh-CN with GNU General Public License v3.0 | 6 votes |
/** * 批量删除 * * @param transportClient */ private static void batchDelete(TransportClient transportClient) throws IOException { BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); DeleteRequestBuilder deleteRequestBuilder1 = transportClient.prepareDelete("product_index", "product", "1"); DeleteRequestBuilder deleteRequestBuilder2 = transportClient.prepareDelete("product_index", "product", "2"); DeleteRequestBuilder deleteRequestBuilder3 = transportClient.prepareDelete("product_index", "product", "3"); bulkRequestBuilder.add(deleteRequestBuilder1); bulkRequestBuilder.add(deleteRequestBuilder2); bulkRequestBuilder.add(deleteRequestBuilder3); BulkResponse bulkResponse = bulkRequestBuilder.get(); for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) { logger.info("--------------------------------version= " + bulkItemResponse.getVersion()); } }
Example #3
Source File: BulkUpdateService.java From vertexium with Apache License 2.0 | 6 votes |
private void handleFailure(BulkItem<?> bulkItem, BulkItemResponse bulkItemResponse) { BulkItemResponse.Failure failure = bulkItemResponse.getFailure(); bulkItem.incrementFailCount(); if (bulkItem.getFailCount() >= maxFailCount) { complete(bulkItem, new BulkVertexiumException("fail count exceeded the max number of failures", failure)); } else { AtomicBoolean retry = new AtomicBoolean(false); try { searchIndex.handleBulkFailure(bulkItem, bulkItemResponse, retry); } catch (Exception ex) { complete(bulkItem, ex); return; } if (retry.get()) { incomingItems.addAll(bulkItem.getItems()); } else { complete(bulkItem, null); } } }
Example #4
Source File: TestElasticsearch7ExceptionHandler.java From vertexium with Apache License 2.0 | 6 votes |
@Override public void handleBulkFailure( Graph graph, Elasticsearch7SearchIndex elasticsearch7SearchIndex, BulkItem<?> bulkItem, BulkItemResponse bulkItemResponse, AtomicBoolean retry ) { numberOfTimesCalled.incrementAndGet(); LOGGER.warn("bulk failure on item %s: %s", bulkItem, bulkItemResponse == null ? null : bulkItemResponse.getFailure()); if (bulkItemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) { LoadAndAddDocumentMissingHelper.handleDocumentMissingException(graph, elasticsearch7SearchIndex, bulkItem, authorizations); } else { retry.set(true); } }
Example #5
Source File: TestPutElasticsearch.java From nifi with Apache License 2.0 | 6 votes |
@Override public BulkResponse get() throws InterruptedException, ExecutionException { BulkResponse response = mock(BulkResponse.class); when(response.hasFailures()).thenReturn(responseHasFailures); BulkItemResponse item1 = mock(BulkItemResponse.class); BulkItemResponse item2 = mock(BulkItemResponse.class); when(item1.getItemId()).thenReturn(1); when(item1.isFailed()).thenReturn(true); BulkItemResponse.Failure failure = mock(BulkItemResponse.Failure.class); when(failure.getMessage()).thenReturn("Bad message"); when(item1.getFailure()).thenReturn(failure); when(item2.getItemId()).thenReturn(2); when(item2.isFailed()).thenReturn(false); when(response.getItems()).thenReturn(new BulkItemResponse[]{item1, item2}); return response; }
Example #6
Source File: ImportBulkListener.java From elasticsearch-inout-plugin with Apache License 2.0 | 6 votes |
@Override public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { bulksInProgress.decrementAndGet(); if (response.hasFailures()) { for (BulkItemResponse item : response.getItems()) { if (item.isFailed()) { counts.failures++; } else { counts.successes++; } } } else { counts.successes += response.getItems().length; } checkRelease(); }
Example #7
Source File: ElasticSearchListener.java From garmadon with Apache License 2.0 | 6 votes |
@Override public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { if (response.hasFailures()) { LOGGER.error("Bulk[{}] executed with failures", executionId); for (BulkItemResponse item : response.getItems()) { if (item.isFailed()) { LOGGER.error("Failed on {} due to {}", item.getId(), item.getFailureMessage()); numberOfEventInError.inc(); } } } else { LOGGER.info("Successfully completed Bulk[{}] in {} ms", executionId, response.getTook().getMillis()); latencyIndexingEvents.observe(response.getTook().getMillis()); } CommittableOffset<String, byte[]> lastOffset = (CommittableOffset<String, byte[]>) request.payloads().get(request.payloads().size() - 1); lastOffset .commitAsync() .whenComplete((topicPartitionOffset, exception) -> { if (exception != null) { LOGGER.warn("Could not commit kafka offset {}|{}", lastOffset.getOffset(), lastOffset.getPartition()); numberOfOffsetCommitError.inc(); } else { LOGGER.info("Committed kafka offset {}|{}", topicPartitionOffset.getOffset(), topicPartitionOffset.getPartition()); } }); }
Example #8
Source File: ESTest.java From canal-1.1.3 with Apache License 2.0 | 6 votes |
private void commit(BulkRequestBuilder bulkRequestBuilder) { if (bulkRequestBuilder.numberOfActions() > 0) { BulkResponse response = bulkRequestBuilder.execute().actionGet(); if (response.hasFailures()) { for (BulkItemResponse itemResponse : response.getItems()) { if (!itemResponse.isFailed()) { continue; } if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) { System.out.println(itemResponse.getFailureMessage()); } else { System.out.println("ES bulk commit error" + itemResponse.getFailureMessage()); } } } } }
Example #9
Source File: ElasticsearchIndexer.java From datashare with GNU Affero General Public License v3.0 | 6 votes |
@Override public <T extends Entity> boolean bulkUpdate(String indexName, List<? extends Entity> entities) throws IOException { BulkRequest bulkRequest = new BulkRequest(); entities.stream().map(e -> createUpdateRequest(indexName, getType(e), e.getId(), getJson(e), getParent(e), getRoot(e))). forEach(bulkRequest::add); bulkRequest.setRefreshPolicy(esCfg.refreshPolicy); BulkResponse bulkResponse = client.bulk(bulkRequest); if (bulkResponse.hasFailures()) { for (BulkItemResponse resp : bulkResponse.getItems()) { if (resp.isFailed()) { LOGGER.error("bulk update failed : {}", resp.getFailureMessage()); } } return false; } return true; }
Example #10
Source File: ESTemplate.java From canal-1.1.3 with Apache License 2.0 | 6 votes |
/** * 提交批次 */ public void commit() { if (getBulk().numberOfActions() > 0) { BulkResponse response = getBulk().execute().actionGet(); if (response.hasFailures()) { for (BulkItemResponse itemResponse : response.getItems()) { if (!itemResponse.isFailed()) { continue; } if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) { logger.error(itemResponse.getFailureMessage()); } else { throw new RuntimeException("ES sync commit error" + itemResponse.getFailureMessage()); } } } } }
Example #11
Source File: TestElasticsearch5ExceptionHandler.java From vertexium with Apache License 2.0 | 6 votes |
@Override public void handleBulkFailure( Graph graph, Elasticsearch5SearchIndex elasticsearch5SearchIndex, BulkItem<?> bulkItem, BulkItemResponse bulkItemResponse, AtomicBoolean retry ) { numberOfTimesCalled.incrementAndGet(); LOGGER.warn("bulk failure on item %s: %s", bulkItem, bulkItemResponse == null ? null : bulkItemResponse.getFailure()); if (bulkItemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) { LoadAndAddDocumentMissingHelper.handleDocumentMissingException(graph, elasticsearch5SearchIndex, bulkItem, authorizations); } else { retry.set(true); } }
Example #12
Source File: Output.java From data-generator with Apache License 2.0 | 6 votes |
private static void writeBatchToES(String index, String type, List<Map<String, Object>> list) throws Exception{ if(list.isEmpty()){ return; } BulkRequest request = new BulkRequest(); for(Map<String, Object> data : list) { String id = data.get("id").toString(); request.add( new IndexRequest(index, type, id) .source(data)); } BulkResponse bulkResponse = CLIENT.bulk(request); if (bulkResponse.hasFailures()) { for (BulkItemResponse bulkItemResponse : bulkResponse) { if (bulkItemResponse.isFailed()) { BulkItemResponse.Failure failure = bulkItemResponse.getFailure(); LOGGER.error("ES索引失败: {}", failure.getMessage()); } } } }
Example #13
Source File: ESRequestMapperTest.java From syncer with BSD 3-Clause "New" or "Revised" License | 6 votes |
private static void remoteCheck(AbstractClient client, List<Object> builderList) throws ExecutionException, InterruptedException { for (Object builder : builderList) { BulkRequestBuilder bulkRequestBuilder = null; if (builder instanceof IndexRequestBuilder) { bulkRequestBuilder = client.prepareBulk().add((IndexRequestBuilder) builder); } else if (builder instanceof UpdateRequestBuilder) { bulkRequestBuilder = client.prepareBulk().add((UpdateRequestBuilder) builder); } else if (builder instanceof DeleteRequestBuilder) { bulkRequestBuilder = client.prepareBulk().add((DeleteRequestBuilder) builder); } else { fail(); } BulkResponse bulkItemResponses = bulkRequestBuilder.execute().get(); assertFalse(Arrays.stream(bulkItemResponses.getItems()).anyMatch(BulkItemResponse::isFailed)); } }
Example #14
Source File: KafkaRiver.java From elasticsearch-river-kafka with Apache License 2.0 | 6 votes |
void executeBuilder(BulkRequestBuilder bulkRequestBuilder) { if(bulkRequestBuilder.numberOfActions() == 0) return; ++stats.flushes; BulkResponse response = bulkRequestBuilder.execute().actionGet(); if (response.hasFailures()) { logger.warn("failed to execute" + response.buildFailureMessage()); } for(BulkItemResponse resp : response){ if(resp.isFailed()){ stats.failed++; }else{ stats.succeeded++; } } }
Example #15
Source File: ElasticSearchUtilImpl.java From metacat with Apache License 2.0 | 6 votes |
/** * Permanently delete index documents. * * @param type index type * @param ids entity ids */ private void hardDeleteDoc(final String type, final List<String> ids) { try { RETRY_ES_PUBLISH.call(() -> { final BulkRequestBuilder bulkRequest = client.prepareBulk(); ids.forEach(id -> bulkRequest.add(client.prepareDelete(esIndex, type, id))); final BulkResponse bulkResponse = bulkRequest.execute().actionGet(esBulkCallTimeout); log.info("Deleting metadata of type {} with count {}", type, ids.size()); if (bulkResponse.hasFailures()) { for (BulkItemResponse item : bulkResponse.getItems()) { if (item.isFailed()) { handleException("ElasticSearchUtil.bulkDelete.item", type, item.getId(), item.getFailure().getCause(), Metrics.CounterElasticSearchDelete.getMetricName()); } } } return null; }); } catch (Exception e) { handleException("ElasticSearchUtil.bulkDelete", type, ids, e, Metrics.CounterElasticSearchBulkDelete.getMetricName()); } }
Example #16
Source File: ElasticSearchUtilImpl.java From metacat with Apache License 2.0 | 6 votes |
private void updateDocs(final String type, final List<String> ids, final ObjectNode node) { try { RETRY_ES_PUBLISH.call(() -> { final BulkRequestBuilder bulkRequest = client.prepareBulk(); ids.forEach(id -> { bulkRequest.add(client.prepareUpdate(esIndex, type, id) .setRetryOnConflict(NO_OF_CONFLICT_RETRIES) .setDoc(metacatJson.toJsonAsBytes(node), XContentType.JSON)); }); final BulkResponse bulkResponse = bulkRequest.execute().actionGet(esBulkCallTimeout); if (bulkResponse.hasFailures()) { for (BulkItemResponse item : bulkResponse.getItems()) { if (item.isFailed()) { handleException("ElasticSearchUtil.updateDocs.item", type, item.getId(), item.getFailure().getCause(), Metrics.CounterElasticSearchUpdate.getMetricName()); } } } return null; }); } catch (Exception e) { handleException("ElasticSearchUtil.updatDocs", type, ids, e, Metrics.CounterElasticSearchBulkUpdate.getMetricName()); } }
Example #17
Source File: ElasticsearchSinkBaseTest.java From flink with Apache License 2.0 | 5 votes |
@Nullable @Override public Throwable extractFailureCauseFromBulkItemResponse(BulkItemResponse bulkItemResponse) { if (bulkItemResponse.isFailed()) { return new Exception(bulkItemResponse.getFailure().getMessage()); } else { return null; } }
Example #18
Source File: ElasticSearchBulkIndexStrategy.java From camunda-bpm-elasticsearch with Apache License 2.0 | 5 votes |
@Override public void executeRequest(List<HistoryEvent> historyEvents) { BulkRequestBuilder bulkRequestBuilder = esClient.prepareBulk(); try { for (HistoryEvent historyEvent : historyEvents) { UpdateRequestBuilder updateRequestBuilder = prepareUpdateRequest(historyEvent); bulkRequestBuilder.add(updateRequestBuilder); } BulkResponse bulkResponse; if (WAIT_FOR_RESPONSE > 0) { bulkResponse = bulkRequestBuilder.get(TimeValue.timeValueSeconds(WAIT_FOR_RESPONSE)); } else { bulkResponse = bulkRequestBuilder.get(); } if (bulkResponse.hasFailures()) { LOGGER.severe("Error while executing bulk request: " + bulkResponse.buildFailureMessage()); } if (LOGGER.isLoggable(Level.FINEST)) { for (BulkItemResponse bulkItemResponse : bulkResponse) { LOGGER.finest("[" + bulkItemResponse.getIndex() + "][" + bulkItemResponse.getType() + "][" + bulkItemResponse.getOpType() + "] process instance with id '" + bulkItemResponse.getId() + "'"); } } } catch (IOException e) { LOGGER.log(Level.SEVERE, e.getMessage(), e); } }
Example #19
Source File: DcBulkItemResponseImpl.java From io with Apache License 2.0 | 5 votes |
/** * . * @param response . * @return . */ public static DcBulkItemResponse getInstance(BulkItemResponse response) { if (response == null) { return null; } return new DcBulkItemResponseImpl(response.getItemId(), response.getOpType(), response.getResponse()); }
Example #20
Source File: DcBulkResponseImpl.java From io with Apache License 2.0 | 5 votes |
@Override public DcBulkItemResponse[] items() { List<DcBulkItemResponse> list = new ArrayList<DcBulkItemResponse>(); for (BulkItemResponse response : this.bulkResponse.getItems()) { list.add(DcBulkItemResponseImpl.getInstance(response)); } return list.toArray(new DcBulkItemResponse[0]); }
Example #21
Source File: DcBulkItemResponseImpl.java From io with Apache License 2.0 | 5 votes |
/** * . * @param response . * @return . */ public static DcBulkItemResponse getInstance(BulkItemResponse response) { if (response == null) { return null; } return new DcBulkItemResponseImpl(response.getItemId(), response.getOpType(), response.getResponse()); }
Example #22
Source File: DcBulkResponseImpl.java From io with Apache License 2.0 | 5 votes |
@Override public DcBulkItemResponse[] items() { List<DcBulkItemResponse> list = new ArrayList<DcBulkItemResponse>(); for (BulkItemResponse response : this.bulkResponse.getItems()) { list.add(DcBulkItemResponseImpl.getInstance(response)); } return list.toArray(new DcBulkItemResponse[0]); }
Example #23
Source File: Elasticsearch7ApiCallBridge.java From flink with Apache License 2.0 | 5 votes |
@Override public Throwable extractFailureCauseFromBulkItemResponse(BulkItemResponse bulkItemResponse) { if (!bulkItemResponse.isFailed()) { return null; } else { return bulkItemResponse.getFailure().getCause(); } }
Example #24
Source File: Elasticsearch5ExceptionHandler.java From vertexium with Apache License 2.0 | 5 votes |
default void handleBulkFailure( Graph graph, Elasticsearch5SearchIndex elasticsearch5SearchIndex, BulkItem<?> bulkItem, BulkItemResponse bulkItemResponse, AtomicBoolean retry ) throws Exception { VertexiumLoggerFactory.getLogger(Elasticsearch5ExceptionHandler.class) .error("bulk failure: %s: %s", bulkItem, bulkItemResponse.getFailureMessage()); retry.set(true); }
Example #25
Source File: TestPutElasticsearch5.java From nifi with Apache License 2.0 | 5 votes |
@Override public BulkResponse get() throws InterruptedException, ExecutionException { BulkResponse response = mock(BulkResponse.class); when(response.hasFailures()).thenReturn(responseHasFailures); BulkItemResponse item = mock(BulkItemResponse.class); when(item.getItemId()).thenReturn(1); when(item.isFailed()).thenReturn(true); when(response.getItems()).thenReturn(new BulkItemResponse[]{item}); TransportAddress remoteAddress = mock(TransportAddress.class); when(remoteAddress.getAddress()).thenReturn(address.toString()); when(response.remoteAddress()).thenReturn(remoteAddress); return response; }
Example #26
Source File: ElasticSearchUtilImpl.java From metacat with Apache License 2.0 | 5 votes |
/** * Bulk save of the entities. * * @param type index type * @param docs metacat documents */ private void bulkSaveToIndex(final String type, final List<ElasticSearchDoc> docs, final String index) { if (docs != null && !docs.isEmpty()) { try { RETRY_ES_PUBLISH.call(() -> { final BulkRequestBuilder bulkRequest = client.prepareBulk(); for (ElasticSearchDoc doc : docs) { final IndexRequestBuilder indexRequestBuilder = prepareIndexRequest(index, type, doc); if (indexRequestBuilder != null) { bulkRequest.add(indexRequestBuilder); } } if (bulkRequest.numberOfActions() > 0) { final BulkResponse bulkResponse = bulkRequest.execute().actionGet(esBulkCallTimeout); log.info("Bulk saving metadata of index {} type {} with size {}.", index, type, docs.size()); if (bulkResponse.hasFailures()) { for (BulkItemResponse item : bulkResponse.getItems()) { if (item.isFailed()) { handleException("ElasticSearchUtil.bulkSaveToIndex.index", type, item.getId(), item.getFailure().getCause(), Metrics.CounterElasticSearchSave.getMetricName()); } } } } return null; }); } catch (Exception e) { final List<String> docIds = docs.stream().map(ElasticSearchDoc::getId).collect(Collectors.toList()); handleException("ElasticSearchUtil.bulkSaveToIndex", type, docIds, e, Metrics.CounterElasticSearchBulkSave.getMetricName()); } } }
Example #27
Source File: EsDoc.java From AsuraFramework with Apache License 2.0 | 5 votes |
private void handBulkResponseException(BulkResponse bulkResponse){ if (bulkResponse.hasFailures()) { long count = 0L; for (BulkItemResponse bulkItemResponse : bulkResponse) { logger.error("发生错误的 索引id为 : "+bulkItemResponse.getId()+" ,错误信息为:"+ bulkItemResponse.getFailureMessage()); count++; } logger.error("发生错误的 总个数为 : "+count); } }
Example #28
Source File: Elasticsearch7SearchIndex.java From vertexium with Apache License 2.0 | 5 votes |
public void handleBulkFailure(BulkItem<?> bulkItem, BulkItemResponse bulkItemResponse, AtomicBoolean retry) throws Exception { if (exceptionHandler == null) { LOGGER.error("bulk failure: %s: %s", bulkItem, bulkItemResponse.getFailureMessage()); return; } if (LOGGER.isTraceEnabled()) { LOGGER.trace("handleBulkFailure: (bulkItem: %s, bulkItemResponse: %s)", bulkItem, bulkItemResponse); } exceptionHandler.handleBulkFailure(graph, this, bulkItem, bulkItemResponse, retry); }
Example #29
Source File: ElasticsearchPersistWriter.java From streams with Apache License 2.0 | 5 votes |
private void updateTotals(final BulkResponse bulkItemResponses, final Long sent, final Long sizeInBytes) { long failed = 0; long passed = 0; long millis = bulkItemResponses.getTookInMillis(); // keep track of the number of totalFailed and items that we have totalOk. for (BulkItemResponse resp : bulkItemResponses.getItems()) { if (resp == null || resp.isFailed()) { failed++; LOGGER.debug("{} ({},{},{}) failed: {}", resp.getOpType(), resp.getIndex(), resp.getType(), resp.getId(), resp.getFailureMessage()); } else { passed++; } } if (failed > 0) { LOGGER.warn("Bulk Uploading had {} failures of {}", failed, sent); } this.totalOk.addAndGet(passed); this.totalFailed.addAndGet(failed); this.totalSeconds.addAndGet(millis / 1000); this.totalSizeInBytes.addAndGet(sizeInBytes); if (sent != (passed + failed)) { LOGGER.error("Count MisMatch: Sent[{}] Passed[{}] Failed[{}]", sent, passed, failed); } LOGGER.debug("Batch[{}mb {} items with {} failures in {}ms] - Total[{}mb {} items with {} failures in {}seconds] {} outstanding]", MEGABYTE_FORMAT.format(sizeInBytes / (double) (1024 * 1024)), NUMBER_FORMAT.format(passed), NUMBER_FORMAT.format(failed), NUMBER_FORMAT.format(millis), MEGABYTE_FORMAT.format((double) totalSizeInBytes.get() / (double) (1024 * 1024)), NUMBER_FORMAT.format(totalOk), NUMBER_FORMAT.format(totalFailed), NUMBER_FORMAT.format(totalSeconds), NUMBER_FORMAT.format(getTotalOutstanding())); }
Example #30
Source File: ElasticSearchUtilImpl.java From metacat with Apache License 2.0 | 5 votes |
private void softDeleteDoc( final String type, final List<String> ids, final MetacatRequestContext metacatRequestContext) { try { RETRY_ES_PUBLISH.call(() -> { final BulkRequestBuilder bulkRequest = client.prepareBulk(); final XContentBuilder builder = XContentFactory.contentBuilder(contentType); builder.startObject().field(ElasticSearchDoc.Field.DELETED, true) .field(ElasticSearchDoc.Field.TIMESTAMP, java.time.Instant.now().toEpochMilli()) .field(ElasticSearchDoc.Field.USER, metacatRequestContext.getUserName()).endObject(); ids.forEach(id -> bulkRequest.add(client.prepareUpdate(esIndex, type, id) .setRetryOnConflict(NO_OF_CONFLICT_RETRIES).setDoc(builder))); final BulkResponse bulkResponse = bulkRequest.execute().actionGet(esBulkCallTimeout); if (bulkResponse.hasFailures()) { for (BulkItemResponse item : bulkResponse.getItems()) { if (item.isFailed()) { handleException("ElasticSearchUtil.bulkSoftDelete.item", type, item.getId(), item.getFailure().getCause(), Metrics.CounterElasticSearchDelete.getMetricName()); } } } return null; }); } catch (Exception e) { handleException("ElasticSearchUtil.bulkSoftDelete", type, ids, e, Metrics.CounterElasticSearchBulkDelete.getMetricName()); } }