org.elasticsearch.action.bulk.BulkItemResponse Java Examples

The following examples show how to use org.elasticsearch.action.bulk.BulkItemResponse. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestPutElasticsearch.java    From localization_nifi with Apache License 2.0 7 votes vote down vote up
@Override
public BulkResponse get() throws InterruptedException, ExecutionException {
    BulkResponse response = mock(BulkResponse.class);
    when(response.hasFailures()).thenReturn(responseHasFailures);
    BulkItemResponse item1 = mock(BulkItemResponse.class);
    BulkItemResponse item2 = mock(BulkItemResponse.class);
    when(item1.getItemId()).thenReturn(1);
    when(item1.isFailed()).thenReturn(true);
    BulkItemResponse.Failure failure = mock(BulkItemResponse.Failure.class);
    when(failure.getMessage()).thenReturn("Bad message");
    when(item1.getFailure()).thenReturn(failure);
    when(item2.getItemId()).thenReturn(2);
    when(item2.isFailed()).thenReturn(false);
    when(response.getItems()).thenReturn(new BulkItemResponse[]{item1, item2});
    return response;
}
 
Example #2
Source File: BaseDemo.java    From Elasticsearch-Tutorial-zh-CN with GNU General Public License v3.0 6 votes vote down vote up
/**
 * 批量删除
 *
 * @param transportClient
 */
private static void batchDelete(TransportClient transportClient) throws IOException {
	BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();

	DeleteRequestBuilder deleteRequestBuilder1 = transportClient.prepareDelete("product_index", "product", "1");
	DeleteRequestBuilder deleteRequestBuilder2 = transportClient.prepareDelete("product_index", "product", "2");
	DeleteRequestBuilder deleteRequestBuilder3 = transportClient.prepareDelete("product_index", "product", "3");

	bulkRequestBuilder.add(deleteRequestBuilder1);
	bulkRequestBuilder.add(deleteRequestBuilder2);
	bulkRequestBuilder.add(deleteRequestBuilder3);

	BulkResponse bulkResponse = bulkRequestBuilder.get();
	for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) {
		logger.info("--------------------------------version= " + bulkItemResponse.getVersion());
	}

}
 
Example #3
Source File: BulkUpdateService.java    From vertexium with Apache License 2.0 6 votes vote down vote up
private void handleFailure(BulkItem<?> bulkItem, BulkItemResponse bulkItemResponse) {
    BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
    bulkItem.incrementFailCount();
    if (bulkItem.getFailCount() >= maxFailCount) {
        complete(bulkItem, new BulkVertexiumException("fail count exceeded the max number of failures", failure));
    } else {
        AtomicBoolean retry = new AtomicBoolean(false);
        try {
            searchIndex.handleBulkFailure(bulkItem, bulkItemResponse, retry);
        } catch (Exception ex) {
            complete(bulkItem, ex);
            return;
        }
        if (retry.get()) {
            incomingItems.addAll(bulkItem.getItems());
        } else {
            complete(bulkItem, null);
        }
    }
}
 
Example #4
Source File: TestElasticsearch7ExceptionHandler.java    From vertexium with Apache License 2.0 6 votes vote down vote up
@Override
public void handleBulkFailure(
    Graph graph,
    Elasticsearch7SearchIndex elasticsearch7SearchIndex,
    BulkItem<?> bulkItem,
    BulkItemResponse bulkItemResponse,
    AtomicBoolean retry
) {
    numberOfTimesCalled.incrementAndGet();
    LOGGER.warn("bulk failure on item %s: %s", bulkItem, bulkItemResponse == null ? null : bulkItemResponse.getFailure());
    if (bulkItemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) {
        LoadAndAddDocumentMissingHelper.handleDocumentMissingException(graph, elasticsearch7SearchIndex, bulkItem, authorizations);
    } else {
        retry.set(true);
    }
}
 
Example #5
Source File: TestPutElasticsearch.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Override
public BulkResponse get() throws InterruptedException, ExecutionException {
    BulkResponse response = mock(BulkResponse.class);
    when(response.hasFailures()).thenReturn(responseHasFailures);
    BulkItemResponse item1 = mock(BulkItemResponse.class);
    BulkItemResponse item2 = mock(BulkItemResponse.class);
    when(item1.getItemId()).thenReturn(1);
    when(item1.isFailed()).thenReturn(true);
    BulkItemResponse.Failure failure = mock(BulkItemResponse.Failure.class);
    when(failure.getMessage()).thenReturn("Bad message");
    when(item1.getFailure()).thenReturn(failure);
    when(item2.getItemId()).thenReturn(2);
    when(item2.isFailed()).thenReturn(false);
    when(response.getItems()).thenReturn(new BulkItemResponse[]{item1, item2});
    return response;
}
 
Example #6
Source File: ImportBulkListener.java    From elasticsearch-inout-plugin with Apache License 2.0 6 votes vote down vote up
@Override
public void afterBulk(long executionId, BulkRequest request,
        BulkResponse response) {
    bulksInProgress.decrementAndGet();
    if (response.hasFailures()) {
        for (BulkItemResponse item : response.getItems()) {
            if (item.isFailed()) {
                counts.failures++;
            } else {
                counts.successes++;
            }
        }
    } else {
        counts.successes += response.getItems().length;
    }
    checkRelease();
}
 
Example #7
Source File: ElasticSearchListener.java    From garmadon with Apache License 2.0 6 votes vote down vote up
@Override
public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
    if (response.hasFailures()) {
        LOGGER.error("Bulk[{}] executed with failures", executionId);
        for (BulkItemResponse item : response.getItems()) {
            if (item.isFailed()) {
                LOGGER.error("Failed on {} due to {}", item.getId(), item.getFailureMessage());
                numberOfEventInError.inc();
            }
        }
    } else {
        LOGGER.info("Successfully completed Bulk[{}] in {} ms", executionId, response.getTook().getMillis());
        latencyIndexingEvents.observe(response.getTook().getMillis());
    }
    CommittableOffset<String, byte[]> lastOffset = (CommittableOffset<String, byte[]>) request.payloads().get(request.payloads().size() - 1);
    lastOffset
            .commitAsync()
            .whenComplete((topicPartitionOffset, exception) -> {
                if (exception != null) {
                    LOGGER.warn("Could not commit kafka offset {}|{}", lastOffset.getOffset(), lastOffset.getPartition());
                    numberOfOffsetCommitError.inc();
                } else {
                    LOGGER.info("Committed kafka offset {}|{}", topicPartitionOffset.getOffset(), topicPartitionOffset.getPartition());
                }
            });
}
 
Example #8
Source File: ESTest.java    From canal-1.1.3 with Apache License 2.0 6 votes vote down vote up
private void commit(BulkRequestBuilder bulkRequestBuilder) {
    if (bulkRequestBuilder.numberOfActions() > 0) {
        BulkResponse response = bulkRequestBuilder.execute().actionGet();
        if (response.hasFailures()) {
            for (BulkItemResponse itemResponse : response.getItems()) {
                if (!itemResponse.isFailed()) {
                    continue;
                }

                if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) {
                    System.out.println(itemResponse.getFailureMessage());
                } else {
                    System.out.println("ES bulk commit error" + itemResponse.getFailureMessage());
                }
            }
        }
    }
}
 
Example #9
Source File: ElasticsearchIndexer.java    From datashare with GNU Affero General Public License v3.0 6 votes vote down vote up
@Override
public <T extends Entity> boolean bulkUpdate(String indexName, List<? extends Entity> entities) throws IOException {
    BulkRequest bulkRequest = new BulkRequest();
    entities.stream().map(e -> createUpdateRequest(indexName, getType(e), e.getId(), getJson(e), getParent(e), getRoot(e))).
            forEach(bulkRequest::add);
    bulkRequest.setRefreshPolicy(esCfg.refreshPolicy);

    BulkResponse bulkResponse = client.bulk(bulkRequest);
    if (bulkResponse.hasFailures()) {
        for (BulkItemResponse resp : bulkResponse.getItems()) {
            if (resp.isFailed()) {
                LOGGER.error("bulk update failed : {}", resp.getFailureMessage());
            }
        }
        return false;
    }
    return true;
}
 
Example #10
Source File: ESTemplate.java    From canal-1.1.3 with Apache License 2.0 6 votes vote down vote up
/**
 * 提交批次
 */
public void commit() {
    if (getBulk().numberOfActions() > 0) {
        BulkResponse response = getBulk().execute().actionGet();
        if (response.hasFailures()) {
            for (BulkItemResponse itemResponse : response.getItems()) {
                if (!itemResponse.isFailed()) {
                    continue;
                }

                if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) {
                    logger.error(itemResponse.getFailureMessage());
                } else {
                    throw new RuntimeException("ES sync commit error" + itemResponse.getFailureMessage());
                }
            }
        }
    }
}
 
Example #11
Source File: TestElasticsearch5ExceptionHandler.java    From vertexium with Apache License 2.0 6 votes vote down vote up
@Override
public void handleBulkFailure(
    Graph graph,
    Elasticsearch5SearchIndex elasticsearch5SearchIndex,
    BulkItem<?> bulkItem,
    BulkItemResponse bulkItemResponse,
    AtomicBoolean retry
) {
    numberOfTimesCalled.incrementAndGet();
    LOGGER.warn("bulk failure on item %s: %s", bulkItem, bulkItemResponse == null ? null : bulkItemResponse.getFailure());
    if (bulkItemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) {
        LoadAndAddDocumentMissingHelper.handleDocumentMissingException(graph, elasticsearch5SearchIndex, bulkItem, authorizations);
    } else {
        retry.set(true);
    }
}
 
Example #12
Source File: Output.java    From data-generator with Apache License 2.0 6 votes vote down vote up
private static void writeBatchToES(String index, String type, List<Map<String, Object>> list) throws Exception{
    if(list.isEmpty()){
        return;
    }
    BulkRequest request = new BulkRequest();
    for(Map<String, Object> data : list) {
        String id = data.get("id").toString();
        request.add(
                new IndexRequest(index, type, id)
                        .source(data));

    }
    BulkResponse bulkResponse = CLIENT.bulk(request);
    if (bulkResponse.hasFailures()) {
        for (BulkItemResponse bulkItemResponse : bulkResponse) {
            if (bulkItemResponse.isFailed()) {
                BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
                LOGGER.error("ES索引失败: {}", failure.getMessage());
            }
        }
    }
}
 
Example #13
Source File: ESRequestMapperTest.java    From syncer with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
private static void remoteCheck(AbstractClient client, List<Object> builderList) throws ExecutionException, InterruptedException {
  for (Object builder : builderList) {
    BulkRequestBuilder bulkRequestBuilder = null;
    if (builder instanceof IndexRequestBuilder) {
      bulkRequestBuilder = client.prepareBulk().add((IndexRequestBuilder) builder);
    } else if (builder instanceof UpdateRequestBuilder) {
      bulkRequestBuilder = client.prepareBulk().add((UpdateRequestBuilder) builder);
    }  else if (builder instanceof DeleteRequestBuilder) {
      bulkRequestBuilder = client.prepareBulk().add((DeleteRequestBuilder) builder);
    } else {
      fail();
    }
    BulkResponse bulkItemResponses = bulkRequestBuilder.execute().get();
    assertFalse(Arrays.stream(bulkItemResponses.getItems()).anyMatch(BulkItemResponse::isFailed));
  }
}
 
Example #14
Source File: KafkaRiver.java    From elasticsearch-river-kafka with Apache License 2.0 6 votes vote down vote up
void executeBuilder(BulkRequestBuilder bulkRequestBuilder)
{
	if(bulkRequestBuilder.numberOfActions() == 0)
		return;
	
	++stats.flushes;
	BulkResponse response = bulkRequestBuilder.execute().actionGet();
	if (response.hasFailures()) {
              logger.warn("failed to execute" + response.buildFailureMessage());
          }
	
	for(BulkItemResponse resp : response){
		if(resp.isFailed()){
			stats.failed++;
		}else{
			stats.succeeded++;
		}
	}
}
 
Example #15
Source File: ElasticSearchUtilImpl.java    From metacat with Apache License 2.0 6 votes vote down vote up
/**
 * Permanently delete index documents.
 *
 * @param type index type
 * @param ids  entity ids
 */
private void hardDeleteDoc(final String type, final List<String> ids) {
    try {
        RETRY_ES_PUBLISH.call(() -> {
            final BulkRequestBuilder bulkRequest = client.prepareBulk();
            ids.forEach(id -> bulkRequest.add(client.prepareDelete(esIndex, type, id)));
            final BulkResponse bulkResponse = bulkRequest.execute().actionGet(esBulkCallTimeout);
            log.info("Deleting metadata of type {} with count {}", type, ids.size());
            if (bulkResponse.hasFailures()) {
                for (BulkItemResponse item : bulkResponse.getItems()) {
                    if (item.isFailed()) {
                        handleException("ElasticSearchUtil.bulkDelete.item", type, item.getId(),
                            item.getFailure().getCause(), Metrics.CounterElasticSearchDelete.getMetricName());
                    }
                }
            }
            return null;
        });
    } catch (Exception e) {
        handleException("ElasticSearchUtil.bulkDelete", type, ids, e,
            Metrics.CounterElasticSearchBulkDelete.getMetricName());
    }
}
 
Example #16
Source File: ElasticSearchUtilImpl.java    From metacat with Apache License 2.0 6 votes vote down vote up
private void updateDocs(final String type, final List<String> ids, final ObjectNode node) {
    try {
        RETRY_ES_PUBLISH.call(() -> {
            final BulkRequestBuilder bulkRequest = client.prepareBulk();
            ids.forEach(id -> {
                bulkRequest.add(client.prepareUpdate(esIndex, type, id)
                    .setRetryOnConflict(NO_OF_CONFLICT_RETRIES)
                    .setDoc(metacatJson.toJsonAsBytes(node), XContentType.JSON));
            });
            final BulkResponse bulkResponse = bulkRequest.execute().actionGet(esBulkCallTimeout);
            if (bulkResponse.hasFailures()) {
                for (BulkItemResponse item : bulkResponse.getItems()) {
                    if (item.isFailed()) {
                        handleException("ElasticSearchUtil.updateDocs.item", type, item.getId(),
                            item.getFailure().getCause(), Metrics.CounterElasticSearchUpdate.getMetricName());
                    }
                }
            }
            return null;
        });
    } catch (Exception e) {
        handleException("ElasticSearchUtil.updatDocs", type, ids, e,
            Metrics.CounterElasticSearchBulkUpdate.getMetricName());
    }
}
 
Example #17
Source File: ElasticsearchSinkBaseTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Nullable
@Override
public Throwable extractFailureCauseFromBulkItemResponse(BulkItemResponse bulkItemResponse) {
	if (bulkItemResponse.isFailed()) {
		return new Exception(bulkItemResponse.getFailure().getMessage());
	} else {
		return null;
	}
}
 
Example #18
Source File: ElasticSearchBulkIndexStrategy.java    From camunda-bpm-elasticsearch with Apache License 2.0 5 votes vote down vote up
@Override
public void executeRequest(List<HistoryEvent> historyEvents) {
  BulkRequestBuilder bulkRequestBuilder = esClient.prepareBulk();
  try {
    for (HistoryEvent historyEvent : historyEvents) {
      UpdateRequestBuilder updateRequestBuilder = prepareUpdateRequest(historyEvent);
      bulkRequestBuilder.add(updateRequestBuilder);
    }

    BulkResponse bulkResponse;
    if (WAIT_FOR_RESPONSE > 0) {
      bulkResponse = bulkRequestBuilder.get(TimeValue.timeValueSeconds(WAIT_FOR_RESPONSE));
    } else {
      bulkResponse = bulkRequestBuilder.get();
    }

    if (bulkResponse.hasFailures()) {
      LOGGER.severe("Error while executing bulk request: " + bulkResponse.buildFailureMessage());
    }

    if (LOGGER.isLoggable(Level.FINEST)) {
      for (BulkItemResponse bulkItemResponse : bulkResponse) {
        LOGGER.finest("[" + bulkItemResponse.getIndex() +
            "][" + bulkItemResponse.getType() +
            "][" + bulkItemResponse.getOpType() +
            "] process instance with id '" + bulkItemResponse.getId() + "'");
      }
    }
  } catch (IOException e) {
    LOGGER.log(Level.SEVERE, e.getMessage(), e);
  }
}
 
Example #19
Source File: DcBulkItemResponseImpl.java    From io with Apache License 2.0 5 votes vote down vote up
/**
 * .
 * @param response .
 * @return .
 */
public static DcBulkItemResponse getInstance(BulkItemResponse response) {
    if (response == null) {
        return null;
    }
    return new DcBulkItemResponseImpl(response.getItemId(), response.getOpType(), response.getResponse());
}
 
Example #20
Source File: DcBulkResponseImpl.java    From io with Apache License 2.0 5 votes vote down vote up
@Override
public DcBulkItemResponse[] items() {
    List<DcBulkItemResponse> list = new ArrayList<DcBulkItemResponse>();
    for (BulkItemResponse response : this.bulkResponse.getItems()) {
        list.add(DcBulkItemResponseImpl.getInstance(response));
    }
    return list.toArray(new DcBulkItemResponse[0]);
}
 
Example #21
Source File: DcBulkItemResponseImpl.java    From io with Apache License 2.0 5 votes vote down vote up
/**
 * .
 * @param response .
 * @return .
 */
public static DcBulkItemResponse getInstance(BulkItemResponse response) {
    if (response == null) {
        return null;
    }
    return new DcBulkItemResponseImpl(response.getItemId(), response.getOpType(), response.getResponse());
}
 
Example #22
Source File: DcBulkResponseImpl.java    From io with Apache License 2.0 5 votes vote down vote up
@Override
public DcBulkItemResponse[] items() {
    List<DcBulkItemResponse> list = new ArrayList<DcBulkItemResponse>();
    for (BulkItemResponse response : this.bulkResponse.getItems()) {
        list.add(DcBulkItemResponseImpl.getInstance(response));
    }
    return list.toArray(new DcBulkItemResponse[0]);
}
 
Example #23
Source File: Elasticsearch7ApiCallBridge.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public Throwable extractFailureCauseFromBulkItemResponse(BulkItemResponse bulkItemResponse) {
	if (!bulkItemResponse.isFailed()) {
		return null;
	} else {
		return bulkItemResponse.getFailure().getCause();
	}
}
 
Example #24
Source File: Elasticsearch5ExceptionHandler.java    From vertexium with Apache License 2.0 5 votes vote down vote up
default void handleBulkFailure(
    Graph graph,
    Elasticsearch5SearchIndex elasticsearch5SearchIndex,
    BulkItem<?> bulkItem,
    BulkItemResponse bulkItemResponse,
    AtomicBoolean retry
) throws Exception {
    VertexiumLoggerFactory.getLogger(Elasticsearch5ExceptionHandler.class)
        .error("bulk failure: %s: %s", bulkItem, bulkItemResponse.getFailureMessage());
    retry.set(true);
}
 
Example #25
Source File: TestPutElasticsearch5.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Override
public BulkResponse get() throws InterruptedException, ExecutionException {
    BulkResponse response = mock(BulkResponse.class);
    when(response.hasFailures()).thenReturn(responseHasFailures);
    BulkItemResponse item = mock(BulkItemResponse.class);
    when(item.getItemId()).thenReturn(1);
    when(item.isFailed()).thenReturn(true);
    when(response.getItems()).thenReturn(new BulkItemResponse[]{item});
    TransportAddress remoteAddress = mock(TransportAddress.class);
    when(remoteAddress.getAddress()).thenReturn(address.toString());
    when(response.remoteAddress()).thenReturn(remoteAddress);
    return response;
}
 
Example #26
Source File: ElasticSearchUtilImpl.java    From metacat with Apache License 2.0 5 votes vote down vote up
/**
 * Bulk save of the entities.
 *
 * @param type index type
 * @param docs metacat documents
 */
private void bulkSaveToIndex(final String type, final List<ElasticSearchDoc> docs, final String index) {
    if (docs != null && !docs.isEmpty()) {
        try {

            RETRY_ES_PUBLISH.call(() -> {
                final BulkRequestBuilder bulkRequest = client.prepareBulk();
                for (ElasticSearchDoc doc : docs) {
                    final IndexRequestBuilder indexRequestBuilder = prepareIndexRequest(index, type, doc);
                    if (indexRequestBuilder != null) {
                        bulkRequest.add(indexRequestBuilder);
                    }
                }

                if (bulkRequest.numberOfActions() > 0) {
                    final BulkResponse bulkResponse = bulkRequest.execute().actionGet(esBulkCallTimeout);
                    log.info("Bulk saving metadata of index {} type {} with size {}.",
                        index, type, docs.size());
                    if (bulkResponse.hasFailures()) {
                        for (BulkItemResponse item : bulkResponse.getItems()) {
                            if (item.isFailed()) {
                                handleException("ElasticSearchUtil.bulkSaveToIndex.index", type, item.getId(),
                                    item.getFailure().getCause(), Metrics.CounterElasticSearchSave.getMetricName());
                            }
                        }
                    }
                }
                return null;
            });
        } catch (Exception e) {
            final List<String> docIds = docs.stream().map(ElasticSearchDoc::getId).collect(Collectors.toList());
            handleException("ElasticSearchUtil.bulkSaveToIndex", type, docIds, e,
                Metrics.CounterElasticSearchBulkSave.getMetricName());
        }
    }
}
 
Example #27
Source File: EsDoc.java    From AsuraFramework with Apache License 2.0 5 votes vote down vote up
private void handBulkResponseException(BulkResponse bulkResponse){
    if (bulkResponse.hasFailures()) {
        long count = 0L;
        for (BulkItemResponse bulkItemResponse : bulkResponse) {
            logger.error("发生错误的 索引id为 : "+bulkItemResponse.getId()+" ,错误信息为:"+ bulkItemResponse.getFailureMessage());
            count++;
        }
        logger.error("发生错误的 总个数为 : "+count);
    }
}
 
Example #28
Source File: Elasticsearch7SearchIndex.java    From vertexium with Apache License 2.0 5 votes vote down vote up
public void handleBulkFailure(BulkItem<?> bulkItem, BulkItemResponse bulkItemResponse, AtomicBoolean retry) throws Exception {
    if (exceptionHandler == null) {
        LOGGER.error("bulk failure: %s: %s", bulkItem, bulkItemResponse.getFailureMessage());
        return;
    }
    if (LOGGER.isTraceEnabled()) {
        LOGGER.trace("handleBulkFailure: (bulkItem: %s, bulkItemResponse: %s)", bulkItem, bulkItemResponse);
    }
    exceptionHandler.handleBulkFailure(graph, this, bulkItem, bulkItemResponse, retry);
}
 
Example #29
Source File: ElasticsearchPersistWriter.java    From streams with Apache License 2.0 5 votes vote down vote up
private void updateTotals(final BulkResponse bulkItemResponses, final Long sent, final Long sizeInBytes) {
  long failed = 0;
  long passed = 0;
  long millis = bulkItemResponses.getTookInMillis();

  // keep track of the number of totalFailed and items that we have totalOk.
  for (BulkItemResponse resp : bulkItemResponses.getItems()) {
    if (resp == null || resp.isFailed()) {
      failed++;
      LOGGER.debug("{} ({},{},{}) failed: {}", resp.getOpType(), resp.getIndex(), resp.getType(), resp.getId(), resp.getFailureMessage());
    } else {
      passed++;
    }
  }

  if (failed > 0) {
    LOGGER.warn("Bulk Uploading had {} failures of {}", failed, sent);
  }

  this.totalOk.addAndGet(passed);
  this.totalFailed.addAndGet(failed);
  this.totalSeconds.addAndGet(millis / 1000);
  this.totalSizeInBytes.addAndGet(sizeInBytes);

  if (sent != (passed + failed)) {
    LOGGER.error("Count MisMatch: Sent[{}] Passed[{}] Failed[{}]", sent, passed, failed);
  }

  LOGGER.debug("Batch[{}mb {} items with {} failures in {}ms] - Total[{}mb {} items with {} failures in {}seconds] {} outstanding]",
      MEGABYTE_FORMAT.format(sizeInBytes / (double) (1024 * 1024)), NUMBER_FORMAT.format(passed), NUMBER_FORMAT.format(failed), NUMBER_FORMAT.format(millis),
      MEGABYTE_FORMAT.format((double) totalSizeInBytes.get() / (double) (1024 * 1024)), NUMBER_FORMAT.format(totalOk), NUMBER_FORMAT.format(totalFailed), NUMBER_FORMAT.format(totalSeconds), NUMBER_FORMAT.format(getTotalOutstanding()));
}
 
Example #30
Source File: ElasticSearchUtilImpl.java    From metacat with Apache License 2.0 5 votes vote down vote up
private void softDeleteDoc(
    final String type,
    final List<String> ids,
    final MetacatRequestContext metacatRequestContext) {
    try {
        RETRY_ES_PUBLISH.call(() -> {
            final BulkRequestBuilder bulkRequest = client.prepareBulk();
            final XContentBuilder builder = XContentFactory.contentBuilder(contentType);
            builder.startObject().field(ElasticSearchDoc.Field.DELETED, true)
                .field(ElasticSearchDoc.Field.TIMESTAMP, java.time.Instant.now().toEpochMilli())
                .field(ElasticSearchDoc.Field.USER, metacatRequestContext.getUserName()).endObject();
            ids.forEach(id -> bulkRequest.add(client.prepareUpdate(esIndex, type, id)
                .setRetryOnConflict(NO_OF_CONFLICT_RETRIES).setDoc(builder)));
            final BulkResponse bulkResponse = bulkRequest.execute().actionGet(esBulkCallTimeout);
            if (bulkResponse.hasFailures()) {
                for (BulkItemResponse item : bulkResponse.getItems()) {
                    if (item.isFailed()) {
                        handleException("ElasticSearchUtil.bulkSoftDelete.item", type, item.getId(),
                            item.getFailure().getCause(), Metrics.CounterElasticSearchDelete.getMetricName());
                    }
                }
            }
            return null;
        });
    } catch (Exception e) {
        handleException("ElasticSearchUtil.bulkSoftDelete", type, ids, e,
            Metrics.CounterElasticSearchBulkDelete.getMetricName());
    }
}