Java Code Examples for org.elasticsearch.action.bulk.BulkItemResponse#Failure

The following examples show how to use org.elasticsearch.action.bulk.BulkItemResponse#Failure . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestPutElasticsearch.java    From localization_nifi with Apache License 2.0 7 votes vote down vote up 
@Override
public BulkResponse get() throws InterruptedException, ExecutionException {
    BulkResponse response = mock(BulkResponse.class);
    when(response.hasFailures()).thenReturn(responseHasFailures);
    BulkItemResponse item1 = mock(BulkItemResponse.class);
    BulkItemResponse item2 = mock(BulkItemResponse.class);
    when(item1.getItemId()).thenReturn(1);
    when(item1.isFailed()).thenReturn(true);
    BulkItemResponse.Failure failure = mock(BulkItemResponse.Failure.class);
    when(failure.getMessage()).thenReturn("Bad message");
    when(item1.getFailure()).thenReturn(failure);
    when(item2.getItemId()).thenReturn(2);
    when(item2.isFailed()).thenReturn(false);
    when(response.getItems()).thenReturn(new BulkItemResponse[]{item1, item2});
    return response;
}
Example 2
Source File: Output.java    From data-generator with Apache License 2.0 6 votes vote down vote up 
private static void writeBatchToES(String index, String type, List<Map<String, Object>> list) throws Exception{
    if(list.isEmpty()){
        return;
    }
    BulkRequest request = new BulkRequest();
    for(Map<String, Object> data : list) {
        String id = data.get("id").toString();
        request.add(
                new IndexRequest(index, type, id)
                        .source(data));

    }
    BulkResponse bulkResponse = CLIENT.bulk(request);
    if (bulkResponse.hasFailures()) {
        for (BulkItemResponse bulkItemResponse : bulkResponse) {
            if (bulkItemResponse.isFailed()) {
                BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
                LOGGER.error("ES索引失败: {}", failure.getMessage());
            }
        }
    }
}
Example 3
Source File: BulkUpdateService.java    From vertexium with Apache License 2.0 6 votes vote down vote up 
private void handleFailure(BulkItem<?> bulkItem, BulkItemResponse bulkItemResponse) {
    BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
    bulkItem.incrementFailCount();
    if (bulkItem.getFailCount() >= maxFailCount) {
        complete(bulkItem, new BulkVertexiumException("fail count exceeded the max number of failures", failure));
    } else {
        AtomicBoolean retry = new AtomicBoolean(false);
        try {
            searchIndex.handleBulkFailure(bulkItem, bulkItemResponse, retry);
        } catch (Exception ex) {
            complete(bulkItem, ex);
            return;
        }
        if (retry.get()) {
            incomingItems.addAll(bulkItem.getItems());
        } else {
            complete(bulkItem, null);
        }
    }
}
Example 4
Source File: BulkUpdateService.java    From vertexium with Apache License 2.0 6 votes vote down vote up 
private void handleFailure(BulkItem<?> bulkItem, BulkItemResponse bulkItemResponse) {
    BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
    bulkItem.incrementFailCount();
    if (bulkItem.getFailCount() >= maxFailCount) {
        complete(bulkItem, new BulkVertexiumException("fail count exceeded the max number of failures", failure));
    } else {
        AtomicBoolean retry = new AtomicBoolean(false);
        try {
            searchIndex.handleBulkFailure(bulkItem, bulkItemResponse, retry);
        } catch (Exception ex) {
            complete(bulkItem, ex);
            return;
        }
        if (retry.get()) {
            incomingItems.addAll(bulkItem.getItems());
        } else {
            complete(bulkItem, null);
        }
    }
}
Example 5
Source File: TestPutElasticsearch.java    From nifi with Apache License 2.0 6 votes vote down vote up 
@Override
public BulkResponse get() throws InterruptedException, ExecutionException {
    BulkResponse response = mock(BulkResponse.class);
    when(response.hasFailures()).thenReturn(responseHasFailures);
    BulkItemResponse item1 = mock(BulkItemResponse.class);
    BulkItemResponse item2 = mock(BulkItemResponse.class);
    when(item1.getItemId()).thenReturn(1);
    when(item1.isFailed()).thenReturn(true);
    BulkItemResponse.Failure failure = mock(BulkItemResponse.Failure.class);
    when(failure.getMessage()).thenReturn("Bad message");
    when(item1.getFailure()).thenReturn(failure);
    when(item2.getItemId()).thenReturn(2);
    when(item2.isFailed()).thenReturn(false);
    when(response.getItems()).thenReturn(new BulkItemResponse[]{item1, item2});
    return response;
}
Example 6
Source File: ElasticsearchSystemProducerTest.java    From samza with Apache License 2.0 5 votes vote down vote up 
private BulkResponse getRespWithFailedDocument(RestStatus status) {
  BulkResponse response = mock(BulkResponse.class);
  when(response.hasFailures()).thenReturn(true);

  BulkItemResponse itemResp = mock(BulkItemResponse.class);
  when(itemResp.isFailed()).thenReturn(true);
  BulkItemResponse.Failure failure = mock(BulkItemResponse.Failure.class);
  when(failure.getStatus()).thenReturn(status);
  when(itemResp.getFailure()).thenReturn(failure);
  BulkItemResponse[] itemResponses = new BulkItemResponse[]{itemResp};

  when(response.getItems()).thenReturn(itemResponses);

  return response;
}
Example 7
Source File: ElasticsearchBulkDocumentWriterTest.java    From metron with Apache License 2.0 5 votes vote down vote up 
private void setupElasticsearchToFail() throws IOException {
    final String errorMessage = "error message";
    final Exception cause = new Exception("test exception");
    final boolean isFailed = true;
    final int itemID = 0;

    // define the item failure
    BulkItemResponse.Failure failure = mock(BulkItemResponse.Failure.class);
    when(failure.getCause()).thenReturn(cause);
    when(failure.getMessage()).thenReturn(errorMessage);

    // define the item level response
    BulkItemResponse itemResponse = mock(BulkItemResponse.class);
    when(itemResponse.isFailed()).thenReturn(isFailed);
    when(itemResponse.getItemId()).thenReturn(itemID);

    when(itemResponse.getFailure()).thenReturn(failure);
    when(itemResponse.getFailureMessage()).thenReturn("error message");
    List<BulkItemResponse> itemsResponses = Collections.singletonList(itemResponse);

    // define the bulk response to indicate failure
    BulkResponse response = mock(BulkResponse.class);
    when(response.iterator()).thenReturn(itemsResponses.iterator());
    when(response.hasFailures()).thenReturn(isFailed);

    // have the client return the mock response
    when(highLevelClient.bulk(any(BulkRequest.class))).thenReturn(response);
}
Example 8
Source File: EsHighLevelRestTest2.java    From java-study with Apache License 2.0 4 votes vote down vote up 
/**
	 * 索引复制
	 * 
	 * @throws IOException
	 */
	private static void reindex() throws IOException {
		// 创建索引复制请求并进行索引复制
		ReindexRequest request = new ReindexRequest();
		// 需要复制的索引
		request.setSourceIndices("user");
		// 复制的目标索引
		request.setDestIndex("dest_test");

		// 表示如果在复制索引的时候有缺失的文档的话会进行创建,默认是index
		request.setDestOpType("create");
		// 如果在复制的过程中发现版本冲突，那么会继续进行复制
		request.setConflicts("proceed");

		// 只复制文档类型为 userindex 的数据
		request.setSourceDocTypes("userindex");
		// 只复制 pancm 用户的数据
		request.setSourceQuery(new TermQueryBuilder("user", "pancm"));
		// 设置复制文档的数量
		request.setSize(10);
		// 设置一次批量处理的条数，默认是1000
		request.setSourceBatchSize(100);

		// 进行排序
//		request.addSortField("postDate", SortOrder.DESC);

		// 指定切片大小
		request.setSlices(2);
		
		//设置超时时间
		request.setTimeout(TimeValue.timeValueMinutes(2));
		//允许刷新
		request.setRefresh(true);
		
		// 同步执行
		BulkByScrollResponse bulkResponse = client.reindex(request, RequestOptions.DEFAULT);

		// 异步执行
//		client.reindexAsync(request, RequestOptions.DEFAULT, listener);

		// 响应结果处理

		TimeValue timeTaken = bulkResponse.getTook();
		boolean timedOut = bulkResponse.isTimedOut();
		long totalDocs = bulkResponse.getTotal();
		long updatedDocs = bulkResponse.getUpdated();
		long createdDocs = bulkResponse.getCreated();
		long deletedDocs = bulkResponse.getDeleted();
		long batches = bulkResponse.getBatches();
		long noops = bulkResponse.getNoops();
		long versionConflicts = bulkResponse.getVersionConflicts();
		long bulkRetries = bulkResponse.getBulkRetries();
		long searchRetries = bulkResponse.getSearchRetries();
		TimeValue throttledMillis = bulkResponse.getStatus().getThrottled();
		TimeValue throttledUntilMillis = bulkResponse.getStatus().getThrottledUntil();
		List<ScrollableHitSource.SearchFailure> searchFailures = bulkResponse.getSearchFailures();
		List<BulkItemResponse.Failure> bulkFailures = bulkResponse.getBulkFailures();

		System.out.println("索引复制总共花费了:" + timeTaken.getMillis() + " 毫秒，总条数:" + totalDocs + ",创建数:" + createdDocs
				+ ",更新数:" + updatedDocs);
	}
Example 9
Source File: EsHighLevelRestTest2.java    From java-study with Apache License 2.0 4 votes vote down vote up 
/**
	 * 根据查询条件删除
	 * @throws IOException
	 */
	private static void deleteByQuery() throws IOException {
		//
		DeleteByQueryRequest request = new DeleteByQueryRequest("user");

		// 设置查询条件
		request.setQuery(new TermQueryBuilder("user", "pancm"));

		// 设置复制文档的数量
		request.setSize(10);
		// 设置一次批量处理的条数，默认是1000
		request.setBatchSize(100);
		//设置路由
		request.setRouting("=cat");
		//设置超时时间
		request.setTimeout(TimeValue.timeValueMinutes(2));
		//允许刷新
		request.setRefresh(true);
		//索引选项
		request.setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN);
		
		// 同步执行
		BulkByScrollResponse bulkResponse = client.deleteByQuery(request, RequestOptions.DEFAULT);

		// 异步执行
//		client.updateByQueryAsync(request, RequestOptions.DEFAULT, listener); 

		// 返回结果
		TimeValue timeTaken = bulkResponse.getTook();
		boolean timedOut = bulkResponse.isTimedOut();
		long totalDocs = bulkResponse.getTotal();
		long deletedDocs = bulkResponse.getDeleted();
		long batches = bulkResponse.getBatches();
		long noops = bulkResponse.getNoops();
		long versionConflicts = bulkResponse.getVersionConflicts();
		long bulkRetries = bulkResponse.getBulkRetries();
		long searchRetries = bulkResponse.getSearchRetries();
		TimeValue throttledMillis = bulkResponse.getStatus().getThrottled();
		TimeValue throttledUntilMillis = bulkResponse.getStatus().getThrottledUntil();
		List<ScrollableHitSource.SearchFailure> searchFailures = bulkResponse.getSearchFailures();
		List<BulkItemResponse.Failure> bulkFailures = bulkResponse.getBulkFailures();
		System.out.println("查询更新总共花费了:" + timeTaken.getMillis() + " 毫秒，总条数:" + totalDocs + ",删除数:" + deletedDocs);

	}
Example 10
Source File: BulkVertexiumException.java    From vertexium with Apache License 2.0 4 votes vote down vote up 
public BulkVertexiumException(String message, BulkItemResponse.Failure failure) {
    super(message + ":" + failure.getMessage());
    this.failure = failure;
}
Example 11
Source File: BulkVertexiumException.java    From vertexium with Apache License 2.0 4 votes vote down vote up 
public BulkItemResponse.Failure getFailure() {
    return failure;
}
Example 12
Source File: BulkVertexiumException.java    From vertexium with Apache License 2.0 4 votes vote down vote up 
public BulkVertexiumException(String message, BulkItemResponse.Failure failure) {
    super(message + ":" + failure.getMessage());
    this.failure = failure;
}
Example 13
Source File: BulkVertexiumException.java    From vertexium with Apache License 2.0 4 votes vote down vote up 
public BulkItemResponse.Failure getFailure() {
    return failure;
}
Example 14
Source File: StatusUpdaterBolt.java    From storm-crawler with Apache License 2.0 4 votes vote down vote up 
@Override
public void afterBulk(long executionId, BulkRequest request,
        BulkResponse response) {
    LOG.debug("afterBulk [{}] with {} responses", executionId,
            request.numberOfActions());
    long msec = response.getTook().getMillis();
    eventCounter.scope("bulks_received").incrBy(1);
    eventCounter.scope("bulk_msec").incrBy(msec);
    Iterator<BulkItemResponse> bulkitemiterator = response.iterator();
    int itemcount = 0;
    int acked = 0;
    int failurecount = 0;

    synchronized (waitAck) {
        while (bulkitemiterator.hasNext()) {
            BulkItemResponse bir = bulkitemiterator.next();
            itemcount++;
            String id = bir.getId();
            BulkItemResponse.Failure f = bir.getFailure();
            boolean failed = false;
            if (f != null) {
                // already discovered
                if (f.getStatus().equals(RestStatus.CONFLICT)) {
                    eventCounter.scope("doc_conflicts").incrBy(1);
                    LOG.debug("Doc conflict ID {}", id);
                } else {
                    LOG.error("Update ID {}, failure: {}", id, f);
                    failed = true;
                }
            }
            List<Tuple> xx = waitAck.getIfPresent(id);
            if (xx != null) {
                LOG.debug("Acked {} tuple(s) for ID {}", xx.size(), id);
                for (Tuple x : xx) {
                    if (!failed) {
                        String url = x.getStringByField("url");
                        acked++;
                        // ack and put in cache
                        LOG.debug("Acked {} with ID {}", url, id);
                        super.ack(x, url);
                    } else {
                        failurecount++;
                        _collector.fail(x);
                    }
                }
                waitAck.invalidate(id);
            } else {
                LOG.warn("Could not find unacked tuple for {}", id);
            }
        }

        LOG.info(
                "Bulk response [{}] : items {}, waitAck {}, acked {}, failed {}",
                executionId, itemcount, waitAck.size(), acked, failurecount);
        if (waitAck.size() > 0 && LOG.isDebugEnabled()) {
            for (String kinaw : waitAck.asMap().keySet()) {
                LOG.debug(
                        "Still in wait ack after bulk response [{}] => {}",
                        executionId, kinaw);
            }
        }
    }
}
Example 15
Source File: IndexerBolt.java    From storm-crawler with Apache License 2.0 4 votes vote down vote up 
@Override
public void afterBulk(long executionId, BulkRequest request,
        BulkResponse response) {
    long msec = response.getTook().getMillis();
    eventCounter.scope("bulks_received").incrBy(1);
    eventCounter.scope("bulk_msec").incrBy(msec);
    Iterator<BulkItemResponse> bulkitemiterator = response.iterator();
    int itemcount = 0;
    int acked = 0;
    int failurecount = 0;

    synchronized (waitAck) {
        while (bulkitemiterator.hasNext()) {
            BulkItemResponse bir = bulkitemiterator.next();
            itemcount++;
            String id = bir.getId();
            BulkItemResponse.Failure f = bir.getFailure();
            boolean failed = false;
            if (f != null) {
                if (f.getStatus().equals(RestStatus.CONFLICT)) {
                    eventCounter.scope("doc_conflicts").incrBy(1);
                } else {
                    failed = true;
                }
            }
            Tuple t = waitAck.getIfPresent(id);
            if (t == null) {
                LOG.warn("Could not find unacked tuple for {}", id);
                continue;
            }

            LOG.debug("Acked  tuple for ID {}", id);
            String u = (String) t.getValueByField("url");

            Metadata metadata = (Metadata) t.getValueByField("metadata");

            if (!failed) {
                acked++;
                _collector.emit(StatusStreamName, t, new Values(u,
                        metadata, Status.FETCHED));
                _collector.ack(t);
            } else {
                failurecount++;
                LOG.error("update ID {}, URL {}, failure: {}", id, u, f);
                // there is something wrong with the content we should treat
                // it as an ERROR
                if (f.getStatus().equals(RestStatus.BAD_REQUEST)) {
                    metadata.setValue(Constants.STATUS_ERROR_SOURCE,
                            "ES indexing");
                    metadata.setValue(Constants.STATUS_ERROR_MESSAGE,
                            "invalid content");
                    _collector.emit(StatusStreamName, t, new Values(u,
                            metadata, Status.ERROR));
                    _collector.ack(t);
                }
                // otherwise just fail it
                else {
                    _collector.fail(t);
                }
            }
            waitAck.invalidate(id);
        }

        LOG.info(
                "Bulk response [{}] : items {}, waitAck {}, acked {}, failed {}",
                executionId, itemcount, waitAck.size(), acked, failurecount);

        if (waitAck.size() > 0 && LOG.isDebugEnabled()) {
            for (String kinaw : waitAck.asMap().keySet()) {
                LOG.debug(
                        "Still in wait ack after bulk response [{}] => {}",
                        executionId, kinaw);
            }
        }
    }
}