org.elasticsearch.action.bulk.BulkItemResponse#isFailed

Source File: KafkaRiver.java From elasticsearch-river-kafka with Apache License 2.0

6 votes

void executeBuilder(BulkRequestBuilder bulkRequestBuilder)
{
	if(bulkRequestBuilder.numberOfActions() == 0)
		return;
	
	++stats.flushes;
	BulkResponse response = bulkRequestBuilder.execute().actionGet();
	if (response.hasFailures()) {
              logger.warn("failed to execute" + response.buildFailureMessage());
          }
	
	for(BulkItemResponse resp : response){
		if(resp.isFailed()){
			stats.failed++;
		}else{
			stats.succeeded++;
		}
	}
}

Source File: Output.java From data-generator with Apache License 2.0

6 votes

private static void writeBatchToES(String index, String type, List<Map<String, Object>> list) throws Exception{
    if(list.isEmpty()){
        return;
    }
    BulkRequest request = new BulkRequest();
    for(Map<String, Object> data : list) {
        String id = data.get("id").toString();
        request.add(
                new IndexRequest(index, type, id)
                        .source(data));

    }
    BulkResponse bulkResponse = CLIENT.bulk(request);
    if (bulkResponse.hasFailures()) {
        for (BulkItemResponse bulkItemResponse : bulkResponse) {
            if (bulkItemResponse.isFailed()) {
                BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
                LOGGER.error("ES索引失败: {}", failure.getMessage());
            }
        }
    }
}

Source File: ESTest.java From canal-1.1.3 with Apache License 2.0

6 votes

private void commit(BulkRequestBuilder bulkRequestBuilder) {
    if (bulkRequestBuilder.numberOfActions() > 0) {
        BulkResponse response = bulkRequestBuilder.execute().actionGet();
        if (response.hasFailures()) {
            for (BulkItemResponse itemResponse : response.getItems()) {
                if (!itemResponse.isFailed()) {
                    continue;
                }

                if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) {
                    System.out.println(itemResponse.getFailureMessage());
                } else {
                    System.out.println("ES bulk commit error" + itemResponse.getFailureMessage());
                }
            }
        }
    }
}

Source File: ImportBulkListener.java From elasticsearch-inout-plugin with Apache License 2.0

6 votes

@Override
public void afterBulk(long executionId, BulkRequest request,
        BulkResponse response) {
    bulksInProgress.decrementAndGet();
    if (response.hasFailures()) {
        for (BulkItemResponse item : response.getItems()) {
            if (item.isFailed()) {
                counts.failures++;
            } else {
                counts.successes++;
            }
        }
    } else {
        counts.successes += response.getItems().length;
    }
    checkRelease();
}

Source File: ESTemplate.java From canal-1.1.3 with Apache License 2.0

6 votes

/**
 * 提交批次
 */
public void commit() {
    if (getBulk().numberOfActions() > 0) {
        BulkResponse response = getBulk().execute().actionGet();
        if (response.hasFailures()) {
            for (BulkItemResponse itemResponse : response.getItems()) {
                if (!itemResponse.isFailed()) {
                    continue;
                }

                if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) {
                    logger.error(itemResponse.getFailureMessage());
                } else {
                    throw new RuntimeException("ES sync commit error" + itemResponse.getFailureMessage());
                }
            }
        }
    }
}

Source File: ElasticsearchSinkBaseTest.java From flink with Apache License 2.0

5 votes

@Nullable
@Override
public Throwable extractFailureCauseFromBulkItemResponse(BulkItemResponse bulkItemResponse) {
	if (bulkItemResponse.isFailed()) {
		return new Exception(bulkItemResponse.getFailure().getMessage());
	} else {
		return null;
	}
}

Source File: BaseElasticSearchIndexBuilder.java From sakai with Educational Community License v2.0

5 votes

protected void executeBulkRequest(BulkRequestBuilder bulkRequest) {
    BulkResponse bulkResponse = bulkRequest.execute().actionGet();

    getLog().info("Bulk request of batch size: " + bulkRequest.numberOfActions() + " took "
            + bulkResponse.getTookInMillis() + " ms in index builder: " + getName());

    for (BulkItemResponse response : bulkResponse.getItems()) {
        if (response.getResponse() instanceof DeleteResponse) {
            DeleteResponse deleteResponse = response.getResponse();

            if (response.isFailed()) {
                getLog().error("Problem deleting doc: " + response.getId() + " in index builder: " + getName()
                        + " error: " + response.getFailureMessage());
            } else if (!deleteResponse.isFound()) {
                getLog().debug("ES could not find a doc with id: " + deleteResponse.getId()
                        + " to delete in index builder: " + getName());
            } else {
                getLog().debug("ES deleted a doc with id: " + deleteResponse.getId() + " in index builder: "
                        + getName());
            }
        } else if (response.getResponse() instanceof IndexResponse) {
            IndexResponse indexResponse = response.getResponse();

            if (response.isFailed()) {
                getLog().error("Problem updating content for doc: " + response.getId() + " in index builder: "
                        + getName() + " error: " + response.getFailureMessage());
            } else {
                getLog().debug("ES indexed content for doc with id: " + indexResponse.getId()
                        + " in index builder: " + getName());
            }
        }
    }
}

Source File: ElasticsearchChannel.java From syncer with BSD 3-Clause "New" or "Revised" License

5 votes

@Override
public void retryFailed(List<SyncWrapper<WriteRequest>> aim, Throwable e) {
  BulkItemResponse[] items = ((ElasticsearchBulkException) e).getBulkItemResponses().getItems();
  LinkedList<SyncWrapper<WriteRequest>> tmp = new LinkedList<>();
  for (int i = 0; i < aim.size(); i++) {
    BulkItemResponse item = items[i];
    SyncWrapper<WriteRequest> wrapper = aim.get(i);
    if (!item.isFailed()) {
      ack.remove(wrapper.getSourceId(), wrapper.getSyncDataId());
      continue;
    }

    // failed item handle
    ErrorLevel level = level(e, wrapper, batchConfig.getMaxRetry());
    if (level.retriable()) {
      logger.info("Retry {} because {}", requestStr(wrapper), item.getFailure());
      handle404(wrapper, item);
      tmp.add(wrapper);
    } else {
      logger.error("Met {} in {} because {}", level, wrapper, item.getFailure());
      switch (level) {
        case MAX_TRY_EXCEED:
        case SYNCER_BUG:
        case WARN:
          singleRequest.log(wrapper.getEvent(), item.getFailure().toString());
          ack.remove(wrapper.getSourceId(), wrapper.getSyncDataId());
          break;
      }

    }
  }
  batchBuffer.addAllInHead(tmp);
}

Source File: Elasticsearch6ApiCallBridge.java From flink with Apache License 2.0

5 votes

@Override
public Throwable extractFailureCauseFromBulkItemResponse(BulkItemResponse bulkItemResponse) {
	if (!bulkItemResponse.isFailed()) {
		return null;
	} else {
		return bulkItemResponse.getFailure().getCause();
	}
}

Source File: ElasticsearchIndexer.java From datashare with GNU Affero General Public License v3.0

5 votes

@Override
public boolean bulkAdd(final String indexName, Pipeline.Type nerType, List<NamedEntity> namedEntities, Document parent) throws IOException {
    BulkRequest bulkRequest = new BulkRequest();

    String routing = ofNullable(parent.getRootDocument()).orElse(parent.getId());
    bulkRequest.add(new UpdateRequest(indexName, esCfg.indexType, parent.getId()).doc(
            jsonBuilder().startObject()
                    .field("status", Document.Status.DONE)
                    .endObject()).routing(routing));
    bulkRequest.add(new UpdateRequest(indexName, esCfg.indexType, parent.getId())
            .script(new Script(ScriptType.INLINE, "painless",
                    "if (!ctx._source.nerTags.contains(params.nerTag)) ctx._source.nerTags.add(params.nerTag);",
                    new HashMap<String, Object>() {{put("nerTag", nerType.toString());}})).routing(routing));

    for (Entity child : namedEntities) {
        bulkRequest.add(createIndexRequest(indexName, JsonObjectMapper.getType(child), child.getId(),
                getJson(child), parent.getId(), routing));
    }
    bulkRequest.setRefreshPolicy(esCfg.refreshPolicy);

    BulkResponse bulkResponse = client.bulk(bulkRequest);
    if (bulkResponse.hasFailures()) {
        for (BulkItemResponse resp : bulkResponse.getItems()) {
            if (resp.isFailed()) {
                LOGGER.error("bulk add failed : {}", resp.getFailureMessage());
            }
        }
        return false;
    }
    return true;
}

Source File: Elasticsearch7ApiCallBridge.java From flink with Apache License 2.0

5 votes

@Override
public Throwable extractFailureCauseFromBulkItemResponse(BulkItemResponse bulkItemResponse) {
	if (!bulkItemResponse.isFailed()) {
		return null;
	} else {
		return bulkItemResponse.getFailure().getCause();
	}
}

Source File: Elasticsearch5ApiCallBridge.java From flink with Apache License 2.0

5 votes

@Override
public Throwable extractFailureCauseFromBulkItemResponse(BulkItemResponse bulkItemResponse) {
	if (!bulkItemResponse.isFailed()) {
		return null;
	} else {
		return bulkItemResponse.getFailure().getCause();
	}
}

Source File: Elasticsearch5ApiCallBridge.java From flink with Apache License 2.0

5 votes

@Override
public Throwable extractFailureCauseFromBulkItemResponse(BulkItemResponse bulkItemResponse) {
	if (!bulkItemResponse.isFailed()) {
		return null;
	} else {
		return bulkItemResponse.getFailure().getCause();
	}
}

Source File: DefaultElasticSearchService.java From jetlinks-community with Apache License 2.0

5 votes

@SneakyThrows
protected void checkResponse(BulkResponse response) {
    if (response.hasFailures()) {
        for (BulkItemResponse item : response.getItems()) {
            if (item.isFailed()) {
                throw item.getFailure().getCause();
            }
        }
    }
}

Source File: ElasticsearchSinkBaseTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Nullable
@Override
public Throwable extractFailureCauseFromBulkItemResponse(BulkItemResponse bulkItemResponse) {
	if (bulkItemResponse.isFailed()) {
		return new Exception(bulkItemResponse.getFailure().getMessage());
	} else {
		return null;
	}
}

Source File: Elasticsearch6ApiCallBridge.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public Throwable extractFailureCauseFromBulkItemResponse(BulkItemResponse bulkItemResponse) {
	if (!bulkItemResponse.isFailed()) {
		return null;
	} else {
		return bulkItemResponse.getFailure().getCause();
	}
}

Source File: ESConnection.java From canal with Apache License 2.0

5 votes

@Override
public void processFailBulkResponse(String errorMsg) {
    for (BulkItemResponse itemResponse : bulkResponse.getItems()) {
        if (!itemResponse.isFailed()) {
            continue;
        }

        if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) {
            logger.error(itemResponse.getFailureMessage());
        } else {
            throw new RuntimeException(errorMsg + itemResponse.getFailureMessage());
        }
    }
}

Source File: Elasticsearch1ApiCallBridge.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public Throwable extractFailureCauseFromBulkItemResponse(BulkItemResponse bulkItemResponse) {
	if (!bulkItemResponse.isFailed()) {
		return null;
	} else {
		return new RuntimeException(bulkItemResponse.getFailureMessage());
	}
}

Source File: ESConnector.java From Siamese with GNU General Public License v3.0

4 votes

/***
	 * A method for bulk insertion of documents
	 * @param index the index name
	 * @param type the doc type name
	 * @param documents array of documents
	 * @return status of bulk insert (true = no failure, false = failures)
	 */
    public boolean bulkInsert(String index, String type, ArrayList<Document> documents) {

		BulkRequestBuilder bulkRequest = client.prepareBulk();

		// keep adding documents
		for (Document d : documents) {
			try {
				bulkRequest.add(client.prepareIndex(index, type)
						.setSource(jsonBuilder().startObject()
								.field("id", d.getId())
								.field("file", d.getFile())
								.field("startline", d.getStartLine())
								.field("endline", d.getEndLine())
								.field("src", d.getSource())
								.field("t2src", d.getT2Source())
								.field("t1src", d.getT1Source())
                                .field("tokenizedsrc", d.getTokenizedSource())
								.field("origsrc", d.getOriginalSource())
								.field("license", d.getLicense())
								.field("url", d.getUrl())
                            .endObject()));
			} catch (IOException e) {
//				e.printStackTrace();
				System.out.println(e.getMessage());
				return false;
			}
		}

		// bulk insert once
		BulkResponse bulkResponse = bulkRequest.execute().actionGet();
        if (bulkResponse.hasFailures()) {
            for (BulkItemResponse brsp : bulkResponse) {
                if (brsp.isFailed()) {
                    System.out.println("Failed to index (message: " + brsp.getFailureMessage() + ")");
                }
            }
        }

        return !bulkResponse.hasFailures();
	}

Source File: ElasticSearchAuditDestination.java From ranger with Apache License 2.0

4 votes

@Override
public boolean log(Collection<AuditEventBase> events) {
    boolean ret = false;
    try {
        logStatusIfRequired();
        addTotalCount(events.size());

        RestHighLevelClient client = getClient();
        if (null == client) {
            // ElasticSearch is still not initialized. So need return error
            addDeferredCount(events.size());
            return ret;
        }

        ArrayList<AuditEventBase> eventList = new ArrayList<>(events);
        BulkRequest bulkRequest = new BulkRequest();
        try {
            for (AuditEventBase event : eventList) {
                AuthzAuditEvent authzEvent = (AuthzAuditEvent) event;
                String id = authzEvent.getEventId();
                Map<String, Object> doc = toDoc(authzEvent);
                bulkRequest.add(new IndexRequest(index).id(id).source(doc).type(""));
            }
        } catch (Exception ex) {
            addFailedCount(eventList.size());
            logFailedEvent(eventList, ex);
        }
        BulkResponse response = client.bulk(bulkRequest, RequestOptions.DEFAULT);
        if (response.status().getStatus() >= 400) {
            addFailedCount(eventList.size());
            logFailedEvent(eventList, "HTTP " + response.status().getStatus());
        } else {
            BulkItemResponse[] items = response.getItems();
            for (int i = 0; i < items.length; i++) {
                AuditEventBase itemRequest = eventList.get(i);
                BulkItemResponse itemResponse = items[i];
                if (itemResponse.isFailed()) {
                    addFailedCount(1);
                    logFailedEvent(Arrays.asList(itemRequest), itemResponse.getFailureMessage());
                } else {
                    if(LOG.isDebugEnabled()) {
                        LOG.debug(String.format("Indexed %s", itemRequest.getEventKey()));
                    }
                    addSuccessCount(1);
                    ret = true;
                }
            }
        }
    } catch (Throwable t) {
        addDeferredCount(events.size());
        logError("Error sending message to ElasticSearch", t);
    }
    return ret;
}

Java Code Examples for org.elasticsearch.action.bulk.BulkItemResponse#isFailed()