Java Code Examples for org.elasticsearch.action.bulk.BulkItemResponse#getFailure()
The following examples show how to use
org.elasticsearch.action.bulk.BulkItemResponse#getFailure() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: Output.java From data-generator with Apache License 2.0 | 6 votes |
|
private static void writeBatchToES(String index, String type, List<Map<String, Object>> list) throws Exception{
if(list.isEmpty()){
return;
}
BulkRequest request = new BulkRequest();
for(Map<String, Object> data : list) {
String id = data.get("id").toString();
request.add(
new IndexRequest(index, type, id)
.source(data));
}
BulkResponse bulkResponse = CLIENT.bulk(request);
if (bulkResponse.hasFailures()) {
for (BulkItemResponse bulkItemResponse : bulkResponse) {
if (bulkItemResponse.isFailed()) {
BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
LOGGER.error("ES索引失败: {}", failure.getMessage());
}
}
}
}
Example 2
| Source File: BulkUpdateService.java From vertexium with Apache License 2.0 | 6 votes |
|
private void handleFailure(BulkItem<?> bulkItem, BulkItemResponse bulkItemResponse) {
BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
bulkItem.incrementFailCount();
if (bulkItem.getFailCount() >= maxFailCount) {
complete(bulkItem, new BulkVertexiumException("fail count exceeded the max number of failures", failure));
} else {
AtomicBoolean retry = new AtomicBoolean(false);
try {
searchIndex.handleBulkFailure(bulkItem, bulkItemResponse, retry);
} catch (Exception ex) {
complete(bulkItem, ex);
return;
}
if (retry.get()) {
incomingItems.addAll(bulkItem.getItems());
} else {
complete(bulkItem, null);
}
}
}
Example 3
| Source File: BulkUpdateService.java From vertexium with Apache License 2.0 | 6 votes |
|
private void handleFailure(BulkItem<?> bulkItem, BulkItemResponse bulkItemResponse) {
BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
bulkItem.incrementFailCount();
if (bulkItem.getFailCount() >= maxFailCount) {
complete(bulkItem, new BulkVertexiumException("fail count exceeded the max number of failures", failure));
} else {
AtomicBoolean retry = new AtomicBoolean(false);
try {
searchIndex.handleBulkFailure(bulkItem, bulkItemResponse, retry);
} catch (Exception ex) {
complete(bulkItem, ex);
return;
}
if (retry.get()) {
incomingItems.addAll(bulkItem.getItems());
} else {
complete(bulkItem, null);
}
}
}
Example 4
| Source File: ElasticIndexer.java From bluima with Apache License 2.0 | 5 votes |
|
private void flushBulk() {
BulkResponse bulkResponse = bulkRequest.execute().actionGet(); // flush
if (bulkResponse.hasFailures()) { // log failures
for (BulkItemResponse r : bulkResponse.getItems()) {
LOG.error(r.getFailureMessage());
Failure failure = r.getFailure();
// e.g. when ES server is overloaded
throw new ElasticsearchException(failure.toString());
}
}
}
Example 5
| Source File: FessEsClient.java From fess with Apache License 2.0 | 5 votes |
|
public void addAll(final String index, final List<Map<String, Object>> docList,
final BiConsumer<Map<String, Object>, IndexRequestBuilder> options) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
for (final Map<String, Object> doc : docList) {
final Object id = doc.remove(fessConfig.getIndexFieldId());
final IndexRequestBuilder builder = client.prepareIndex().setIndex(index).setId(id.toString()).setSource(new DocMap(doc));
options.accept(doc, builder);
bulkRequestBuilder.add(builder);
}
final BulkResponse response = bulkRequestBuilder.execute().actionGet(ComponentUtil.getFessConfig().getIndexBulkTimeout());
if (response.hasFailures()) {
if (logger.isDebugEnabled()) {
final List<DocWriteRequest<?>> requests = bulkRequestBuilder.request().requests();
final BulkItemResponse[] items = response.getItems();
if (requests.size() == items.length) {
for (int i = 0; i < requests.size(); i++) {
final BulkItemResponse resp = items[i];
if (resp.isFailed() && resp.getFailure() != null) {
final DocWriteRequest<?> req = requests.get(i);
final Failure failure = resp.getFailure();
logger.debug("Failed Request: {}\n=>{}", req, failure.getMessage());
}
}
}
}
throw new FessEsClientException(response.buildFailureMessage());
}
}
Example 6
| Source File: StatusUpdaterBolt.java From storm-crawler with Apache License 2.0 | 4 votes |
|
@Override
public void afterBulk(long executionId, BulkRequest request,
BulkResponse response) {
LOG.debug("afterBulk [{}] with {} responses", executionId,
request.numberOfActions());
long msec = response.getTook().getMillis();
eventCounter.scope("bulks_received").incrBy(1);
eventCounter.scope("bulk_msec").incrBy(msec);
Iterator<BulkItemResponse> bulkitemiterator = response.iterator();
int itemcount = 0;
int acked = 0;
int failurecount = 0;
synchronized (waitAck) {
while (bulkitemiterator.hasNext()) {
BulkItemResponse bir = bulkitemiterator.next();
itemcount++;
String id = bir.getId();
BulkItemResponse.Failure f = bir.getFailure();
boolean failed = false;
if (f != null) {
// already discovered
if (f.getStatus().equals(RestStatus.CONFLICT)) {
eventCounter.scope("doc_conflicts").incrBy(1);
LOG.debug("Doc conflict ID {}", id);
} else {
LOG.error("Update ID {}, failure: {}", id, f);
failed = true;
}
}
List<Tuple> xx = waitAck.getIfPresent(id);
if (xx != null) {
LOG.debug("Acked {} tuple(s) for ID {}", xx.size(), id);
for (Tuple x : xx) {
if (!failed) {
String url = x.getStringByField("url");
acked++;
// ack and put in cache
LOG.debug("Acked {} with ID {}", url, id);
super.ack(x, url);
} else {
failurecount++;
_collector.fail(x);
}
}
waitAck.invalidate(id);
} else {
LOG.warn("Could not find unacked tuple for {}", id);
}
}
LOG.info(
"Bulk response [{}] : items {}, waitAck {}, acked {}, failed {}",
executionId, itemcount, waitAck.size(), acked, failurecount);
if (waitAck.size() > 0 && LOG.isDebugEnabled()) {
for (String kinaw : waitAck.asMap().keySet()) {
LOG.debug(
"Still in wait ack after bulk response [{}] => {}",
executionId, kinaw);
}
}
}
}
Example 7
| Source File: IndexerBolt.java From storm-crawler with Apache License 2.0 | 4 votes |
|
@Override
public void afterBulk(long executionId, BulkRequest request,
BulkResponse response) {
long msec = response.getTook().getMillis();
eventCounter.scope("bulks_received").incrBy(1);
eventCounter.scope("bulk_msec").incrBy(msec);
Iterator<BulkItemResponse> bulkitemiterator = response.iterator();
int itemcount = 0;
int acked = 0;
int failurecount = 0;
synchronized (waitAck) {
while (bulkitemiterator.hasNext()) {
BulkItemResponse bir = bulkitemiterator.next();
itemcount++;
String id = bir.getId();
BulkItemResponse.Failure f = bir.getFailure();
boolean failed = false;
if (f != null) {
if (f.getStatus().equals(RestStatus.CONFLICT)) {
eventCounter.scope("doc_conflicts").incrBy(1);
} else {
failed = true;
}
}
Tuple t = waitAck.getIfPresent(id);
if (t == null) {
LOG.warn("Could not find unacked tuple for {}", id);
continue;
}
LOG.debug("Acked tuple for ID {}", id);
String u = (String) t.getValueByField("url");
Metadata metadata = (Metadata) t.getValueByField("metadata");
if (!failed) {
acked++;
_collector.emit(StatusStreamName, t, new Values(u,
metadata, Status.FETCHED));
_collector.ack(t);
} else {
failurecount++;
LOG.error("update ID {}, URL {}, failure: {}", id, u, f);
// there is something wrong with the content we should treat
// it as an ERROR
if (f.getStatus().equals(RestStatus.BAD_REQUEST)) {
metadata.setValue(Constants.STATUS_ERROR_SOURCE,
"ES indexing");
metadata.setValue(Constants.STATUS_ERROR_MESSAGE,
"invalid content");
_collector.emit(StatusStreamName, t, new Values(u,
metadata, Status.ERROR));
_collector.ack(t);
}
// otherwise just fail it
else {
_collector.fail(t);
}
}
waitAck.invalidate(id);
}
LOG.info(
"Bulk response [{}] : items {}, waitAck {}, acked {}, failed {}",
executionId, itemcount, waitAck.size(), acked, failurecount);
if (waitAck.size() > 0 && LOG.isDebugEnabled()) {
for (String kinaw : waitAck.asMap().keySet()) {
LOG.debug(
"Still in wait ack after bulk response [{}] => {}",
executionId, kinaw);
}
}
}
}
