org.elasticsearch.action.bulk.BulkRequestBuilder#numberOfActions

Source File: CassandraRiver.java From cassandra-river with Apache License 2.0

6 votes

@Override
public void run() {
	logger.info("Starting thread with {} keys", this.keys.rowColumnMap.size());
              if (closed) {
                  return;
              }
              
              BulkRequestBuilder bulk = client.prepareBulk();
              for(String key : this.keys.rowColumnMap.keySet()){
              	
  				try {
  					String id = UUID.nameUUIDFromBytes(key.getBytes()).toString();
  					bulk.add(indexRequest(this.indexName).type(this.typeName)
					    							 .id(id)
					    							 .source(this.keys.rowColumnMap.get(key)));
  				} catch (Exception e) {
  					logger.error("failed to entry to bulk indexing");
  				}
  				
              	if(bulk.numberOfActions() >= this.batchSize){
              		saveToEs(bulk);
  					bulk = client.prepareBulk();
              	}
              }
}

Source File: ESIndexState.java From storm-trident-elasticsearch with Apache License 2.0

6 votes

public void bulkUpdateIndices(List<TridentTuple> inputs, TridentTupleMapper<Document<T>> mapper, BulkResponseHandler handler) {
    BulkRequestBuilder bulkRequest = client.prepareBulk();
    for (TridentTuple input : inputs) {
        Document<T> doc = mapper.map(input);
        byte[] source = serializeSourceOrFail(doc);
        IndexRequestBuilder request = client.prepareIndex(doc.getName(), doc.getType(), doc.getId()).setSource(source);

        if(doc.getParentId() != null) {
            request.setParent(doc.getParentId());
        }
        bulkRequest.add(request);
    }

    if( bulkRequest.numberOfActions() > 0) {
        try {
            handler.handle(bulkRequest.execute().actionGet());
        } catch(ElasticsearchException e) {
            LOGGER.error("error while executing bulk request to elasticsearch");
            throw new FailedException("Failed to store data into elasticsearch", e);
        }
    }
}

Source File: ESTest.java From canal-1.1.3 with Apache License 2.0

6 votes

private void commit(BulkRequestBuilder bulkRequestBuilder) {
    if (bulkRequestBuilder.numberOfActions() > 0) {
        BulkResponse response = bulkRequestBuilder.execute().actionGet();
        if (response.hasFailures()) {
            for (BulkItemResponse itemResponse : response.getItems()) {
                if (!itemResponse.isFailed()) {
                    continue;
                }

                if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) {
                    System.out.println(itemResponse.getFailureMessage());
                } else {
                    System.out.println("ES bulk commit error" + itemResponse.getFailureMessage());
                }
            }
        }
    }
}

Source File: ESUpdateState.java From sql4es with Apache License 2.0

6 votes

/**
 * Executes the list with requests as a bulk with maximum number of requests per bulk
 * @param requests
 * @param maxRequestsPerBulk
 * @return
 * @throws SQLException
 */
private int execute(List<?> requests, int maxRequestsPerBulk) throws SQLException{
	int result = 0;
	BulkRequestBuilder bulkReq = client.prepareBulk();
	for(Object req : requests){
		if(req instanceof IndexRequest)	bulkReq.add((IndexRequest)req);
		else if(req instanceof UpdateRequest) bulkReq.add((UpdateRequest)req);
		else if(req instanceof DeleteRequest) bulkReq.add((DeleteRequest)req);
		else if(req instanceof IndexRequestBuilder) bulkReq.add((IndexRequestBuilder)req);
		else if(req instanceof UpdateRequestBuilder) bulkReq.add((UpdateRequestBuilder)req);
		else if(req instanceof DeleteRequestBuilder) bulkReq.add((DeleteRequestBuilder)req);
		else throw new SQLException("Type "+req.getClass()+" cannot be added to a bulk request");
		
		if(bulkReq.numberOfActions() > maxRequestsPerBulk){
			result += bulkReq.get().getItems().length;
			bulkReq = client.prepareBulk();
		}
	}
	
	if(bulkReq.numberOfActions() > 0){
		result += bulkReq.get().getItems().length;
	}
	return result;
}

Source File: KafkaRiver.java From elasticsearch-river-kafka with Apache License 2.0

6 votes

void executeBuilder(BulkRequestBuilder bulkRequestBuilder)
{
	if(bulkRequestBuilder.numberOfActions() == 0)
		return;
	
	++stats.flushes;
	BulkResponse response = bulkRequestBuilder.execute().actionGet();
	if (response.hasFailures()) {
              logger.warn("failed to execute" + response.buildFailureMessage());
          }
	
	for(BulkItemResponse resp : response){
		if(resp.isFailed()){
			stats.failed++;
		}else{
			stats.succeeded++;
		}
	}
}

Source File: IndexingComponent.java From elasticsearch-reindex-tool with Apache License 2.0

5 votes

private Optional<BulkResult> executeBulk(int indexedCount, BulkRequestBuilder bulkRequest) {
  if (bulkRequest.numberOfActions() > 0) {
    BulkResponse bulkItemResponses = bulkRequest.execute().actionGet();
    Set<String> failedIds = Stream.of(bulkItemResponses.getItems())
        .filter(BulkItemResponse::isFailed)
        .map(BulkItemResponse::getId)
        .collect(Collectors.toSet());
    return Optional.of(new BulkResult(indexedCount, failedIds));
  }
  return Optional.empty();
}

Source File: SearchUpdateJobImpl.java From stash-codesearch-plugin with Apache License 2.0

5 votes

@Override
public void doReindex(Client client, GitScm gitScm, GlobalSettings globalSettings) {
    if (!globalSettings.getIndexingEnabled()) {
        return;
    }
    deleteLatestIndexedNote(client);
    while (true) {
        try {
            SearchRequestBuilder esReq = client.prepareSearch(ES_UPDATEALIAS)
                .setSize(1000)
                .setFetchSource(false)
                .setRouting(getRepoDesc())
                .setQuery(filteredQuery(matchAllQuery(), andFilter(
                    sfu.projectRepositoryFilter(
                        repository.getProject().getKey(), repository.getSlug()),
                    sfu.exactRefFilter(ref))));
            BulkRequestBuilder bulkDelete = client.prepareBulk().setRefresh(true);
            for (SearchHit hit : esReq.get().getHits().getHits()) {
                bulkDelete.add(buildDeleteFromRef(client, hit.getType(), hit.getId()));
            }
            if (bulkDelete.numberOfActions() == 0) {
                break;
            }
            bulkDelete.get();
        } catch (Exception e) {
            log.error("Could not delete documents for {}, aborting", toString(), e);
            return;
        }
    }
    doUpdate(client, gitScm, globalSettings);
}

Source File: IndexBatchBolt.java From storm-trident-elasticsearch with Apache License 2.0

5 votes

protected void bulkUpdateIndexes( ) {

        List<Tuple> inputs = new ArrayList<>(queue.size());
        queue.drainTo(inputs);
        BulkRequestBuilder bulkRequest = client.prepareBulk();
        for (Tuple input : inputs) {
            Document<T> doc = mapper.map(input);
            IndexRequestBuilder request = client.prepareIndex(doc.getName(), doc.getType(), doc.getId()).setSource((String)doc.getSource());

            if(doc.getParentId() != null) {
                request.setParent(doc.getParentId());
            }
            bulkRequest.add(request);
        }

        try {
            if (bulkRequest.numberOfActions() > 0) {
                BulkResponse bulkItemResponses = bulkRequest.execute().actionGet();
                if (bulkItemResponses.hasFailures()) {
                    BulkItemResponse[] items = bulkItemResponses.getItems();
                    for (int i = 0; i < items.length; i++) {
                        ackOrFail(items[i], inputs.get(i));
                    }
                } else {
                    ackAll(inputs);
                }
            }
        } catch (ElasticsearchException e) {
            LOGGER.error("Unable to process bulk request, " + inputs.size() + " tuples are in failure", e);
            outputCollector.reportError(e.getRootCause());
            failAll(inputs);
        }
    }