Java Code Examples for org.elasticsearch.action.bulk.BulkRequestBuilder#get()
The following examples show how to use
org.elasticsearch.action.bulk.BulkRequestBuilder#get() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: Elasticsearch2Client.java From presto-connectors with Apache License 2.0 | 6 votes |
|
@Override
public void insertMany(List<Document> docs)
{
final BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
for (Document doc : docs) {
bulkRequestBuilder.add(new IndexRequest()
.index(doc.getIndex())
.type(doc.getType())
.id(doc.getId())
.source(doc.getSource()));
}
BulkResponse response = bulkRequestBuilder.get();
if (response.hasFailures()) {
throw new PrestoException(IO_ERROR, response.buildFailureMessage());
}
}
Example 2
| Source File: ElasticSearch.java From hsweb-learning with Apache License 2.0 | 6 votes |
|
private static void BulkIndex(Client client) throws IOException {
BulkRequestBuilder requestBuilder = client.prepareBulk();
requestBuilder.add(client.prepareIndex("twitter","tweet","4")
.setSource(jsonBuilder()
.startObject()
.field("user","niekaijie")
.field("school","beiyou")
.field("address","haidianqu")
.endObject())
);
requestBuilder.add(client.prepareIndex("twitter","tweet","3")
.setSource(jsonBuilder()
.startObject()
.field("user","林志颖aa")
.field("school","台湾大学")
.field("address","台湾")
.endObject())
);
BulkResponse response = requestBuilder.get();
}
Example 3
| Source File: TestTransportClient.java From jframe with Apache License 2.0 | 6 votes |
|
@Test
public void testBulk() throws Exception {
BulkRequestBuilder bulkRequest = client.prepareBulk();
// either use client#prepare, or use Requests# to directly build
// index/delete requests
bulkRequest.add(client.prepareIndex("twitter", "tweet", "1").setSource(XContentFactory.jsonBuilder().startObject().field("user", "kimchy")
.field("postDate", new Date()).field("message", "trying out Elasticsearch").endObject()));
bulkRequest.add(client.prepareIndex("twitter", "tweet", "2").setSource(XContentFactory.jsonBuilder().startObject().field("user", "kimchy")
.field("postDate", new Date()).field("message", "another post").endObject()));
BulkResponse bulkResponse = bulkRequest.get();
if (bulkResponse.hasFailures()) {
// process failures by iterating through each bulk response item
}
}
Example 4
| Source File: BaseDemo.java From Elasticsearch-Tutorial-zh-CN with GNU General Public License v3.0 | 6 votes |
|
/**
* 批量删除
*
* @param transportClient
*/
private static void batchDelete(TransportClient transportClient) throws IOException {
BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();
DeleteRequestBuilder deleteRequestBuilder1 = transportClient.prepareDelete("product_index", "product", "1");
DeleteRequestBuilder deleteRequestBuilder2 = transportClient.prepareDelete("product_index", "product", "2");
DeleteRequestBuilder deleteRequestBuilder3 = transportClient.prepareDelete("product_index", "product", "3");
bulkRequestBuilder.add(deleteRequestBuilder1);
bulkRequestBuilder.add(deleteRequestBuilder2);
bulkRequestBuilder.add(deleteRequestBuilder3);
BulkResponse bulkResponse = bulkRequestBuilder.get();
for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) {
logger.info("--------------------------------version= " + bulkItemResponse.getVersion());
}
}
Example 5
| Source File: BaseDemo.java From Elasticsearch-Tutorial-zh-CN with GNU General Public License v3.0 | 5 votes |
|
/**
* 批量更新
*
* @param transportClient
*/
private static void batchUpdate(TransportClient transportClient) throws IOException {
BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();
UpdateRequestBuilder updateRequestBuilder1 = transportClient.prepareUpdate("product_index", "product", "1")
.setDoc(XContentFactory.jsonBuilder()
.startObject()
.field("product_name", "更新后的商品名称1")
.endObject());
UpdateRequestBuilder updateRequestBuilder2 = transportClient.prepareUpdate("product_index", "product", "2")
.setDoc(XContentFactory.jsonBuilder()
.startObject()
.field("product_name", "更新后的商品名称2")
.endObject());
UpdateRequestBuilder updateRequestBuilder3 = transportClient.prepareUpdate("product_index", "product", "3")
.setDoc(XContentFactory.jsonBuilder()
.startObject()
.field("product_name", "更新后的商品名称3")
.endObject());
bulkRequestBuilder.add(updateRequestBuilder1);
bulkRequestBuilder.add(updateRequestBuilder2);
bulkRequestBuilder.add(updateRequestBuilder3);
BulkResponse bulkResponse = bulkRequestBuilder.get();
for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) {
logger.info("--------------------------------version= " + bulkItemResponse.getVersion());
}
}
Example 6
| Source File: BulkDemo.java From elasticsearch-full with Apache License 2.0 | 5 votes |
|
@Test
public void testForClient() throws Exception {
BulkRequestBuilder bulkRequest = client.prepareBulk();
// either use client#prepare, or use Requests# to directly build index/delete requests
bulkRequest.add(client.prepareIndex("twitter", "tweet", "1")
.setSource(XContentFactory.jsonBuilder()
.startObject()
.field("user", "kimchy")
.field("postDate", new Date())
.field("message", "trying out Elasticsearch")
.endObject()
)
);
bulkRequest.add(client.prepareIndex("twitter", "tweet", "2")
.setSource(XContentFactory.jsonBuilder()
.startObject()
.field("user", "kimchy")
.field("postDate", new Date())
.field("message", "another post")
.endObject()
)
);
BulkResponse bulkResponse = bulkRequest.get();
if (bulkResponse.hasFailures()) {
// process failures by iterating through each bulk response item
}
}
Example 7
| Source File: MainTestSuite.java From elasticsearch-sql with Apache License 2.0 | 5 votes |
|
/**
* Loads all data from the json into the test
* elasticsearch cluster, using TEST_INDEX
* @param jsonPath the json file represents the bulk
* @param defaultIndex
* @throws Exception
*/
public static void loadBulk(String jsonPath, String defaultIndex) throws Exception {
System.out.println(String.format("Loading file %s into elasticsearch cluster", jsonPath));
BulkRequestBuilder bulkBuilder = client.prepareBulk();
byte[] buffer = ByteStreams.toByteArray(new FileInputStream(jsonPath));
bulkBuilder.add(buffer, 0, buffer.length, defaultIndex, XContentType.JSON);
BulkResponse response = bulkBuilder.get();
if(response.hasFailures()) {
throw new Exception(String.format("Failed during bulk load of file %s. failure message: %s", jsonPath, response.buildFailureMessage()));
}
}
Example 8
| Source File: InvokeChainDataCollectHandler.java From uavstack with Apache License 2.0 | 5 votes |
|
@Override
public void handle(CollectDataFrame frame) {
if (this.log.isDebugEnable()) {
this.log.debug(this, frame.toJSONString());
}
String appUUID = frame.getTarget();
// cm.beginBatch();
BulkRequestBuilder bulkRequest = client.getClient().prepareBulk();
for (Line line : frame.getLines()) {
try {
String content = line.getContent();
Span span = new Span(content);
pushLatestIVCDataToCache(appUUID, span);
pushSpanToBulkRequest(appUUID, frame.getAppgroup(), span, bulkRequest);
}
catch (Exception e) {
this.log.err(this, "unsupported ivc content :" + line.getContent(), e);
}
}
// cm.submitBatch();
BulkResponse bulkResponse = bulkRequest.get();
if (bulkResponse.hasFailures()) {
log.err(this, "INSERT InvokeChain Data to ES FAIL: " + bulkResponse.buildFailureMessage());
}
}
Example 9
| Source File: EsDoc.java From AsuraFramework with Apache License 2.0 | 5 votes |
|
/**
* 批量save
* @param indexName
* @param indexType
* @param list
* @return
*/
private BulkResponse bulkSaveOrUpdate(String indexName, String indexType, List<DocPo> list){
TransportClient client = esClientFactory.getClient();
BulkResponse bulkResponse=null;
BulkRequestBuilder bulkRequest = client.prepareBulk();
for(int i=0;i<list.size();i++){
bulkRequest.add(new IndexRequest(esClientFactory.getIndexs(indexName), indexType,list.get(i).getId()).source(JSON.toJSONString(list.get(i).getObj())));
}
bulkResponse = bulkRequest.get();
// 处理错误信息
handBulkResponseException(bulkResponse);
return bulkResponse;
}
Example 10
| Source File: CommonWebpageDAO.java From Gather-Platform with GNU General Public License v3.0 | 5 votes |
|
/**
* 批量更新网页
*
* @param webpageList 网页列表
* @return
*/
public boolean update(List<Webpage> webpageList) throws ExecutionException, InterruptedException {
BulkRequestBuilder bulkRequest = client.prepareBulk();
for (Webpage webpage : webpageList) {
UpdateRequest updateRequest = new UpdateRequest();
updateRequest.index(INDEX_NAME);
updateRequest.type(TYPE_NAME);
updateRequest.id(webpage.getId());
updateRequest.doc(gson.toJson(webpage));
bulkRequest.add(updateRequest);
}
BulkResponse bulkResponse = bulkRequest.get();
return bulkResponse.hasFailures();
}
Example 11
| Source File: EkmKnowledgeCollectRepositoryImpl.java From youkefu with Apache License 2.0 | 5 votes |
|
public void delete(List<EkmKnowledgeCollect> ekmKnowledgeCollectList){
BulkRequestBuilder bulkRequest = elasticsearchTemplate.getClient().prepareBulk();
if(ekmKnowledgeCollectList != null && ekmKnowledgeCollectList.size()>0) {
for(EkmKnowledgeCollect collect : ekmKnowledgeCollectList){
bulkRequest.add(new DeleteRequest().index(UKDataContext.SYSTEM_INDEX).type("uk_ekm_kb_collect").id(collect.getId()).routing(collect.getKbid()));
}
bulkRequest.get();
}
}
Example 12
| Source File: ElasticSearchBulkIndexStrategy.java From camunda-bpm-elasticsearch with Apache License 2.0 | 5 votes |
|
@Override
public void executeRequest(List<HistoryEvent> historyEvents) {
BulkRequestBuilder bulkRequestBuilder = esClient.prepareBulk();
try {
for (HistoryEvent historyEvent : historyEvents) {
UpdateRequestBuilder updateRequestBuilder = prepareUpdateRequest(historyEvent);
bulkRequestBuilder.add(updateRequestBuilder);
}
BulkResponse bulkResponse;
if (WAIT_FOR_RESPONSE > 0) {
bulkResponse = bulkRequestBuilder.get(TimeValue.timeValueSeconds(WAIT_FOR_RESPONSE));
} else {
bulkResponse = bulkRequestBuilder.get();
}
if (bulkResponse.hasFailures()) {
LOGGER.severe("Error while executing bulk request: " + bulkResponse.buildFailureMessage());
}
if (LOGGER.isLoggable(Level.FINEST)) {
for (BulkItemResponse bulkItemResponse : bulkResponse) {
LOGGER.finest("[" + bulkItemResponse.getIndex() +
"][" + bulkItemResponse.getType() +
"][" + bulkItemResponse.getOpType() +
"] process instance with id '" + bulkItemResponse.getId() + "'");
}
}
} catch (IOException e) {
LOGGER.log(Level.SEVERE, e.getMessage(), e);
}
}
Example 13
| Source File: BaseDemo.java From Elasticsearch-Tutorial-zh-CN with GNU General Public License v3.0 | 4 votes |
|
/**
* 批量创建
*
* @param transportClient
*/
private static void batchCreate(TransportClient transportClient) throws IOException {
BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();
IndexRequestBuilder indexRequestBuilder1 = transportClient.prepareIndex("product_index", "product", "1")
.setSource(XContentFactory.jsonBuilder()
.startObject()
.field("product_name", "飞利浦电动牙刷 HX6700-1")
.field("product_desc", "前 1000 名赠刷头，6 月 1 日 0 点火爆开抢，618 开门红巅峰 48 小时")
.field("price", 399.00)
.field("created_date_time", new SimpleDateFormat("yyyyMMdd'T'HHmmss.SSSZ").format(new Date()))
.field("last_modified_date_time", new SimpleDateFormat("yyyyMMdd'T'HHmmss.SSSZ").format(new Date()))
.field("version", 1)
.endObject());
IndexRequestBuilder indexRequestBuilder2 = transportClient.prepareIndex("product_index", "product", "2")
.setSource(XContentFactory.jsonBuilder()
.startObject()
.field("product_name", "飞利浦电动牙刷 HX6700-2")
.field("product_desc", "前 1000 名赠刷头，6 月 1 日 0 点火爆开抢，618 开门红巅峰 48 小时")
.field("price", 399.00)
.field("created_date_time", new SimpleDateFormat("yyyyMMdd'T'HHmmss.SSSZ").format(new Date()))
.field("last_modified_date_time", new SimpleDateFormat("yyyyMMdd'T'HHmmss.SSSZ").format(new Date()))
.field("version", 1)
.endObject());
IndexRequestBuilder indexRequestBuilder3 = transportClient.prepareIndex("product_index", "product", "3")
.setSource(XContentFactory.jsonBuilder()
.startObject()
.field("product_name", "飞利浦电动牙刷 HX6700-3")
.field("product_desc", "前 1000 名赠刷头，6 月 1 日 0 点火爆开抢，618 开门红巅峰 48 小时")
.field("price", 399.00)
.field("created_date_time", new SimpleDateFormat("yyyyMMdd'T'HHmmss.SSSZ").format(new Date()))
.field("last_modified_date_time", new SimpleDateFormat("yyyyMMdd'T'HHmmss.SSSZ").format(new Date()))
.field("version", 1)
.endObject());
bulkRequestBuilder.add(indexRequestBuilder1);
bulkRequestBuilder.add(indexRequestBuilder2);
bulkRequestBuilder.add(indexRequestBuilder3);
BulkResponse bulkResponse = bulkRequestBuilder.get();
for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) {
logger.info("--------------------------------version= " + bulkItemResponse.getVersion());
}
}
Example 14
| Source File: SearchItemServiceImpl.java From BigDataPlatform with GNU General Public License v3.0 | 4 votes |
|
@Override
public int importAllItems() {
try{
Settings settings = Settings.builder()
.put("cluster.name", ES_CLUSTER_NAME).build();
TransportClient client = new PreBuiltTransportClient(settings)
.addTransportAddress(new TransportAddress(InetAddress.getByName(ES_CONNECT_IP), 9300));
//批量添加
BulkRequestBuilder bulkRequest = client.prepareBulk();
//查询商品列表
List<SearchItem> itemList = itemMapper.getItemList();
//遍历商品列表
for (SearchItem searchItem : itemList) {
String image=searchItem.getProductImageBig();
if (image != null && !"".equals(image)) {
String[] strings = image.split(",");
image=strings[0];
}else{
image="";
}
searchItem.setProductImageBig(image);
bulkRequest.add(client.prepareIndex(ITEM_INDEX, ITEM_TYPE, String.valueOf(searchItem.getProductId()))
.setSource(jsonBuilder()
.startObject()
.field("productId", searchItem.getProductId())
.field("salePrice", searchItem.getSalePrice())
.field("productName", searchItem.getProductName())
.field("subTitle", searchItem.getSubTitle())
.field("productImageBig", searchItem.getProductImageBig())
.field("categoryName", searchItem.getCategoryName())
.field("cid", searchItem.getCid())
.endObject()
)
);
}
BulkResponse bulkResponse = bulkRequest.get();
log.info("更新索引成功");
client.close();
}catch (Exception e){
e.printStackTrace();
throw new XmallException("导入ES索引库出错，请再次尝试");
}
return 1;
}
Example 15
| Source File: ElasticsearchGis.java From Elasticsearch-Tutorial-zh-CN with GNU General Public License v3.0 | 4 votes |
|
/**
* 批量创建
* 资料：http://blog.csdn.net/loveisnull/article/details/45914115
* 纬度在前，经度在后
*
* @param transportClient
*/
private static void batchCreate(TransportClient transportClient) throws IOException {
BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();
IndexRequestBuilder indexRequestBuilder1 = transportClient.prepareIndex("shop_index", "shop", "1")
.setSource(XContentFactory.jsonBuilder()
.startObject()
.field("shop_name", "麻辣香锅1")
.latlon("location", 40.12, -71.34)
.field("created_date_time", new SimpleDateFormat("yyyyMMdd'T'HHmmss.SSSZ").format(new Date()))
.field("last_modified_date_time", new SimpleDateFormat("yyyyMMdd'T'HHmmss.SSSZ").format(new Date()))
.field("version", 1)
.endObject());
IndexRequestBuilder indexRequestBuilder2 = transportClient.prepareIndex("shop_index", "shop", "2")
.setSource(XContentFactory.jsonBuilder()
.startObject()
.field("shop_name", "麻辣香锅2")
.latlon("location", 40.12, -72.34)
.field("created_date_time", new SimpleDateFormat("yyyyMMdd'T'HHmmss.SSSZ").format(new Date()))
.field("last_modified_date_time", new SimpleDateFormat("yyyyMMdd'T'HHmmss.SSSZ").format(new Date()))
.field("version", 1)
.endObject());
IndexRequestBuilder indexRequestBuilder3 = transportClient.prepareIndex("shop_index", "shop", "3")
.setSource(XContentFactory.jsonBuilder()
.startObject()
.field("shop_name", "麻辣香锅3")
.latlon("location", 40.12, -73.34)
.field("created_date_time", new SimpleDateFormat("yyyyMMdd'T'HHmmss.SSSZ").format(new Date()))
.field("last_modified_date_time", new SimpleDateFormat("yyyyMMdd'T'HHmmss.SSSZ").format(new Date()))
.field("version", 1)
.endObject());
bulkRequestBuilder.add(indexRequestBuilder1);
bulkRequestBuilder.add(indexRequestBuilder2);
bulkRequestBuilder.add(indexRequestBuilder3);
BulkResponse bulkResponse = bulkRequestBuilder.get();
for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) {
logger.info("--------------------------------version= " + bulkItemResponse.getVersion());
}
}
Example 16
| Source File: IDAO.java From Gather-Platform with GNU General Public License v3.0 | 4 votes |
|
/**
* 根据query删除数据
*
* @param queryBuilder query
* @param task 任务实体
* @return 是否全部数据删除成功
*/
protected boolean deleteByQuery(QueryBuilder queryBuilder, Task task) {
BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
SearchRequestBuilder searchRequestBuilder = client.prepareSearch(INDEX_NAME)
.setTypes(TYPE_NAME)
.setQuery(queryBuilder)
.setSize(100)
.setScroll(TimeValue.timeValueMinutes(SCROLL_TIMEOUT));
SearchResponse response = searchRequestBuilder.execute().actionGet();
while (true) {
for (SearchHit hit : response.getHits()) {
bulkRequestBuilder.add(new DeleteRequest(INDEX_NAME, TYPE_NAME, hit.id()));
if (task != null) {
task.increaseCount();
}
}
response = client.prepareSearchScroll(response.getScrollId())
.setScroll(TimeValue.timeValueMinutes(SCROLL_TIMEOUT))
.execute().actionGet();
if (response.getHits().getHits().length == 0) {
if (task != null) {
task.setDescription("按query%s删除数据ID添加完毕,已经添加%s条,准备执行删除", queryBuilder.toString(), bulkRequestBuilder.numberOfActions());
}
LOG.debug("按query{}删除数据ID添加完毕,准备执行删除", queryBuilder.toString());
break;
} else {
if (task != null) {
task.setDescription("按query%s删除数据已经添加%s条", queryBuilder.toString(), bulkRequestBuilder.numberOfActions());
}
LOG.debug("按query{}删除数据已经添加{}条", queryBuilder.toString(), bulkRequestBuilder.numberOfActions());
}
}
if (bulkRequestBuilder.numberOfActions() <= 0) {
if (task != null) {
task.setDescription("按query%s删除数据时未找到数据,请检查参数", queryBuilder.toString());
}
LOG.debug("按query{}删除数据时未找到数据,请检查参数", queryBuilder.toString());
return false;
}
BulkResponse bulkResponse = bulkRequestBuilder.get();
if (bulkResponse.hasFailures()) {
if (task != null) {
task.setDescription("按query%s删除数据部分失败,%s", queryBuilder.toString(), bulkResponse.buildFailureMessage());
}
LOG.error("按query{}删除数据部分失败,{}", queryBuilder.toString(), bulkResponse.buildFailureMessage());
} else {
if (task != null) {
task.setDescription("按query%s删除数据成功,耗时:%s毫秒", queryBuilder.toString(), bulkResponse.getTookInMillis());
}
LOG.info("按query{}删除数据成功,耗时:{}毫秒", queryBuilder.toString(), bulkResponse.getTookInMillis());
}
return bulkResponse.hasFailures();
}
Example 17
| Source File: SlowOperDataCollectHandler.java From uavstack with Apache License 2.0 | 4 votes |
|
@Override
public void handle(CollectDataFrame frame) {
if (this.log.isDebugEnable()) {
this.log.debug(this, frame.toJSONString());
}
BulkRequestBuilder bulkRequest = client.getClient().prepareBulk();
String appUUID = frame.getTarget();
// 从uuid中获取appid
String appid = appUUID.split("---")[1];
for (Line line : frame.getLines()) {
String content = line.getContent();
try {
// 提取epinfo
StringBuilder builder = new StringBuilder();
int headPoint = 0;
for (int i = 0; i < content.length(); i++) {
char item = content.charAt(i);
if (item == ';') {
headPoint++;
if (headPoint == 3) {
break;
}
}
else {
if (headPoint > 1) {
builder.append(item);
}
}
}
String epinfo = builder.toString();
// 若epinfo为数字则说明为方法级
if (DataConvertHelper.toInt(epinfo, -1) != -1) {
epinfo = "method";
}
IActionEngine engine = this.getActionEngineMgr().getActionEngine("SlowOperActionEngine");
ActionContext ac = new ActionContext();
ac.putParam("content", content);
ac.putParam("appid", appid);
engine.execute(epinfo, ac);
SlowOperSpan span = (SlowOperSpan) ac.getParam("span");
pushSpanToBulkRequest(appUUID, frame.getAppgroup(), span, bulkRequest,
(String) ac.getParam("protocolType"));
}
catch (Exception e) {
// 防止有不合法的协议报文出现
this.log.err(this, "unsupported protocol,content is" + frame.toJSONString(), e);
}
}
BulkResponse bulkResponse = bulkRequest.get();
if (bulkResponse.hasFailures()) {
log.err(this, "INSERT InvokeChain Data to ES FAIL: " + bulkResponse.buildFailureMessage());
}
}
Example 18
| Source File: NewLogDataMessageHandler.java From uavstack with Apache License 2.0 | 4 votes |
|
/**
* storeToES
*
* @param msg
*/
@SuppressWarnings({ "unchecked", "rawtypes" })
private void storeToES(Message msg) {
String data = msg.getParam(getMsgTypeName());
List<String> array = JSONHelper.toObjectArray(data, String.class);
for (String mdfStr : array) {
// 反序列化为MonitorDataFrame
MonitorDataFrame mdf = new MonitorDataFrame(mdfStr);
/**
* 获取IP和端口，这样唯一性的标识
*/
String ipport = mdf.getIP();
String appurl = mdf.getExt("appurl");
if (!StringHelper.isEmpty(appurl)) {
ipport = appurl.split("/")[2];
}
Map<String, List<Map>> frames = mdf.getDatas();
for (String appid : frames.keySet()) {
BulkRequestBuilder bulkRequest = client.getClient().prepareBulk();
List<Map> applogs = frames.get(appid);
for (Map applog : applogs) {
List<Map> instances = (List<Map>) applog.get("Instances");
for (Map logData : instances) {
// push to ES BulkRequest
pushLogLineToBulkRequest(mdf, appid, ipport, bulkRequest, logData);
}
}
BulkResponse bulkResponse = bulkRequest.get();
if (bulkResponse.hasFailures()) {
log.err(this, "INSERT App[" + appid + "][" + mdf.getIP() + "] on " + mdf.getServerId()
+ " Log Data to ES FAIL: " + bulkResponse.buildFailureMessage());
}
}
}
}
Example 19
| Source File: ElasticsearchDataStructure.java From rdf4j with BSD 3-Clause "New" or "Revised" License | 4 votes |
|
synchronized private void flushRemoveStatementBuffer() {
if (deleteStatementBuffer.isEmpty()) {
return;
}
BulkRequestBuilder bulkRequest = clientProvider.getClient().prepareBulk();
int failures = 0;
do {
deleteStatementBuffer.forEach(statement -> {
bulkRequest.add(clientProvider.getClient()
.prepareDelete(index, ELASTICSEARCH_TYPE, statement.getElasticsearchId()));
});
BulkResponse bulkResponse = bulkRequest.get();
if (bulkResponse.hasFailures()) {
failures++;
if (failures < 10) {
logger.warn("Elasticsearch has failures when adding data, retrying. Message: {}",
bulkResponse.buildFailureMessage());
} else {
throw new RuntimeException("Elasticsearch has failed " + failures
+ " times when adding data, retrying. Message: " + bulkResponse.buildFailureMessage());
}
} else {
failures = 0;
}
} while (failures > 0);
logger.debug("Removed {} statements", deleteStatementBuffer.size());
deleteStatementBuffer = Collections.synchronizedSet(new HashSet<>(BUFFER_THRESHOLD));
}
Example 20
| Source File: IDAO.java From spider with GNU General Public License v3.0 | 4 votes |
|
/**
* 根据query删除数据
*
* @param queryBuilder query
* @param task 任务实体
* @return 是否全部数据删除成功
*/
protected boolean deleteByQuery(QueryBuilder queryBuilder, Task task) {
BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
SearchRequestBuilder searchRequestBuilder = client.prepareSearch(INDEX_NAME)
.setTypes(TYPE_NAME)
.setQuery(queryBuilder)
.setSize(100)
.setScroll(TimeValue.timeValueMinutes(SCROLL_TIMEOUT));
SearchResponse response = searchRequestBuilder.execute().actionGet();
while (true) {
for (SearchHit hit : response.getHits()) {
bulkRequestBuilder.add(new DeleteRequest(INDEX_NAME, TYPE_NAME, hit.id()));
if (task != null) {
task.increaseCount();
}
}
response = client.prepareSearchScroll(response.getScrollId())
.setScroll(TimeValue.timeValueMinutes(SCROLL_TIMEOUT))
.execute().actionGet();
if (response.getHits().getHits().length == 0) {
if (task != null) {
task.setDescription("按query%s删除数据ID添加完毕,已经添加%s条,准备执行删除", queryBuilder.toString(), bulkRequestBuilder.numberOfActions());
}
LOG.debug("按query{}删除数据ID添加完毕,准备执行删除", queryBuilder.toString());
break;
} else {
if (task != null) {
task.setDescription("按query%s删除数据已经添加%s条", queryBuilder.toString(), bulkRequestBuilder.numberOfActions());
}
LOG.debug("按query{}删除数据已经添加{}条", queryBuilder.toString(), bulkRequestBuilder.numberOfActions());
}
}
if (bulkRequestBuilder.numberOfActions() <= 0) {
if (task != null) {
task.setDescription("按query%s删除数据时未找到数据,请检查参数", queryBuilder.toString());
}
LOG.debug("按query{}删除数据时未找到数据,请检查参数", queryBuilder.toString());
return false;
}
BulkResponse bulkResponse = bulkRequestBuilder.get();
if (bulkResponse.hasFailures()) {
if (task != null) {
task.setDescription("按query%s删除数据部分失败,%s", queryBuilder.toString(), bulkResponse.buildFailureMessage());
}
LOG.error("按query{}删除数据部分失败,{}", queryBuilder.toString(), bulkResponse.buildFailureMessage());
} else {
if (task != null) {
task.setDescription("按query%s删除数据成功,耗时:%s毫秒", queryBuilder.toString(), bulkResponse.getTookInMillis());
}
LOG.info("按query{}删除数据成功,耗时:{}毫秒", queryBuilder.toString(), bulkResponse.getTookInMillis());
}
return bulkResponse.hasFailures();
}
