org.elasticsearch.index.reindex.DeleteByQueryRequest Java Examples

/**
 * @return Map
 * @Author pancm
 * @Description //根据条件删除数据
 * @Date 2019/3/21
 * @Param []
 **/
public static Map<String, Object> deleteByQuery(String index, String type, QueryBuilder[] queryBuilders) throws IOException {
    if (index == null || type == null || queryBuilders == null) {
        return null;
    }
    Map<String, Object> map = new HashMap<>();
    try {
        DeleteByQueryRequest request = new DeleteByQueryRequest(index, type);
        if (queryBuilders != null) {
            for (QueryBuilder queryBuilder : queryBuilders) {
                request.setQuery(queryBuilder);
            }
        }
        // 同步执行
        BulkByScrollResponse bulkResponse = client.deleteByQuery(request, RequestOptions.DEFAULT);
        // 响应结果处理
        map.put("time", bulkResponse.getTook().getMillis());
        map.put("total", bulkResponse.getTotal());

    } finally {
        if (isAutoClose) {
            close();
        }
    }
    return map;
}
 

/**
 * Deletes all the documents of a given dataset in a given index.
 *
 * @param esClient client to connect to ES
 * @param datasetKey key of the dataset whose documents will be deleted
 * @param index index where the the documents will be deleted from
 */
public static void deleteByDatasetKey(
    final RestHighLevelClient esClient, String datasetKey, String index) {
  LOG.info("Deleting all documents of dataset {} from ES index {}", datasetKey, index);
  Objects.requireNonNull(esClient);
  Preconditions.checkArgument(!Strings.isNullOrEmpty(datasetKey), "datasetKey is required");
  Preconditions.checkArgument(!Strings.isNullOrEmpty(index), "index is required");

  DeleteByQueryRequest request =
      new DeleteByQueryRequest(index)
          .setBatchSize(5000)
          .setQuery(QueryBuilders.matchQuery(DATASET_KEY_FIELD, datasetKey));

  try {
    esClient.deleteByQuery(request, HEADERS.get());
  } catch (IOException e) {
    LOG.error("Could not delete records of dataset {} from index {}", datasetKey, index);
  }
}
 

/**
 * Description: delete by query
 *
 * @param index   index
 * @param builder builder
 * @author fanxb
 * @date 2019/7/26 15:16
 */
public void deleteByQuery(String index, QueryBuilder builder) {
    if (!status) {
        return;
    }
    DeleteByQueryRequest request = new DeleteByQueryRequest(index);
    request.setQuery(builder);
    //设置批量操作数量,最大为10000
    request.setBatchSize(10000);
    request.setConflicts("proceed");
    try {
        client.deleteByQuery(request, RequestOptions.DEFAULT);
    } catch (Exception e) {
        throw new EsException(e);
    }
}
 

private void parseDeleteByQuery(ElasticDslContext dslContext) {
    dslContext.getParseResult().setSqlOperation(SqlOperation.DELETE_BY_QUERY);
    ElasticsearchParser.DeleteOperationContext deleteOperationContext = dslContext.getSqlContext().deleteOperation();
    List<String> indices = new ArrayList<>(0);
    for (ElasticsearchParser.TableRefContext tableRefContext : deleteOperationContext.tableRef()) {
        indices.add(tableRefContext.indexName.getText());
    }
    DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(indices.toArray(new String[0]));
    BoolExpressionParser boolExpressionParser = new BoolExpressionParser();
    if (deleteOperationContext.whereClause() != null) {
        deleteByQueryRequest.setQuery(boolExpressionParser.parseBoolQueryExpr(deleteOperationContext.whereClause().expression()));
    } else {
        deleteByQueryRequest.setQuery(QueryBuilders.matchAllQuery());
    }
    if (deleteOperationContext.routingClause() != null) {
        deleteByQueryRequest.setRouting(StringManager.removeStringSymbol(deleteOperationContext.routingClause().STRING(0).getText()));
    }
    if (deleteOperationContext.batchClause() != null) {
        deleteByQueryRequest.setBatchSize(Integer.parseInt(deleteOperationContext.batchClause().size.getText()));
    }
    if (deleteOperationContext.limitClause() != null) {
        deleteByQueryRequest.setMaxDocs(Integer.parseInt(deleteOperationContext.limitClause().size.getText()));
    }
    dslContext.getParseResult().setDeleteByQueryRequest(deleteByQueryRequest);
}
 

@Test
public void testDelete() throws Exception {
    BulkByScrollResponse expected = buildBulkResponse();
    when(client.deleteByQuery(any(DeleteByQueryRequest.class), any()))
            .thenReturn(expected);
    ElasticSearchDataCleaner elasticSearchDataCleaner = new ElasticSearchDataCleaner();
    BulkByScrollResponse actual = elasticSearchDataCleaner.deleteData(client);
    assertEquals(actual,expected);
}
 

@Test
public void testDeleteIOException() throws Exception {
    when(client.deleteByQuery(any(DeleteByQueryRequest.class), any()))
            .thenThrow(IOException.class);
    ElasticSearchDataCleaner elasticSearchDataCleaner = new ElasticSearchDataCleaner();
    BulkByScrollResponse actual = elasticSearchDataCleaner.deleteData(client);
    assertNull(actual);
}
 

@Test
public void testDeleteException() throws Exception {
    when(client.deleteByQuery(any(DeleteByQueryRequest.class), any()))
            .thenThrow(Exception.class);
    ElasticSearchDataCleaner elasticSearchDataCleaner = new ElasticSearchDataCleaner();
    BulkByScrollResponse actual = elasticSearchDataCleaner.deleteData(client);
    assertNull(actual);
}
 

@Override
public void run() {
    DeleteByQueryRequest deleteRequest = new DeleteByQueryRequest(CommonName.CHECKPOINT_INDEX_NAME)
        .setQuery(
            QueryBuilders
                .boolQuery()
                .filter(
                    QueryBuilders
                        .rangeQuery(CheckpointDao.TIMESTAMP)
                        .lte(clock.millis() - checkpointTtl.toMillis())
                        .format(CommonName.EPOCH_MILLIS_FORMAT)
                )
        )
        .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN);
    clientUtil
        .execute(
            DeleteByQueryAction.INSTANCE,
            deleteRequest,
            ActionListener
                .wrap(
                    response -> {
                        // if 0 docs get deleted, it means our query cannot find any matching doc
                        LOG.info("{} " + CHECKPOINT_DELETED_MSG, response.getDeleted());
                    },
                    exception -> {
                        if (exception instanceof IndexNotFoundException) {
                            LOG.info(CHECKPOINT_NOT_EXIST_MSG);
                        } else {
                            // Gonna eventually delete in maintenance window.
                            LOG.error(CANNOT_DELETE_OLD_CHECKPOINT_MSG, exception);
                        }
                    }
                )
        );
}
 

/**
	 * 根据查询条件删除
	 *
	 * @throws IOException
	 */
	private static void deleteByQuery() throws IOException {
		String type = "_doc";
		String index = "test1";
		DeleteByQueryRequest request = new DeleteByQueryRequest(index,type);
		// 设置查询条件
		request.setQuery(QueryBuilders.termQuery("uid",1234));
		// 同步执行
		BulkByScrollResponse bulkResponse = client.deleteByQuery(request, RequestOptions.DEFAULT);

		// 异步执行
//		client.updateByQueryAsync(request, RequestOptions.DEFAULT, listener);

		// 返回结果
		TimeValue timeTaken = bulkResponse.getTook();
		boolean timedOut = bulkResponse.isTimedOut();
		long totalDocs = bulkResponse.getTotal();
		long updatedDocs = bulkResponse.getUpdated();
		long deletedDocs = bulkResponse.getDeleted();
		long batches = bulkResponse.getBatches();
		long noops = bulkResponse.getNoops();
		long versionConflicts = bulkResponse.getVersionConflicts();
		long bulkRetries = bulkResponse.getBulkRetries();
		long searchRetries = bulkResponse.getSearchRetries();
		TimeValue throttledMillis = bulkResponse.getStatus().getThrottled();
		TimeValue throttledUntilMillis = bulkResponse.getStatus().getThrottledUntil();
		List<ScrollableHitSource.SearchFailure> searchFailures = bulkResponse.getSearchFailures();
		List<BulkItemResponse.Failure> bulkFailures = bulkResponse.getBulkFailures();
		System.out.println("查询更新总共花费了:" + timeTaken.getMillis() + " 毫秒，总条数:" + totalDocs + ",更新数:" + updatedDocs);

	}
 

/**
 * Description: delete by query
 *
 * @param index   index
 * @param builder builder
 * @author fanxb
 * @date 2019/7/26 15:16
 */
public void deleteByQuery(String index, QueryBuilder builder) {
    DeleteByQueryRequest request = new DeleteByQueryRequest(index);
    request.setQuery(builder);
    //设置批量操作数量,最大为10000
    request.setBatchSize(10000);
    request.setConflicts("proceed");
    try {
        client.deleteByQuery(request, RequestOptions.DEFAULT);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 

public static DeleteByQueryRequest getDeleteByQueryRequest(String index, QueryBuilder query) {

        final DeleteByQueryRequest request =
                new DeleteByQueryRequest(index);
        request.setQuery(query);
        request.setRefresh(true);
        return request;
    }
 

public int delete(String indexName, String timeBucketColumnName, long endTimeBucket) throws IOException {
    indexName = formatIndexName(indexName);

    DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(indexName);
    deleteByQueryRequest.setAbortOnVersionConflict(false);
    deleteByQueryRequest.setQuery(QueryBuilders.rangeQuery(timeBucketColumnName).lte(endTimeBucket));
    BulkByScrollResponse bulkByScrollResponse = client.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT);
    log.debug(
        "delete indexName: {}, by query request: {}, response: {}", indexName, deleteByQueryRequest,
        bulkByScrollResponse
    );
    return HttpStatus.SC_OK;
}
 

/**
	 * Set indices and types to the delete by query request.
	 */
	private void setIndicesAndTypes() {

        DeleteByQueryRequest innerRequest = request.request();
        innerRequest.indices(query.getIndexArr());
        String[] typeArr = query.getTypeArr();
        if (typeArr!=null){
            innerRequest.getSearchRequest().types(typeArr);
        }
//		String[] typeArr = query.getTypeArr();
//		if (typeArr != null) {
//            request.set(typeArr);
//		}
	}
 

public void setDeleteByQueryRequest(DeleteByQueryRequest deleteByQueryRequest) {
    this.deleteByQueryRequest = deleteByQueryRequest;
}
 

public BulkByScrollResponse deleteByQuery(DeleteByQueryRequest deleteByQueryRequest, RequestOptions requestOptions)
        throws Exception {
    return client.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT);
}
 

/**
	 * 根据查询条件删除
	 * @throws IOException
	 */
	private static void deleteByQuery() throws IOException {
		//
		DeleteByQueryRequest request = new DeleteByQueryRequest("user");

		// 设置查询条件
		request.setQuery(new TermQueryBuilder("user", "pancm"));

		// 设置复制文档的数量
		request.setSize(10);
		// 设置一次批量处理的条数，默认是1000
		request.setBatchSize(100);
		//设置路由
		request.setRouting("=cat");
		//设置超时时间
		request.setTimeout(TimeValue.timeValueMinutes(2));
		//允许刷新
		request.setRefresh(true);
		//索引选项
		request.setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN);
		
		// 同步执行
		BulkByScrollResponse bulkResponse = client.deleteByQuery(request, RequestOptions.DEFAULT);

		// 异步执行
//		client.updateByQueryAsync(request, RequestOptions.DEFAULT, listener); 

		// 返回结果
		TimeValue timeTaken = bulkResponse.getTook();
		boolean timedOut = bulkResponse.isTimedOut();
		long totalDocs = bulkResponse.getTotal();
		long deletedDocs = bulkResponse.getDeleted();
		long batches = bulkResponse.getBatches();
		long noops = bulkResponse.getNoops();
		long versionConflicts = bulkResponse.getVersionConflicts();
		long bulkRetries = bulkResponse.getBulkRetries();
		long searchRetries = bulkResponse.getSearchRetries();
		TimeValue throttledMillis = bulkResponse.getStatus().getThrottled();
		TimeValue throttledUntilMillis = bulkResponse.getStatus().getThrottledUntil();
		List<ScrollableHitSource.SearchFailure> searchFailures = bulkResponse.getSearchFailures();
		List<BulkItemResponse.Failure> bulkFailures = bulkResponse.getBulkFailures();
		System.out.println("查询更新总共花费了:" + timeTaken.getMillis() + " 毫秒，总条数:" + totalDocs + ",删除数:" + deletedDocs);

	}
 

public BulkByScrollResponse deleteByQuery(QueryBuilder query) throws IOException {
    final DeleteByQueryRequest request = ElasticRequestUtils.getDeleteByQueryRequest(defaultIndex, query);
    return client.deleteByQuery(request,RequestOptions.DEFAULT);
}
 

public DeleteByQueryRequest getDeleteByQueryRequest() {
    return deleteByQueryRequest;
}