org.elasticsearch.action.support.WriteRequest Java Examples

/**
 * 批量更新服务
 *
 * @param obj 批量新增请求参数
 * @return <code>true</code>全部更新成功,<code>false</code>部分更新失败.
 * @throws Exception es 执行异常
 */
public SearchBaseResult<Boolean> batchSave(final BatchSaveESObject obj) {
    final BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();
    final List<SaveESObject> saveDatas = obj.getSaveDatas();
    for (SaveESObject esObject : saveDatas) {
        bulkRequestBuilder.add(getIndexRequest(esObject));
    }
    if (obj.isRefresh()) {
        bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL);
    }
    try {
        SearchLogger.log(bulkRequestBuilder);
        BulkResponse bulkResponse = bulkRequestBuilder.execute().actionGet();
        SearchLogger.log(bulkResponse);
        return SearchBaseResult.success(!bulkResponse.hasFailures(), Boolean.class);
    } catch (Exception ex) {
        SearchLogger.error("batchSave", ex);
        return SearchBaseResult.faild(ESErrorCode.ELASTIC_ERROR_CODE, "esMsg:" + ex.getMessage());
    }
}
 

@Override
public void store(String key, Table value) {
    if(key == null || value == null || value.getName() == null) {
        throw new TableMapStoreException(String.format("Illegal Store Request - %s - %s", key, value));
    }
    logger.info("Storing key: {}", key);
    try {
        Map<String, Object> sourceMap = ElasticsearchQueryUtils.toMap(objectMapper, value);
        elasticsearchConnection.getClient()
                .prepareIndex()
                .setIndex(TABLE_META_INDEX)
                .setType(TABLE_META_TYPE)
                .setSource(sourceMap)
                .setId(key)
                .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
                .execute()
                .actionGet();
    } catch (Exception e) {
        throw new TableMapStoreException("Error saving meta: ", e);
    }
}
 

@Override
public void saveV2(ConsoleV2 console, boolean newConsole) {
    preProcess(console, newConsole);
    try {
        connection.getClient()
                .prepareIndex()
                .setIndex(INDEX_V2)
                .setType(TYPE)
                .setId(console.getId())
                .setSource(mapper.writeValueAsBytes(console), XContentType.JSON)
                .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
                .execute()
                .get();
        logger.info("Saved Console : {}", console);
    } catch (Exception e) {
        throw new ConsolePersistenceException(console.getId(), "console save failed", e);
    }
}
 

@Override
public boolean lockBooking()
{
    try
    {
        IndexRequest indexRequest = new IndexRequest(this.bookingsIndex, ES_TYPE);
        indexRequest.id(LOCK_BOOKING_ID);
        indexRequest.create(true);
        indexRequest.source(new HashMap());
        esclient.index(indexRequest, WriteRequest.RefreshPolicy.WAIT_UNTIL.getValue());
    }
    catch (Exception e)
    {
        return false;
    }
    return true;
}
 

@Override
public void save(final Console console) {
    try {
        connection.getClient()
                .prepareIndex()
                .setIndex(INDEX)
                .setType(TYPE)
                .setId(console.getId())
                .setSource(ElasticsearchQueryUtils.toMap(mapper, console))
                .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
                .execute()
                .get();
        logger.info("Saved Console : {}", console);
    } catch (Exception e) {
        throw new ConsolePersistenceException(console.getId(), "console save failed", e);
    }
}
 

/**
 * Index a Fact into ElasticSearch.
 *
 * @param fact Fact to index
 * @return Indexed Fact
 */
public FactDocument indexFact(FactDocument fact) {
  if (fact == null || fact.getId() == null) return null;
  IndexResponse response;

  try {
    IndexRequest request = new IndexRequest(INDEX_NAME, TYPE_NAME, fact.getId().toString())
            .setRefreshPolicy(isTestEnvironment ? WriteRequest.RefreshPolicy.IMMEDIATE : WriteRequest.RefreshPolicy.NONE)
            .source(FACT_DOCUMENT_WRITER.writeValueAsBytes(fact), XContentType.JSON);
    response = clientFactory.getClient().index(request, RequestOptions.DEFAULT);
  } catch (ElasticsearchException | IOException ex) {
    throw logAndExit(ex, String.format("Could not perform request to index Fact with id = %s.", fact.getId()));
  }

  if (response.status() != RestStatus.OK && response.status() != RestStatus.CREATED) {
    LOGGER.warning("Could not index Fact with id = %s.", fact.getId());
  } else if (response.getResult() == DocWriteResponse.Result.CREATED) {
    LOGGER.info("Successfully indexed Fact with id = %s.", fact.getId());
  } else if (response.getResult() == DocWriteResponse.Result.UPDATED) {
    LOGGER.info("Successfully re-indexed existing Fact with id = %s.", fact.getId());
  }

  return fact;
}
 

@Test
public void testLoadAllKeys() throws Exception {
    Map<String, Table> tables = Maps.newHashMap();
    for(int i = 0; i < 10; i++) {
        Table table = new Table();
        table.setName(UUID.randomUUID()
                              .toString());
        table.setTtl(20);
        tables.put(table.getName(), table);
        Map<String, Object> sourceMap = ElasticsearchQueryUtils.toMap(mapper, table);
        elasticsearchConnection.getClient()
                .prepareIndex()
                .setIndex(TABLE_META_INDEX)
                .setType(TABLE_META_TYPE)
                .setSource(sourceMap)
                .setId(table.getName())
                .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
                .execute()
                .actionGet();
    }

    Set<String> responseKeys = tableMapStore.loadAllKeys();
    for(String name : tables.keySet()) {
        assertTrue(responseKeys.contains(name));
    }
}
 

/**
 * Constructor function.
 *
 * @param clusterService          ClusterService
 * @param client                  ES node client that executes actions on the local node
 * @param channel                 ES channel used to construct bytes / builder based outputs, and send responses
 * @param anomalyDetectionIndices anomaly detector index manager
 * @param detectorId              detector identifier
 * @param seqNo                   sequence number of last modification
 * @param primaryTerm             primary term of last modification
 * @param refreshPolicy           refresh policy
 * @param requestTimeout          request time out configuration
 */
public IndexAnomalyDetectorJobActionHandler(
    ClusterService clusterService,
    NodeClient client,
    RestChannel channel,
    AnomalyDetectionIndices anomalyDetectionIndices,
    String detectorId,
    Long seqNo,
    Long primaryTerm,
    WriteRequest.RefreshPolicy refreshPolicy,
    TimeValue requestTimeout
) {
    super(client, channel);
    this.clusterService = clusterService;
    this.anomalyDetectionIndices = anomalyDetectionIndices;
    this.detectorId = detectorId;
    this.seqNo = seqNo;
    this.primaryTerm = primaryTerm;
    this.refreshPolicy = refreshPolicy;
    this.requestTimeout = requestTimeout;
}
 

@Override
public void deleteV2(String id) {
    try {
        connection.getClient()
                .prepareDelete()
                .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
                .setIndex(INDEX_V2)
                .setType(TYPE)
                .setId(id)
                .execute()
                .actionGet();
        logger.info("Deleted Console : {}", id);
    } catch (Exception e) {
        throw new ConsolePersistenceException(id, "console deletion_failed", e);
    }
}
 

@BeforeEach
public void setup() throws Exception {
    client = ElasticsearchClientFactory.create(globals());
    retrieveDao = new ElasticsearchRetrieveLatestDao(client);
    writer = new ElasticsearchBulkDocumentWriter<>(client)
            .withRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL);

    // add bro template
    JSONObject broTemplate = JSONUtils.INSTANCE.load(new File(broTemplatePath), JSONObject.class);
    String broTemplateJson = JSONUtils.INSTANCE.toJSON(broTemplate, true);
    HttpEntity broEntity = new NStringEntity(broTemplateJson, ContentType.APPLICATION_JSON);
    Response response = client
            .getLowLevelClient()
            .performRequest("PUT", "/_template/bro_template", Collections.emptyMap(), broEntity);
    assertThat(response.getStatusLine().getStatusCode(), CoreMatchers.equalTo(200));
}
 

private void deleteAnomalyDetectorJobDoc(NodeClient client, String detectorId, RestChannel channel) {
    logger.info("Delete anomaly detector job {}", detectorId);
    DeleteRequest deleteRequest = new DeleteRequest(AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX, detectorId)
        .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
    client.delete(deleteRequest, ActionListener.wrap(response -> {
        if (response.getResult() == DocWriteResponse.Result.DELETED || response.getResult() == DocWriteResponse.Result.NOT_FOUND) {
            deleteAnomalyDetectorDoc(client, detectorId, channel);
        } else {
            logger.error("Fail to delete anomaly detector job {}", detectorId);
        }
    }, exception -> {
        if (exception instanceof IndexNotFoundException) {
            deleteAnomalyDetectorDoc(client, detectorId, channel);
        } else {
            logger.error("Failed to delete anomaly detector job", exception);
            try {
                channel.sendResponse(new BytesRestResponse(channel, exception));
            } catch (IOException e) {
                logger.error("Failed to send response of delete anomaly detector job exception", e);
            }
        }
    }));
}
 

private void saveOldConsole(ConsoleV2 console) {
    String id = UUID.randomUUID()
            .toString();
    console.setId(id);
    try {
        connection.getClient()
                .prepareIndex()
                .setIndex(INDEX_HISTORY)
                .setType(TYPE)
                .setId(id)
                .setSource(mapper.writeValueAsBytes(console), XContentType.JSON)
                .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
                .execute()
                .get();
        logger.info("Saved Old Console : {}", console);
    } catch (Exception e) {
        throw new ConsolePersistenceException(console.getId(), "old console save failed", e);
    }
}
 

@Test
public void testLoad() throws Exception {
    Table table = new Table();
    table.setName(TEST_TABLE);
    table.setTtl(30);
    Map<String, Object> sourceMap = ElasticsearchQueryUtils.toMap(mapper, table);
    elasticsearchConnection.getClient()
            .prepareIndex()
            .setIndex(TABLE_META_INDEX)
            .setType(TABLE_META_TYPE)
            .setSource(sourceMap)
            .setId(table.getName())
            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
            .execute()
            .actionGet();

    Table responseTable = tableMapStore.load(table.getName());
    compareTables(table, responseTable);
}
 

@Override
public void deleteOldVersion(String id) {
    try {
        connection.getClient()
                .prepareDelete()
                .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
                .setIndex(INDEX_HISTORY)
                .setType(TYPE)
                .setId(id)
                .execute()
                .actionGet();
        logger.info("Deleted Old Console : {}", id);
    } catch (Exception e) {
        throw new ConsolePersistenceException(id, "old console deletion_failed", e);
    }
}
 

/**
 * 根据文档唯一Id删除指定的文档
 *
 * @param esObject 删除参数
 * @return <code>true</code>文档删除成功,<code>true</code>未找到对应文档.
 */
public SearchBaseResult<Boolean> delete(final DeleteESObject esObject) {
    final DeleteRequestBuilder deleteRequest = getDeleteRequest(esObject);
    try {
        SearchLogger.log(deleteRequest);
        if (esObject.isRefresh()) {
            deleteRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL);
        }
        DeleteResponse deleteResponse = deleteRequest.execute().actionGet();
        SearchLogger.log(deleteRequest);
        if (DocWriteResponse.Result.DELETED == deleteResponse.getResult()) {
            return SearchBaseResult.success(Boolean.TRUE, Boolean.class);
        }
        if (DocWriteResponse.Result.NOT_FOUND == deleteResponse.getResult()) {
            return SearchBaseResult.success(Boolean.FALSE, Boolean.class);
        }
        return SearchBaseResult.faild(ESErrorCode.ELASTIC_ERROR_CODE, "ES返回结果不在预期范围内");
    } catch (Exception ex) {
        SearchLogger.error("delete", ex);
        return SearchBaseResult.faild(ESErrorCode.ELASTIC_ERROR_CODE, "esMsg:" + ex.getMessage());
    }

}
 

/**
 * 更新操作
 *
 * @param esObject es通用更新请求参数
 * @return <code>true</code> 保存或更新成功; 否则更新失败
 */
public SearchBaseResult<Boolean> update(UpdateESObject esObject) {
    final UpdateRequestBuilder updateRequest = esObject.nestedUpdate() ? getNestedListUpdateRequest(esObject)
            : getUpdateRequest(esObject);
    SearchLogger.log(updateRequest);
    try {
        updateRequest.setDetectNoop(false);
        if (esObject.isRefresh()) {
            updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL);
        }
        UpdateResponse updateResponse = updateRequest.execute().get();
        SearchLogger.log(updateResponse);
        final DocWriteResponse.Result result = updateResponse.getResult();
        return SearchBaseResult.success(DocWriteResponse.Result.UPDATED == result, Boolean.class);
    } catch (Exception ex) {
        SearchLogger.error("update", ex);
        final String message = ex.getMessage();
        if (message != null && message.contains("document missing")) {
            return SearchBaseResult.faild(ESErrorCode.DOC_NOT_EXIST_ERROR_CODE, "更新文档不存在");
        }
        return SearchBaseResult.faild(ESErrorCode.ELASTIC_ERROR_CODE, "esMsg:" + message);
    }
}
 

/**
 * 存储通用对象
 *
 * @param esObject es通用存储数据结构
 * @return <code>true</code> 保存成功，<code>false</code>保存失败;
 */
public SearchBaseResult<Boolean> save(SaveESObject esObject) {
    try {
        final IndexRequestBuilder indexRequest = getIndexRequest(esObject);
        SearchLogger.log(indexRequest);
        if (esObject.isRefresh()) {
            indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL);
        }
        IndexResponse indexResponse = indexRequest.execute().get();
        SearchLogger.log(indexResponse);
        return SearchBaseResult.success(DocWriteResponse.Result.CREATED == indexResponse.getResult(), Boolean.class);
    } catch (Exception ex) {
        SearchLogger.error("save", ex);
        return SearchBaseResult.faild(ESErrorCode.ELASTIC_ERROR_CODE, "esMsg:" + ex.getMessage());
    }
}
 

public void forceUpdate(String indexName, String id, XContentBuilder source, long version) throws IOException {
    org.elasticsearch.action.update.UpdateRequest request = (org.elasticsearch.action.update.UpdateRequest) prepareUpdate(
        indexName, id, source);
    request.version(version);
    request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
    client.update(request);
}
 

private void buildSendProcess(List<SyncWrapper<WriteRequest>> aim) throws InterruptedException {
  if (aim != null) {
    logger.info("Flush batch({})", aim.size());
    BulkResponse bulkResponse = buildAndSend(aim);
    if (!bulkResponse.hasFailures()) {
      ackSuccess(aim);
    } else {
      retryFailed(aim, new ElasticsearchBulkException("Bulk request has failures", bulkResponse));
    }
  }
}
 

public ElasticsearchUpdateDao(ElasticsearchClient client,
    AccessConfig accessConfig,
    ElasticsearchRetrieveLatestDao searchDao) {
  this.accessConfig = accessConfig;
  this.retrieveLatestDao = searchDao;
  this.documentWriter = new ElasticsearchBulkDocumentWriter<>(client)
          .withRefreshPolicy(WriteRequest.RefreshPolicy.NONE);
}
 

@Override
public void delete(String key) {
    logger.info("Delete called for value: {}", key);
    elasticsearchConnection.getClient()
            .prepareDelete()
            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
            .setIndex(TABLE_META_INDEX)
            .setType(TABLE_META_TYPE)
            .setId(key)
            .execute()
            .actionGet();
    logger.info("Deleted value: {}", key);
}
 

public void testLangDetectBinary() throws Exception {
    try {
        CreateIndexRequestBuilder createIndexRequestBuilder =
                new CreateIndexRequestBuilder(client(), CreateIndexAction.INSTANCE).setIndex("test");
        createIndexRequestBuilder.addMapping("someType", jsonBuilder()
                .startObject()
                .startObject("properties")
                .startObject("content")
                .field("type", "binary")
                .startObject("fields")
                .startObject("language")
                .field("type", "langdetect")
                .field("binary", true)
                .endObject()
                .endObject()
                .endObject()
                .endObject()
                .endObject());
        createIndexRequestBuilder.execute().actionGet();
        IndexRequestBuilder indexRequestBuilder = new IndexRequestBuilder(client(), IndexAction.INSTANCE)
                .setIndex("test").setType("someType").setId("1")
                //\"God Save the Queen\" (alternatively \"God Save the King\"
                .setSource("content", "IkdvZCBTYXZlIHRoZSBRdWVlbiIgKGFsdGVybmF0aXZlbHkgIkdvZCBTYXZlIHRoZSBLaW5nIg==");
        indexRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
                .execute().actionGet();
        SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client(), SearchAction.INSTANCE)
                .setIndices("test")
                .setQuery(QueryBuilders.termQuery("content.language", "en"))
                .addStoredField("content.language");
        SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
        assertEquals(1L, searchResponse.getHits().getTotalHits());
        assertEquals("en", searchResponse.getHits().getAt(0).field("content.language").getValue());
    } finally {
        DeleteIndexRequestBuilder deleteIndexRequestBuilder =
                new DeleteIndexRequestBuilder(client(), DeleteIndexAction.INSTANCE, "test");
        deleteIndexRequestBuilder.execute().actionGet();
    }
}
 

public void synchronousBulk(BulkRequest request) {
    request.timeout(TimeValue.timeValueMinutes(2));
    request.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL);
    request.waitForActiveShards(ActiveShardCount.ONE);
    try {
        int size = request.requests().size();
        BulkResponse responses = client.bulk(request);
        log.info("Synchronous bulk took time: {} millis, size: {}", responses.getTook().getMillis(), size);
    } catch (IOException e) {
        log.error(e.getMessage(), e);
    }
}
 

public void buildIndex() {
    client().admin().indices().prepareCreate("test_index").get();
    client().prepareIndex("test_index", "test")
            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
            .setSource("field1", "hello world", "field2", "bonjour world")
            .get();
}
 

private IndexRequest buildIndexRequest(Task parentTask, FeatureStoreRequest request) throws IOException {
    StorableElement elt = request.getStorableElement();

    IndexRequest indexRequest = client.prepareIndex(request.getStore(), IndexFeatureStore.ES_TYPE, elt.id())
            .setCreate(request.getAction() == FeatureStoreRequest.Action.CREATE)
            .setRouting(request.getRouting())
            .setSource(IndexFeatureStore.toSource(elt))
            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
            .request();
    indexRequest.setParentTask(clusterService.localNode().getId(), parentTask.getId());
    return indexRequest;
}
 

public BulkResponse add(List<Map<String, Object>> jsonDocs) throws IOException {
    final BulkRequest bulkIndexRequest = new BulkRequest();
    jsonDocs.forEach( jsonDoc -> bulkIndexRequest.add(ElasticRequestUtils.getIndexRequest(defaultIndex,jsonDoc)) );
    bulkIndexRequest.timeout(TimeValue.timeValueMillis(connectionTimeOut));
    bulkIndexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
    return client.bulk(bulkIndexRequest, RequestOptions.DEFAULT);
}
 

public BulkResponse add(Map<String, Object> jsonDoc) throws IOException {
    final BulkRequest bulkIndexRequest = new BulkRequest();
    bulkIndexRequest.add(ElasticRequestUtils.getIndexRequest(defaultIndex,jsonDoc));
    bulkIndexRequest.timeout(TimeValue.timeValueMillis(connectionTimeOut));
    bulkIndexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
    return client.bulk(bulkIndexRequest, RequestOptions.DEFAULT);
}
 

private BulkRequest createBulkRequest(List<DocWriteRequest> doc) {
    BulkRequest br = new BulkRequest();
    br.add(doc);
    br.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
    br.waitForActiveShards(ActiveShardCount.ONE);
    return br;
}
 

private void handle404(SyncWrapper<WriteRequest> wrapper, BulkItemResponse item) {
  if (item.getFailure().getCause() instanceof DocumentMissingException) {
    logger.warn("Make update request upsert to resolve DocumentMissingException");
    UpdateRequest request = ((UpdateRequest) wrapper.getData());
    if (request.doc() != null) {
      request.docAsUpsert(true);
    } else {
      request.upsert(ESRequestMapper.getUpsert(wrapper.getEvent())).scriptedUpsert(true);
    }
  }
}
 

private Object requestStr(SyncWrapper<WriteRequest> wrapper) {
  WriteRequest request = wrapper.getData();
  String reqStr = null;
  if (request instanceof UpdateRequest) {
    reqStr = toString((UpdateRequest) request);
  }
  return reqStr == null ? request : reqStr;
}