org.elasticsearch.action.index.IndexRequest#source

Source File: ElasticSearchRestDAOV5.java From conductor with Apache License 2.0

7 votes

private void indexObject(final String index, final String docType, final String docId, final Object doc) {

        byte[] docBytes;
        try {
            docBytes = objectMapper.writeValueAsBytes(doc);
        } catch (JsonProcessingException e) {
            logger.error("Failed to convert {} '{}' to byte string", docType, docId);
            return;
        }

        IndexRequest request = new IndexRequest(index, docType, docId);
        request.source(docBytes, XContentType.JSON);

        if(bulkRequests.get(docType) == null) {
            bulkRequests.put(docType, new BulkRequests(System.currentTimeMillis(), new BulkRequest()));
        }

        bulkRequests.get(docType).getBulkRequest().add(request);
        if (bulkRequests.get(docType).getBulkRequest().numberOfActions() >= this.indexBatchSize) {
            indexBulkRequest(docType);
        }
    }

Source File: ElasticStatisticsPublisher.java From product-ei with Apache License 2.0

6 votes

/**
 * Publishes the array list of simplified jsons to Elasticsearch using the Java High Level REST Client.
 *
 * @param jsonsToSend array list of json strings to be published to Elasticsearch
 * @param client      elasticsearch RestHighLevelClient
 */
public static void publish(List<String> jsonsToSend, RestHighLevelClient client) {

    // Prepare and Send the bulk request
    IndexRequest indexRequest = new IndexRequest("eidata");
    if (jsonsToSend != null) {
        for (String jsonString : jsonsToSend) {
            BulkProcessor.Builder bulkProcessorBuilder = BulkProcessor.builder(
                    (request, bulkListener) ->
                            client.bulkAsync(request, RequestOptions.DEFAULT, bulkListener),
                    new BulkProcessorListener());
            bulkProcessor = bulkProcessorBuilder.build();
            indexRequest.source(jsonString, XContentType.JSON);
            bulkProcessor.add(indexRequest);
            bulkProcessor.close();
        }
    }
}

Source File: ElasticScrollingFactoryEsTest.java From elastic-crud with Apache License 2.0

6 votes

@Before
public void before() throws IOException {
  if(!client.admin().indices().prepareExists(INDEX).execute().actionGet().isExists()) {
    client.admin().indices().prepareCreate(INDEX).execute().actionGet();
  }
  final JsonSerializer<Person> serializer = mapper.serializer(Person.class);
  final BulkRequestBuilder bulk = client.prepareBulk();
  for (int i = 0; i < SIZE; i++) {

    final String name = UUID.randomUUID().toString();
    final IndexRequest request = new IndexRequest(INDEX, TYPE);
    request.source(serializer.apply(Person.builder().id("").firstname(name).lastname(name).build()), JSON);
    bulk.add(request);
  }

  client.bulk(bulk.request()).actionGet();
  flush(INDEX);
}

Source File: ElasticSearchManualTest.java From tutorials with MIT License

6 votes

@Test
public void givenDocumentId_whenJavaObject_thenDeleteDocument() throws Exception {
    String jsonObject = "{\"age\":10,\"dateOfBirth\":1471455886564,\"fullName\":\"Johan Doe\"}";
    IndexRequest indexRequest = new IndexRequest("people");
    indexRequest.source(jsonObject, XContentType.JSON);

    IndexResponse response = client.index(indexRequest, RequestOptions.DEFAULT);
    String id = response.getId();

    GetRequest getRequest = new GetRequest("people");
    getRequest.id(id);

    GetResponse getResponse = client.get(getRequest, RequestOptions.DEFAULT);
    System.out.println(getResponse.getSourceAsString());

    DeleteRequest deleteRequest = new DeleteRequest("people");
    deleteRequest.id(id);

    DeleteResponse deleteResponse = client.delete(deleteRequest, RequestOptions.DEFAULT);

    assertEquals(Result.DELETED, deleteResponse.getResult());
}

Source File: BookingDaoESImp.java From blue-marlin with Apache License 2.0

6 votes

@Override
public boolean lockBooking()
{
    try
    {
        IndexRequest indexRequest = new IndexRequest(this.bookingsIndex, ES_TYPE);
        indexRequest.id(LOCK_BOOKING_ID);
        indexRequest.create(true);
        indexRequest.source(new HashMap());
        esclient.index(indexRequest, WriteRequest.RefreshPolicy.WAIT_UNTIL.getValue());
    }
    catch (Exception e)
    {
        return false;
    }
    return true;
}

Source File: BulkRequest.java From Elasticsearch with Apache License 2.0

5 votes

BulkRequest internalAdd(IndexRequest request, @Nullable Object payload) {
    requests.add(request);
    addPayload(payload);
    // lack of source is validated in validate() method
    sizeInBytes += (request.source() != null ? request.source().length() : 0) + REQUEST_OVERHEAD;
    return this;
}

Source File: ElasticSearchDAOV5.java From conductor with Apache License 2.0

5 votes

@Override
public void addTaskExecutionLogs(List<TaskExecLog> taskExecLogs) {
    if (taskExecLogs.isEmpty()) {
        return;
    }

    try {
        long startTime = Instant.now().toEpochMilli();
        BulkRequestBuilderWrapper bulkRequestBuilder = new BulkRequestBuilderWrapper(elasticSearchClient.prepareBulk());
        for (TaskExecLog log : taskExecLogs) {
            IndexRequest request = new IndexRequest(logIndexName, LOG_DOC_TYPE);
            request.source(objectMapper.writeValueAsBytes(log), XContentType.JSON);
            bulkRequestBuilder.add(request);
        }
        new RetryUtil<BulkResponse>().retryOnException(
            () -> bulkRequestBuilder.execute().actionGet(5, TimeUnit.SECONDS),
            null,
            BulkResponse::hasFailures,
            RETRY_COUNT,
            "Indexing task execution logs",
            "addTaskExecutionLogs"
        );
        long endTime = Instant.now().toEpochMilli();
        logger.debug("Time taken {} for indexing taskExecutionLogs", endTime - startTime);
        Monitors.recordESIndexTime("index_task_execution_logs", LOG_DOC_TYPE, endTime - startTime);
        Monitors.recordWorkerQueueSize("logQueue", ((ThreadPoolExecutor) logExecutorService).getQueue().size());
    } catch (Exception e) {
        List<String> taskIds = taskExecLogs.stream()
            .map(TaskExecLog::getTaskId)
            .collect(Collectors.toList());
        logger.error("Failed to index task execution logs for tasks: {}", taskIds, e);
    }
}

Source File: ElasticSearchService.java From pybbs with GNU Affero General Public License v3.0

5 votes

public void createDocument(String type, String id, Map<String, Object> source) {
    try {
        if (this.instance() == null) return;
        IndexRequest request = new IndexRequest(name, type, id);
        request.source(source);
        client.index(request, RequestOptions.DEFAULT);
    } catch (IOException e) {
        log.error(e.getMessage());
    }
}

Source File: DataServiceImpl.java From java-11-examples with Apache License 2.0

5 votes

@Override
public Optional<EventId> saveData(EventDataInfo eventDataInfo) throws IOException {
    IndexRequest indexRequest = new IndexRequest(INDEX_NAME);
    indexRequest.source(Utils.createContent(eventDataInfo));
    IndexResponse indexResponse = client.index(indexRequest, RequestOptions.DEFAULT);
    if (RestStatus.CREATED.equals(indexResponse.status())) {
        return Optional.of(new EventId(indexResponse.getId()));
    }
    return Optional.empty();
}

Source File: ElasticSearchDAOV6.java From conductor with Apache License 2.0

5 votes

@Override
public void addTaskExecutionLogs(List<TaskExecLog> taskExecLogs) {
    if (taskExecLogs.isEmpty()) {
        return;
    }

    try {
        long startTime = Instant.now().toEpochMilli();
        BulkRequestBuilderWrapper bulkRequestBuilder = new BulkRequestBuilderWrapper(elasticSearchClient.prepareBulk());
        for (TaskExecLog log : taskExecLogs) {
            String docType = StringUtils.isBlank(docTypeOverride) ? LOG_DOC_TYPE : docTypeOverride;
            IndexRequest request = new IndexRequest(logIndexName, docType);
            request.source(objectMapper.writeValueAsBytes(log), XContentType.JSON);
            bulkRequestBuilder.add(request);
        }
        new RetryUtil<BulkResponse>().retryOnException(
            () -> bulkRequestBuilder.execute().actionGet(5, TimeUnit.SECONDS),
            null,
            BulkResponse::hasFailures,
            RETRY_COUNT,
            "Indexing task execution logs",
            "addTaskExecutionLogs"
        );
        long endTime = Instant.now().toEpochMilli();
        LOGGER.debug("Time taken {} for indexing taskExecutionLogs", endTime - startTime);
        Monitors.recordESIndexTime("index_task_execution_logs", LOG_DOC_TYPE, endTime - startTime);
        Monitors.recordWorkerQueueSize("logQueue", ((ThreadPoolExecutor) logExecutorService).getQueue().size());
    } catch (Exception e) {
        List<String> taskIds = taskExecLogs.stream()
            .map(TaskExecLog::getTaskId)
            .collect(Collectors.toList());
        LOGGER.error("Failed to index task execution logs for tasks: {}", taskIds, e);
    }
}

Source File: ElasticSearchService.java From SpringMVC-Project with MIT License

5 votes

/**
 * 添加文章数据
 */
public void addArticle(Article article) throws IOException {
    IndexRequest indexRequest = new IndexRequest(BLOG_INDEX, ARTICLE_TYPE);
    indexRequest.id(article.getId().toString());
    indexRequest.source(JSON.toJSONString(article), XContentType.JSON);
    IndexResponse indexResponse = restHighLevelClient.index(indexRequest, RequestOptions.DEFAULT);

    logger.info("添加数据 | {}", JSON.toJSONString(indexResponse));
}

Source File: ElasticSearcher.java From jpress with GNU Lesser General Public License v3.0

5 votes

@Override
public void addProduct(Product product) {
    IndexRequest indexRequest = new IndexRequest(index, type, product.getId().toString());
    indexRequest.source(product.toJson(), XContentType.JSON);
    try {
        IndexResponse response = client.index(indexRequest, RequestOptions.DEFAULT);
        if (LogKit.isDebugEnabled()) {
            LogKit.debug(response.toString());
        }
    } catch (Exception e) {
        LOG.error(e.toString(), e);
    }
}

Source File: DefaultIndexRequestFactory.java From samza with Apache License 2.0

5 votes

protected void setSource(OutgoingMessageEnvelope envelope, IndexRequest indexRequest) {
  Object message = envelope.getMessage();
  if (message instanceof byte[]) {
    indexRequest.source((byte[]) message);
  } else if (message instanceof Map) {
    indexRequest.source((Map) message);
  } else {
    throw new SamzaException("Unsupported message type: " + message.getClass().getCanonicalName());
  }
}

Source File: BookingDaoESImp.java From blue-marlin with Apache License 2.0

5 votes

@Override
public String createBooking(Booking booking) throws IOException
{
    Map sourceMap = CommonUtil.convertToMap(booking);
    IndexRequest indexRequest = new IndexRequest(this.bookingsIndex, ES_TYPE);
    indexRequest.source(sourceMap);
    IndexResponse indexResponse = esclient.index(indexRequest, WriteRequest.RefreshPolicy.WAIT_UNTIL.getValue());
    return indexResponse.getId();
}

Source File: Save.java From code with Apache License 2.0

5 votes

/**
 * HttpHost : url地址封装
 * RestClientBuilder： rest客户端构建器
 * RestHighLevelClient： rest高级客户端
 * IndexRequest： 新增或修改请求
 * IndexResponse：新增或修改的响应结果
 */
public static void main(String[] args) throws IOException {
    //1.连接rest接口
    HttpHost http = new HttpHost("127.0.0.1", 9200, "http");
    // rest构建器
    RestClientBuilder builder = RestClient.builder(http);
    // 高级客户端对象
    RestHighLevelClient restHighLevelClient = new RestHighLevelClient(builder);
    //2.封装请求对象
    IndexRequest indexRequest = new IndexRequest("sku", "doc", "3");
    Map<String, Object> skuMap = new HashMap<String, Object>();
    skuMap.put("name", "华为p30pro");
    skuMap.put("brandName", "华为");
    skuMap.put("categoryName", "手机");
    skuMap.put("price", 1010221);
    skuMap.put("createTime", "2019-05-01");
    skuMap.put("saleNum", 101021);
    skuMap.put("commentNum", 10102321);
    Map<String, Object> spec = new HashMap<String, Object>();
    spec.put("网络制式", "移动4G");
    spec.put("屏幕尺寸", "5");
    skuMap.put("spec", spec);
    indexRequest.source(skuMap);
    //3.获取响应结果
    IndexResponse response = restHighLevelClient.index(indexRequest, RequestOptions.DEFAULT);
    int status = response.status().getStatus();
    System.out.println("响应状态：" + status);
    restHighLevelClient.close();
}

Source File: EsUtil.java From bookmark with MIT License

5 votes

/**
 * Description: 插入/更新一条记录
 *
 * @param index  index
 * @param entity 对象
 * @author fanxb
 * @date 2019/7/24 15:02
 */
public void insertOrUpdateOne(String index, EsEntity entity) {
    if (!status) {
        return;
    }
    IndexRequest request = new IndexRequest(index);
    request.id(entity.getId());
    request.source(JSON.toJSONString(entity.getData()), XContentType.JSON);
    try {
        client.index(request, RequestOptions.DEFAULT);
    } catch (Exception e) {
        throw new EsException(e);
    }
}

Source File: LowLevelRestController.java From ProjectStudy with MIT License

5 votes

/**
   * 分词分页查询列表
   *
   * @param page
* @param rows
* @param keyword
   * @return com.example.common.ResponseBean
   * @author wliduo[[email protected]]
   * @date 2019/8/9 15:32
   */
  @GetMapping("/book")
  public ResponseBean getBookList(@RequestParam(defaultValue = "1") Integer page,
                                  @RequestParam(defaultValue = "10") Integer rows,
                                  String keyword) {
      Request request = new Request("POST", new StringBuilder("/_search").toString());
      // 添加Json返回优化
      request.addParameter("pretty", "true");
      // 拼接查询Json
      IndexRequest indexRequest = new IndexRequest();
      XContentBuilder builder = null;
      Response response = null;
      String responseBody = null;
      try {
          builder = JsonXContent.contentBuilder()
                  .startObject()
                  .startObject("query")
                  .startObject("multi_match")
                  .field("query", keyword)
                  .array("fields", new String[]{"name", "desc"})
                  .endObject()
                  .endObject()
                  .startObject("sort")
                  .startObject("id")
                  .field("order", "desc")
                  .endObject()
                  .endObject()
                  .endObject();
          indexRequest.source(builder);
          // 设置请求体并指定ContentType，如果不指定会乱码
          request.setEntity(new NStringEntity(indexRequest.source().utf8ToString(), ContentType.APPLICATION_JSON));
          // 执行HTTP请求
          response = restClient.performRequest(request);
          responseBody = EntityUtils.toString(response.getEntity());
      } catch (IOException e) {
          return new ResponseBean(HttpStatus.NOT_FOUND.value(), "can not found the book by your id", null);
      }
      return new ResponseBean(HttpStatus.OK.value(), "查询成功", JSON.parseObject(responseBody));
  }

Source File: FlsPerfTest.java From deprecated-security-advanced-modules with Apache License 2.0

4 votes

protected void populateData(TransportClient tc) {

        Map<String, Object> indexSettings = new HashMap<>(3);
        indexSettings.put("index.mapping.total_fields.limit",50000);
        indexSettings.put("number_of_shards", 10);
        indexSettings.put("number_of_replicas", 0);

        tc.admin().indices().create(new CreateIndexRequest("deals")
        .settings(indexSettings))
        .actionGet();

        try {

            IndexRequest ir =  new IndexRequest("deals").type("deals2").id("idx1");
            XContentBuilder b = XContentBuilder.builder(JsonXContent.jsonXContent);
            b.startObject();

            b.field("amount",1000);

            b.startObject("xyz");
            b.field("abc","val");
            b.endObject();

            b.endObject();
            ir.source(b);

            tc.index(ir).actionGet();

            for(int i=0; i<1500; i++) {

                ir =  new IndexRequest("deals").type("deals").id("id"+i);
                b = XContentBuilder.builder(JsonXContent.jsonXContent);
                b.startObject();
                for(int j=0; j<2000;j++) {
                    b.field("field"+j,"val"+j);
                }

                b.endObject();
                ir.source(b);

                tc.index(ir).actionGet();

            }

            tc.admin().indices().refresh(new RefreshRequest("deals")).actionGet();
        } catch (IOException e) {
            e.printStackTrace();
            Assert.fail(e.toString());
        }

    }

Source File: ElasticSearchBulkService.java From adaptive-alerting with Apache License 2.0

4 votes

private IndexRequest buildIndexRequest(String json) {
    IndexRequest request = new IndexRequest(INDEX, TYPE);
    request.source(json, XContentType.JSON);
    return request;
}

Source File: ElasticsearchHighRestFactory.java From database-transform-tool with Apache License 2.0

4 votes

public String insert(String index,String type,Object json){
		try {
			if(xclient==null){
				init();
			}
//			XContentBuilder builder = XContentFactory.jsonBuilder();
//			builder.startObject();
//			{
//			    builder.field("user", "kimchy");
//			    builder.field("postDate", new Date());
//			    builder.field("message", "trying out Elasticsearch");
//			}
//			builder.endObject();
			IndexRequest request = new IndexRequest(index, type);
			request.source(JSON.parseObject(JSON.toJSONString(json)),XContentType.JSON);
			IndexResponse response = xclient.index(request);
//			String _index = response.getIndex();
//			String _type = response.getType();
//			String id = response.getId();
//			long version = response.getVersion();
//			if (response.getResult() == DocWriteResponse.Result.CREATED) {
//			    
//			} else if (response.getResult() == DocWriteResponse.Result.UPDATED) {
//			    
//			}
//			ReplicationResponse.ShardInfo shardInfo = response.getShardInfo();
//			if (shardInfo.getTotal() != shardInfo.getSuccessful()) {
//			    
//			}
//			if (shardInfo.getFailed() > 0) {
//			    for (ReplicationResponse.ShardInfo.Failure failure : shardInfo.getFailures()) {
//			        String reason = failure.reason(); 
//			    }
//			}
			return response.toString();
		} catch (Exception e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return null;
	}

Java Code Examples for org.elasticsearch.action.index.IndexRequest#source()