Java Code Examples for org.elasticsearch.action.bulk.BulkRequestBuilder#add()

@Override
public void run() {
	logger.info("Starting thread with {} keys", this.keys.rowColumnMap.size());
              if (closed) {
                  return;
              }
              
              BulkRequestBuilder bulk = client.prepareBulk();
              for(String key : this.keys.rowColumnMap.keySet()){
              	
  				try {
  					String id = UUID.nameUUIDFromBytes(key.getBytes()).toString();
  					bulk.add(indexRequest(this.indexName).type(this.typeName)
					    							 .id(id)
					    							 .source(this.keys.rowColumnMap.get(key)));
  				} catch (Exception e) {
  					logger.error("failed to entry to bulk indexing");
  				}
  				
              	if(bulk.numberOfActions() >= this.batchSize){
              		saveToEs(bulk);
  					bulk = client.prepareBulk();
              	}
              }
}
 

@Before
public void before() throws IOException {
  final IndicesAdminClient indices = client.admin().indices();
  if(!indices.prepareExists(INDEX).execute().actionGet().isExists()) {
    indices.prepareCreate(INDEX).execute().actionGet();
  }
  final JsonSerializer<Person> serializer = mapper.serializer(Person.class);
  final BulkRequestBuilder bulk = client.prepareBulk();
  for (int i = 0; i < SIZE; i++) {

    final String name = UUID.randomUUID().toString();
    final IndexRequest request = new IndexRequest(INDEX, TYPE);
    request.source(serializer.apply(Person.builder().id("").firstname(name).lastname(name).build()), JSON);
    bulk.add(request);
  }

  client.bulk(bulk.request()).actionGet();
  flush(INDEX);
}
 

private static void BulkIndex(Client client) throws IOException {

        BulkRequestBuilder requestBuilder = client.prepareBulk();

        requestBuilder.add(client.prepareIndex("twitter","tweet","4")
                .setSource(jsonBuilder()
                        .startObject()
                        .field("user","niekaijie")
                        .field("school","beiyou")
                        .field("address","haidianqu")
                        .endObject())
        );
        requestBuilder.add(client.prepareIndex("twitter","tweet","3")
                .setSource(jsonBuilder()
                        .startObject()
                        .field("user","林志颖aa")
                        .field("school","台湾大学")
                        .field("address","台湾")
                        .endObject())
        );
        BulkResponse response = requestBuilder.get();


    }
 

@Override
public String bulkDelete(String index, String type, String... ids) {
	try {
		if(client==null){
			init();
		}
		BulkRequestBuilder bulkRequest = client.prepareBulk();
		for (String id : ids) {
			bulkRequest.add(client.prepareDelete(index, type, id));
		}
		BulkResponse result = bulkRequest.execute().get();
		return result.toString();
	}catch (Exception e) {
		// TODO Auto-generated catch block
		e.printStackTrace();
	}
	return null;
}
 

@Override
public void storeSpan(String tenantId, List<Span> spans, Function<Span, String> spanIdSupplier)
        throws StoreException {

    client.initTenant(tenantId);

    BulkRequestBuilder bulkRequestBuilder = client.getClient().prepareBulk();

    for (Span span : spans) {
        String json;
        try {
            json = serialize(span);
        } catch(IOException ex){
            log.errorf("Failed to serialize span %s", span);
            throw new StoreException(ex);
        }

        log.tracef("Storing span: %s", json);
        // modified id is used in index
        final String modifiedId = spanIdSupplier.apply(span);

        bulkRequestBuilder.add(client.getClient()
                .prepareIndex(client.getIndex(tenantId), SPAN_TYPE, modifiedId)
                .setSource(json));
    }

    BulkResponse bulkItemResponses = bulkRequestBuilder.execute().actionGet();

    if (bulkItemResponses.hasFailures()) {
        log.tracef("Failed to store spans to elasticsearch: %s", bulkItemResponses.buildFailureMessage());
        throw new StoreException(bulkItemResponses.buildFailureMessage());
    }

    log.trace("Success storing spans to elasticsearch");
}
 

public int updateBatchData(ESBasicInfo esBasicInfo, Object object) throws IOException {
    BulkRequestBuilder bulkRequest = esClient.prepareBulk();

    for (String id : esBasicInfo.getIds()) {
        bulkRequest.add(esClient.prepareUpdate(esBasicInfo.getIndex(), esBasicInfo.getType(), id)
            .setDoc(mapper.writeValueAsString(object), XContentType.JSON));
    }

    bulkRequest.execute().actionGet();

    return bulkRequest.numberOfActions();
}
 

@Override
public final void write(List<? extends Document> documents) throws Exception {
    BulkRequestBuilder bulkRequest = client.prepareBulk();

    for (Document doc : documents) {
        XContentParser parser = null;
        parser = XContentFactory.xContent(XContentType.JSON)
                .createParser(NamedXContentRegistry.EMPTY,
                        DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
                        doc.getOutputData().getBytes());
        parser.close();
        XContentBuilder builder = jsonBuilder().copyCurrentStructure(parser);


        IndexRequestBuilder request = client.prepareIndex(
                indexName,
                typeName).setSource(
                builder);
        request.setId(doc.getPrimaryKeyFieldValue());
        bulkRequest.add(request);
    }
    //check that no nonessential processes failed
    if (documents.size() != 0) {
        BulkResponse response;
        response = bulkRequest.execute().actionGet(timeout);
        getResponses(response);
    }
}
 

public int addBatchData(ESBasicInfo esBasicInfo, Object object) throws IOException {
    BulkRequestBuilder bulkRequest = esClient.prepareBulk();

    for (String id : esBasicInfo.getIds()) {
        bulkRequest.add(esClient.prepareIndex(esBasicInfo.getIndex(), esBasicInfo.getType(), id)
            .setSource(mapper.writeValueAsString(object), XContentType.JSON));
    }
    bulkRequest.execute().actionGet();

    return bulkRequest.numberOfActions();
}
 

public int updateBatchData(ESBasicInfo esBasicInfo, Object object) throws IOException {
    BulkRequestBuilder bulkRequest = esClient.prepareBulk();

    for (String id : esBasicInfo.getIds()) {
        bulkRequest.add(esClient.prepareUpdate(esBasicInfo.getIndex(), esBasicInfo.getType(), id)
            .setDoc(mapper.writeValueAsString(object), XContentType.JSON));
    }

    bulkRequest.execute().actionGet();

    return bulkRequest.numberOfActions();
}
 

public int deleteBatchData(ESBasicInfo esBasicInfo) {
    BulkRequestBuilder bulkRequest = esClient.prepareBulk();

    for (String id : esBasicInfo.getIds()) {
        bulkRequest.add(esClient.prepareDelete(esBasicInfo.getIndex(), esBasicInfo.getType(), id));
    }

    BulkResponse response = bulkRequest.execute().actionGet();
    log.info("status is:{}", response.status().getStatus());

    return bulkRequest.numberOfActions();
}
 

@Override
public void storeAll(Map<String, Table> map) {
    if(map == null) {
        throw new TableMapStoreException("Illegal Store Request - Null Map");
    }
    if(map.containsKey(null)) {
        throw new TableMapStoreException("Illegal Store Request - Null Key is Present");
    }

    logger.info("Store all called for multiple values");
    BulkRequestBuilder bulkRequestBuilder = elasticsearchConnection.getClient()
            .prepareBulk()
            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
    for(Map.Entry<String, Table> mapEntry : map.entrySet()) {
        try {
            if(mapEntry.getValue() == null) {
                throw new TableMapStoreException(
                        String.format("Illegal Store Request - Object is Null for Table - %s", mapEntry.getKey()));
            }
            Map<String, Object> sourceMap = ElasticsearchQueryUtils.toMap(objectMapper, mapEntry.getValue());
            bulkRequestBuilder.add(elasticsearchConnection.getClient()
                                           .prepareIndex(TABLE_META_INDEX, TABLE_META_TYPE, mapEntry.getKey())
                                           .setSource(sourceMap));
        } catch (Exception e) {
            throw new TableMapStoreException("Error bulk saving meta: ", e);
        }
    }
    bulkRequestBuilder.execute()
            .actionGet();
}
 

@Test
    public void testForClient() throws Exception {
        BulkRequestBuilder bulkRequest = client.prepareBulk();

// either use client#prepare, or use Requests# to directly build index/delete requests
        bulkRequest.add(client.prepareIndex("twitter", "tweet", "1")
                .setSource(XContentFactory.jsonBuilder()
                        .startObject()
                        .field("user", "kimchy")
                        .field("postDate", new Date())
                        .field("message", "trying out Elasticsearch")
                        .endObject()
                )
        );

        bulkRequest.add(client.prepareIndex("twitter", "tweet", "2")
                .setSource(XContentFactory.jsonBuilder()
                        .startObject()
                        .field("user", "kimchy")
                        .field("postDate", new Date())
                        .field("message", "another post")
                        .endObject()
                )
        );

        BulkResponse bulkResponse = bulkRequest.get();
        if (bulkResponse.hasFailures()) {
            // process failures by iterating through each bulk response item
        }
    }
 

@Override
public void doReindex(Client client, GitScm gitScm, GlobalSettings globalSettings) {
    if (!globalSettings.getIndexingEnabled()) {
        return;
    }
    deleteLatestIndexedNote(client);
    while (true) {
        try {
            SearchRequestBuilder esReq = client.prepareSearch(ES_UPDATEALIAS)
                .setSize(1000)
                .setFetchSource(false)
                .setRouting(getRepoDesc())
                .setQuery(filteredQuery(matchAllQuery(), andFilter(
                    sfu.projectRepositoryFilter(
                        repository.getProject().getKey(), repository.getSlug()),
                    sfu.exactRefFilter(ref))));
            BulkRequestBuilder bulkDelete = client.prepareBulk().setRefresh(true);
            for (SearchHit hit : esReq.get().getHits().getHits()) {
                bulkDelete.add(buildDeleteFromRef(client, hit.getType(), hit.getId()));
            }
            if (bulkDelete.numberOfActions() == 0) {
                break;
            }
            bulkDelete.get();
        } catch (Exception e) {
            log.error("Could not delete documents for {}, aborting", toString(), e);
            return;
        }
    }
    doUpdate(client, gitScm, globalSettings);
}
 

@Override
protected int delegateQueryDelete(final ConditionBean cb, final DeleteOption<? extends ConditionBean> option) {
    final SearchRequestBuilder builder = client.prepareSearch(asEsIndex()).setScroll(scrollForDelete).setSize(sizeForDelete);
    final EsAbstractConditionBean esCb = (EsAbstractConditionBean) cb;
    if (esCb.getPreference() != null) {
        esCb.setPreference(esCb.getPreference());
    }
    esCb.request().build(builder);
    SearchResponse response = esCb.build(builder).execute().actionGet(scrollSearchTimeout);
    String scrollId = response.getScrollId();
    int count = 0;
    try {
        while (scrollId != null) {
            final SearchHits searchHits = getSearchHits(response);
            final SearchHit[] hits = searchHits.getHits();
            if (hits.length == 0) {
                break;
            }

            final BulkRequestBuilder bulkRequest = client.prepareBulk();
            for (final SearchHit hit : hits) {
                bulkRequest.add(client.prepareDelete().setIndex(asEsIndex()).setId(hit.getId()));
            }
            count += hits.length;
            final BulkResponse bulkResponse = bulkRequest.execute().actionGet(bulkTimeout);
            if (bulkResponse.hasFailures()) {
                throw new IllegalBehaviorStateException(bulkResponse.buildFailureMessage());
            }

            response = client.prepareSearchScroll(scrollId).setScroll(scrollForDelete).execute().actionGet(scrollSearchTimeout);
            if (!scrollId.equals(response.getScrollId())) {
                deleteScrollContext(scrollId);
            }
        }
    } finally {
        deleteScrollContext(scrollId);
    }
    return count;
}
 

public BulkResponse bulkDelete(String index, String type, List<String> ids) {
    BulkRequestBuilder bulkRequest = client.prepareBulk();
    for (String id : ids) {
        bulkRequest.add(client.prepareDelete(index, type, id));
    }
    return bulkRequest.execute().actionGet();
}
 

@Override
public void doOperation( final Client client, final BulkRequestBuilder bulkRequest ) {


    for ( final String index : indexes ) {
        final DeleteRequestBuilder builder =
                client.prepareDelete( index, IndexingUtils.ES_ENTITY_TYPE, documentId );
        bulkRequest.add( builder );
    }
}
 

private void sendToLocalCluster(final String scrollId, final SearchHit[] hits) {

            // prepare bulk request
            final BulkRequestBuilder bulkRequest = client.prepareBulk();
            for (final SearchHit hit : hits) {
                IndexRequestBuilder builder = client.prepareIndex(toIndex,
                        toType != null ? toType : hit.getType(), hit.getId())
                        .setSource(hit.getSource());
                Map<String, SearchHitField> fields = hit.getFields();
                if (fields != null && fields.containsKey("_parent")) {
                    SearchHitField parentField = fields.get("_parent");
                    if (parentField != null) {
                        String parentId = parentField.getValue();
                        builder.setParent(parentId);
                    }
                }
                bulkRequest.add(builder);
            }

            // send bulk request, if success response got, searching the next 10 results using scroll_id
            // using this listener (inner class) to listen to results
            bulkRequest.execute(new ActionListener<BulkResponse>() {
                @Override
                public void onResponse(final BulkResponse bulkResponse) {
                    if (bulkResponse.hasFailures()) {
                        throw new ReindexingException(bulkResponse
                                .buildFailureMessage());
                    }
                    client.prepareSearchScroll(scrollId).setScroll(scroll)
                            .execute(ReindexingListener.this);
                }

                @Override
                public void onFailure(final Throwable e) {
                    ReindexingListener.this.onFailure(e);
                }
            });
        }
 

@Before
public void createTestIndex() throws Exception {
   // Delete any previously indexed content.
   client = client();
   if (!client.admin().indices().prepareExists(INDEX_TEST).get().isExists()) {
      String testTemplate =
          "{" +
              "  \"test\": {" +
              "    \"properties\": {" +
              "      \"url\": { \"type\": \"text\" }," +
              "      \"title\": { \"type\": \"text\" }," +
              "      \"content\": { \"type\": \"text\" }," +
              "      \"lang\": { \"type\": \"text\" }," +
              "      \"rndlang\": { \"type\": \"text\" }" +
              "    }" +
              "  }" +
              "}";

      String emptyTemplate =
          "{" +
              "  \"empty\": {" +
              "    \"properties\": {" +
              "      \"url\": { \"type\": \"text\" }," +
              "      \"title\": { \"type\": \"text\" }," +
              "      \"content\": { \"type\": \"text\" }," +
              "      \"lang\": { \"type\": \"text\" }," +
              "      \"rndlang\": { \"type\": \"text\" }" +
              "    }" +
              "  }" +
              "}";

      CreateIndexResponse response = client.admin().indices()
          .prepareCreate(INDEX_TEST)
          .addMapping("test", testTemplate, XContentType.JSON)
          .get();
      Assertions.assertThat(response.isAcknowledged()).isTrue();

      response = client.admin().indices()
          .prepareCreate(INDEX_EMPTY)
          .addMapping("empty", emptyTemplate, XContentType.JSON)
          .get();
      Assertions.assertThat(response.isAcknowledged()).isTrue();

      // Create content at random in the test index.
      Random rnd = random();
      String[] languages = LanguageComponents.languages().toArray(new String[0]);
      Arrays.sort(languages);

      BulkRequestBuilder bulk = client.prepareBulk();
      for (String[] data : SampleDocumentData.SAMPLE_DATA) {
         bulk.add(client.prepareIndex()
             .setIndex(INDEX_TEST)
             .setType("test")
             .setSource(XContentFactory.jsonBuilder()
                 .startObject()
                 .field("url", data[0])
                 .field("title", data[1])
                 .field("content", data[2])
                 .field("lang", "English")
                 .field("rndlang", languages[rnd.nextInt(languages.length)])
                 .endObject()));
      }

      bulk.add(client.prepareIndex()
          .setIndex(INDEX_EMPTY)
          .setType("empty")
          .setSource(XContentFactory.jsonBuilder()
              .startObject()
              .field("url", "")
              .field("title", "")
              .field("content", "")
              .endObject()));

      bulk.execute().actionGet();
      flushAndRefresh(INDEX_TEST);
      flushAndRefresh(INDEX_EMPTY);
   }
   ensureGreen(INDEX_TEST);
   ensureGreen(INDEX_EMPTY);

   InetSocketAddress endpoint = randomFrom(cluster().httpAddresses());
   this.restBaseUrl = "http://" + NetworkAddress.format(endpoint);
}
 

@Test
public void test04() {
    BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();
    bulkRequestBuilder.add(transportClient.prepareDelete("test", "osm", "2_4"));
    commit(bulkRequestBuilder);
}
 

private void remove( BulkRequestBuilder bulkBuilder, String identity )
{
    bulkBuilder.add( support.client().
        prepareDelete( support.index(), support.entitiesType(), identity ) );
}