io.searchbox.core.Bulk Java Examples

/**
  * Process the next item in the queue.
  */
 protected void processQueue() {
     try {
         Collection<RequestMetric> batch = new ArrayList<>(this.batchSize);
         RequestMetric rm = queue.take();
         batch.add(rm);
queue.drainTo(batch, this.batchSize - 1);

Builder builder = new Bulk.Builder();
for (RequestMetric metric : batch) {
          Index index = new Index.Builder(metric).refresh(false)
                  .index(getIndexName())
                  .type("request").build(); //$NON-NLS-1$
	builder.addAction(index);
}
         
         BulkResult result = getClient().execute(builder.build());
         if (!result.isSucceeded()) {
             logger.warn("Failed to add metric(s) to ES"); //$NON-NLS-1$
         }
     } catch (Exception e) {
         logger.warn("Error adding metric to ES"); //$NON-NLS-1$
         return;
     }
 }
 

@Override
public BatchBuilder<Bulk> createBatchBuilder() {
    return new BatchBuilder<Bulk>() {

        private final Bulk.Builder builder = new ExtendedBulk.Builder();

        @Override
        public void add(Object item) {
            builder.addAction((BulkableAction) item);
        }

        @Override
        public Bulk build() {
            return builder.build();
        }
    };
}
 

@Override
public Function<Bulk, Boolean> createFailureHandler(FailoverPolicy failover) {
    return bulk -> {
        BufferedBulk bufferedBulk = (BufferedBulk)bulk;
        LOG.warn(String.format("Batch of %s items failed. Redirecting to %s", bufferedBulk.getActions().size(), failover.getClass().getName()));
        try {
            bufferedBulk.getActions().stream()
                    .map(item -> failedItemOps.createItem(((BufferedIndex) item)))
                    .forEach(failover::deliver);
            return true;
        } catch (Exception e) {
            LOG.error("Unable to execute failover", e);
            return false;
        }
    };
}
 

protected JestResultHandler<JestResult> createResultHandler(Bulk bulk, Function<Bulk, Boolean> failureHandler) {
    return new JestResultHandler<JestResult>() {
        @Override
        public void completed(JestResult result) {

            backoffPolicy.deregister(bulk);

            if (!result.isSucceeded()) {
                LOG.warn(result.getErrorMessage());
                failureHandler.apply(bulk);
            }
        }
        @Override
        public void failed(Exception ex) {
            LOG.warn(ex.getMessage(), ex);
            backoffPolicy.deregister(bulk);
            failureHandler.apply(bulk);
        }
    };
}
 

private long doSync( List<DataMap> list,String table,String pk) throws IOException {
    long posi=0;
    //List<Map> list2=new ArrayList<>();
    Bulk.Builder bulk = new Bulk.Builder().defaultIndex(indexName).defaultType(Const.ES_TYPE);
    for(DataMap dataMap:list){
        long id=dataMap.getLong(pk);
        if(id>posi) posi=id;
        Map map=(convertMysql2Es(dataMap));
        logger.info("[full] {}={}",table,map);
        Index index = new Index.Builder(map).id(""+id).build();
        bulk.addAction(index);

    }


    BulkResult br = jest.getJestClient().execute(bulk.build());
    if(!br.isSucceeded()){
        logger.error("error={}, failItems={}",br.getErrorMessage(), JSON.toJSONString(br.getFailedItems()));
        //   br.getFailedItems().get(0).
        throw new RuntimeException("bulk error");
    }
    return  posi;


}
 

@Override
public BatchBuilder<Bulk> createBatchBuilder() {
    return new BatchBuilder<Bulk>() {

        private final BufferedBulk.Builder builder = new BufferedBulk.Builder()
                .withBuffer(pooledItemSourceFactory.createEmptySource())
                .withObjectWriter(objectWriter)
                .withObjectReader(objectReader);

        @Override
        public void add(Object item) {
            builder.addAction((BulkableAction) item);
        }

        @Override
        public Bulk build() {
            return builder.build();
        }

    };
}
 

@Override
public Function<Bulk, Boolean> createFailureHandler(FailoverPolicy failover) {
    return new Function<Bulk, Boolean>() {

        private final JestBatchIntrospector introspector = new JestBatchIntrospector();

        @Override
        public Boolean apply(Bulk bulk) {

            Collection items = introspector.items(bulk);

            LOG.warn(String.format("Batch of %s items failed. Redirecting to %s",
                    items.size(),
                    failover.getClass().getName()));

            items.forEach(item -> {
                Index failedAction = (Index) item;
                failover.deliver(failedItemOps.createItem(failedAction));
            });

            return true;
        }

    };
}
 

@Test
public void builderAddAddsSingleElement() {

    // given
    ExtendedBulk.Builder builder = new ExtendedBulk.Builder();
    BatchIntrospector<Bulk> introspector = new JestBatchIntrospector();

    String source = UUID.randomUUID().toString();
    Index action = new Index.Builder(source).build();

    // when
    builder.addAction(action);

    // then
    ExtendedBulk bulk = builder.build();
    assertEquals(1, introspector.items(bulk).size());

}
 

@Test
public void builderAddCollectionAddsAllElements() {

    // given
    ExtendedBulk.Builder builder = new ExtendedBulk.Builder();
    BatchIntrospector<Bulk> introspector = new JestBatchIntrospector();

    String source = UUID.randomUUID().toString();
    Index action = new Index.Builder(source).build();

    int randomSize = new Random().nextInt(1000) + 10;
    Collection<BulkableAction> actions = new ArrayList<>(randomSize);
    for (int ii = 0; ii < randomSize; ii++) {
        actions.add(action);
    }

    // when
    builder.addAction(actions);

    // then
    ExtendedBulk bulk = builder.build();
    assertEquals(randomSize, introspector.items(bulk).size());

}
 

@Test
public void configReturnsACopyOfServerUrisList() {

    // given
    Builder builder = createTestObjectFactoryBuilder();
    builder.withServerUris("http://localhost:9200;http://localhost:9201;http://localhost:9202");
    ClientObjectFactory<JestClient, Bulk> config = builder.build();

    // when
    Collection<String> serverUrisList = config.getServerList();
    serverUrisList.add("test");

    // then
    assertNotEquals(serverUrisList, config.getServerList());

}
 

@Test
public void failureHandlerExecutesFailoverForEachBatchItemSeparately() {

    // given
    Builder builder = createTestObjectFactoryBuilder();
    ClientObjectFactory<JestClient, Bulk> config = builder.build();

    FailoverPolicy failoverPolicy = Mockito.spy(new NoopFailoverPolicy());

    String payload1 = "test1";
    String payload2 = "test2";
    Bulk bulk = new Bulk.Builder()
            .addAction(spy(new Index.Builder(payload1)).build())
            .addAction(spy(new Index.Builder(payload2)).build())
            .build();

    // when
    config.createFailureHandler(failoverPolicy).apply(bulk);

    // then
    ArgumentCaptor<FailedItemSource> captor = ArgumentCaptor.forClass(FailedItemSource.class);
    verify(failoverPolicy, times(2)).deliver(captor.capture());

    assertTrue(captor.getAllValues().get(0).getSource().equals(payload1));
    assertTrue(captor.getAllValues().get(1).getSource().equals(payload2));
}
 

@Test
public void clientIsCalledWhenListenerIsNotified() {

    // given
    Builder builder = createTestObjectFactoryBuilder();
    ClientObjectFactory<JestClient, Bulk> config = spy(builder.build());

    JestClient mockedJestClient = mock(JestClient.class);
    when(config.createClient()).thenReturn(mockedJestClient);

    FailoverPolicy failoverPolicy = spy(new NoopFailoverPolicy());
    Function<Bulk, Boolean> listener = config.createBatchListener(failoverPolicy);

    String payload1 = "test1";
    String payload2 = "test2";
    Bulk bulk = createTestBatch(payload1, payload2);

    // when
    listener.apply(bulk);

    // then
    ArgumentCaptor<Bulk> captor = ArgumentCaptor.forClass(Bulk.class);
    verify(mockedJestClient, times(1)).executeAsync((Bulk) captor.capture(), Mockito.any());

    assertEquals(bulk, captor.getValue());
}
 

@Test
public void failoverIsExecutedAfterNonSuccessfulRequest() {

    // given
    Builder builder = createTestObjectFactoryBuilder();
    JestHttpObjectFactory config = spy(builder.build());

    String payload1 = "test1";
    String payload2 = "test2";
    Bulk bulk = createTestBatch(payload1, payload2);

    Function<Bulk, Boolean> failoverHandler = mock(Function.class);
    JestResultHandler<JestResult> resultHandler = config.createResultHandler(bulk, failoverHandler);

    JestResult result = mock(JestResult.class);
    when(result.isSucceeded()).thenReturn(false);

    // when
    resultHandler.completed(result);

    // then
    ArgumentCaptor<Bulk> captor = ArgumentCaptor.forClass(Bulk.class);
    verify(failoverHandler, times(1)).apply(captor.capture());

    assertEquals(bulk, captor.getValue());
}
 

@Test
public void failoverIsNotExecutedAfterSuccessfulRequest() {

    // given
    Builder builder = createTestObjectFactoryBuilder();
    JestHttpObjectFactory config = spy(builder.build());

    String payload1 = "test1";
    String payload2 = "test2";
    Bulk bulk = createTestBatch(payload1, payload2);

    Function<Bulk, Boolean> failoverHandler = mock(Function.class);
    JestResultHandler<JestResult> resultHandler = config.createResultHandler(bulk, failoverHandler);

    JestResult result = mock(JestResult.class);
    when(result.isSucceeded()).thenReturn(true);

    // when
    resultHandler.completed(result);

    // then
    verify(failoverHandler, never()).apply(Mockito.any(Bulk.class));
}
 

@Test
public void failoverIsExecutedAfterFailedRequest() {

    // given
    Builder builder = createTestObjectFactoryBuilder();
    JestHttpObjectFactory config = spy(builder.build());

    String payload1 = "test1";
    String payload2 = "test2";
    Bulk bulk = createTestBatch(payload1, payload2);

    Function<Bulk, Boolean> failoverHandler = mock(Function.class);
    JestResultHandler<JestResult> resultHandler = config.createResultHandler(bulk, failoverHandler);

    // when
    resultHandler.failed(new IOException());

    // then
    ArgumentCaptor<Bulk> captor = ArgumentCaptor.forClass(Bulk.class);
    verify(failoverHandler, times(1)).apply(captor.capture());

    assertEquals(bulk, captor.getValue());
}
 

@Test
public void failoverHandlerIsNotExecutedImmediatelyIfBackoffPolicyShouldNotApply() {

    // given
    BackoffPolicy<AbstractAction<BulkResult>> backoffPolicy = mock(BackoffPolicy.class);
    when(backoffPolicy.shouldApply(any())).thenReturn(false);

    Builder builder = createTestObjectFactoryBuilder();
    builder.withBackoffPolicy(backoffPolicy);

    JestHttpObjectFactory config = spy(builder.build());

    String payload1 = "test1";
    Bulk bulk = createTestBatch(payload1);

    FailoverPolicy failoverPolicy = mock(FailoverPolicy.class);
    Function<Bulk, Boolean> listener = config.createBatchListener(failoverPolicy);

    // when
    listener.apply(bulk);

    // then
    ArgumentCaptor<FailedItemSource> captor = ArgumentCaptor.forClass(FailedItemSource.class);
    verify(failoverPolicy, never()).deliver(captor.capture());

}
 

@Test
public void configReturnsACopyOfServerUrisList() {

    // given
    BufferedJestHttpObjectFactory.Builder builder = createTestObjectFactoryBuilder();
    builder.withServerUris("http://localhost:9200;http://localhost:9201;http://localhost:9202");
    ClientObjectFactory<JestClient, Bulk> config = builder.build();

    // when
    Collection<String> serverUrisList = config.getServerList();
    serverUrisList.add("test");

    // then
    assertNotEquals(serverUrisList, config.getServerList());

}
 

@Test
public void clientIsCalledWhenListenerIsNotified() {

    // given
    BufferedJestHttpObjectFactory.Builder builder = createTestObjectFactoryBuilder();
    ClientObjectFactory<JestClient, Bulk> config = spy(builder.build());

    JestClient mockedJestClient = mock(JestClient.class);
    when(config.createClient()).thenReturn(mockedJestClient);

    FailoverPolicy failoverPolicy = spy(new NoopFailoverPolicy());
    Function<Bulk, Boolean> listener = config.createBatchListener(failoverPolicy);

    ItemSource<ByteBuf> payload1 = createDefaultTestBuffereItemSource("test1");
    ItemSource<ByteBuf> payload2 = createDefaultTestBuffereItemSource("test2");
    Bulk bulk = createTestBatch(payload1, payload2);

    // when
    listener.apply(bulk);

    // then
    ArgumentCaptor<Bulk> captor = ArgumentCaptor.forClass(Bulk.class);
    verify(mockedJestClient, times(1)).executeAsync((Bulk) captor.capture(), Mockito.any());

    assertEquals(bulk, captor.getValue());
}
 

@Test
public void failoverIsNotExecutedAfterSuccessfulRequest() {

    // given
    BufferedJestHttpObjectFactory.Builder builder = createTestObjectFactoryBuilder();
    BufferedJestHttpObjectFactory config = spy(builder.build());

    ItemSource<ByteBuf> payload1 = createDefaultTestBuffereItemSource("test1");
    ItemSource<ByteBuf> payload2 = createDefaultTestBuffereItemSource("test2");
    Bulk bulk = createTestBatch(payload1, payload2);

    Function<Bulk, Boolean> failoverHandler = mock(Function.class);
    JestResultHandler<JestResult> resultHandler = config.createResultHandler(bulk, failoverHandler);

    JestResult result = mock(JestResult.class);
    when(result.isSucceeded()).thenReturn(true);

    // when
    resultHandler.completed(result);

    // then
    verify(failoverHandler, never()).apply(Mockito.any(Bulk.class));
}
 

@Test
public void failoverIsExecutedAfterFailedRequest() {

    // given
    BufferedJestHttpObjectFactory.Builder builder = createTestObjectFactoryBuilder();
    BufferedJestHttpObjectFactory config = spy(builder.build());

    ItemSource<ByteBuf> payload1 = createDefaultTestBuffereItemSource("test1");
    ItemSource<ByteBuf> payload2 = createDefaultTestBuffereItemSource("test2");

    Bulk bulk = createTestBatch(payload1, payload2);

    Function<Bulk, Boolean> failoverHandler = mock(Function.class);
    JestResultHandler<JestResult> resultHandler = config.createResultHandler(bulk, failoverHandler);

    // when
    resultHandler.failed(new IOException());

    // then
    ArgumentCaptor<Bulk> captor = ArgumentCaptor.forClass(Bulk.class);
    verify(failoverHandler, times(1)).apply(captor.capture());
    verify((BufferedBulk)bulk, times(1)).completed();

    assertEquals(bulk, captor.getValue());

}
 

@Test
public void bulkContainsAddedStringItem() {

    // given
    BatchOperations<Bulk> bulkOperations = JestHttpObjectFactoryTest.createTestObjectFactoryBuilder().build().createBatchOperations();
    BatchBuilder<Bulk> batchBuilder = bulkOperations.createBatchBuilder();

    String testPayload = "{ \"testfield\": \"testvalue\" }";
    Index item = (Index) bulkOperations.createBatchItem("testIndex", testPayload);

    // when
    batchBuilder.add(item);
    Bulk bulk = batchBuilder.build();

    // then
    JestBatchIntrospector introspector = new JestBatchIntrospector();
    AbstractAction action = (AbstractAction) introspector.items(bulk).iterator().next();
    assertEquals(testPayload, introspector.itemIntrospector().getPayload(action));

}
 

@Test
public void bulkContainsAddedSourceItem() {

    // given
    BatchOperations<Bulk> bulkOperations = JestHttpObjectFactoryTest.createTestObjectFactoryBuilder().build().createBatchOperations();
    BatchBuilder<Bulk> batchBuilder = bulkOperations.createBatchBuilder();

    String testPayload = "{ \"testfield\": \"testvalue\" }";
    StringItemSource itemSource = spy(new StringItemSource(testPayload));
    Index item = (Index) bulkOperations.createBatchItem("testIndex", itemSource);

    // when
    batchBuilder.add(item);
    Bulk bulk = batchBuilder.build();

    // then
    verify(itemSource, times(2)).getSource();
    JestBatchIntrospector introspector = new JestBatchIntrospector();
    AbstractAction action = (AbstractAction) introspector.items(bulk).iterator().next();
    assertEquals(testPayload, introspector.itemIntrospector().getPayload(action));

}
 

@Test
public void defaultJestBulkOperationsSetsDefaultMappingType() {

    // given
    BatchOperations<Bulk> bulkOperations = new JestBulkOperations();

    String testPayload = "{ \"testfield\": \"testvalue\" }";
    StringItemSource itemSource = spy(new StringItemSource(testPayload));
    Index item = (Index) bulkOperations.createBatchItem("testIndex", itemSource);

    // when
    String type = item.getType();

    // then
    assertEquals("index", type);

}
 

@Test
public void mappingTypeCanBeSet() {

    // given
    String expectedMappingType = UUID.randomUUID().toString();
    BatchOperations<Bulk> bulkOperations = new JestBulkOperations(expectedMappingType);

    String testPayload = "{ \"testfield\": \"testvalue\" }";
    StringItemSource itemSource = spy(new StringItemSource(testPayload));
    Index item = (Index) bulkOperations.createBatchItem("testIndex", itemSource);

    // when
    String type = item.getType();

    // then
    assertEquals(expectedMappingType, type);

}
 

@Test
public void executeAsyncDelegatesToConfiguredAsyncClient() {

    // given
    BufferedJestHttpClient client = spy(createDefaultTestHttpClient());
    CloseableHttpAsyncClient asyncClient = mockAsyncClient(client);

    Bulk bulk = createDefaultTestBufferedBulk();

    // when
    client.executeAsync(bulk, createMockTestResultHandler());

    // then
    verify(client).getAsyncClient();
    verify(asyncClient).execute(any(HttpUriRequest.class), any());

}
 

public void indexLocations(List<Location> locations) {
    if (locations != null && !locations.isEmpty()) {
        log.info("Indexing locations with Elasticsearch Jest....");

        Bulk.Builder builder = new Bulk.Builder();

        for (Location location : locations) {
            builder.addAction(new Index.Builder(location).index(LOCATION_INDEX_NAME).type(LOCATION_INDEX_TYPE).build());
        }

        Bulk bulk = builder.build();

        try {
            JestResult result = jestClient.execute(bulk);
            log.info("Bulk search success: " + result.isSucceeded());
        } catch (Exception e) {
            log.error(e.getMessage(), e);
        }

        log.info(String.format("Indexed %d documents", locations.size()));
    }
}
 

@Override
public JestResult deleteBulk(String clustName, String indexName, String Type, List<SearchResultDetailVO> results) {
    JestResult result = null ;
    try {
        Bulk.Builder bulkBulder = new Bulk.Builder().defaultIndex(indexName).defaultType(Type);
        if(CollectionUtils.isNotEmpty(results)){
            for (SearchResultDetailVO resultDetailVO:results){
                bulkBulder.addAction(new Delete.Builder(resultDetailVO.getId()).index(indexName).type(Type).build());
            }
        }
        result = JestManager.getJestClient(clustName).execute(bulkBulder.build());
    } catch (Exception e) {
        e.printStackTrace();
        LOGGER.error("deleteBulk失败:",e);
    }
    return result ;

}
 

@Test
public void indexDocumentIterationsTest() throws IndexerRequestException, IndexerNotAvailableException {
    List<DocumentIteration> documentIterations = new ArrayList<>();
    documentIterations.add(new DocumentIteration());
    documentIterations.add(new DocumentIteration());

    indexerManagerBean.indexDocumentIterations(documentIterations);

    Mockito.verify(indexManager,times(1))
            .sendBulk(Matchers.any(Bulk.Builder.class));

}
 

@Test
public void prepareRequestCreatesRequestWithContentTypeHeader() throws IOException {

    // given
    Bulk bulk = createDefaultTestBufferedBulk();

    // when
    BufferedJestHttpClient client = createDefaultTestHttpClient();
    HttpUriRequest request = client.prepareRequest((BufferedBulk) bulk);

    // then
    HttpEntity entity = ((HttpEntityEnclosingRequest) request).getEntity();
    Assert.assertEquals(ContentType.APPLICATION_JSON.toString(), entity.getContentType().getValue());

}
 

private Bulk createTestBatch(BufferedBulk.Builder builder, ItemSource<ByteBuf>... payloads) {

        builder.withObjectWriter(createDefaultTestObjectWriter());
        builder.withObjectReader(createDefaultTestObjectReader());

        for (ItemSource<ByteBuf> payload : payloads) {
            builder.addAction(spy(new BufferedIndex.Builder(payload))
                    .index(UUID.randomUUID().toString())
                    .build());
        }
        return spy(builder.build());
    }