io.searchbox.core.Index Java Examples

@Test
public void builderAddAddsSingleElement() {

    // given
    ExtendedBulk.Builder builder = new ExtendedBulk.Builder();
    BatchIntrospector<Bulk> introspector = new JestBatchIntrospector();

    String source = UUID.randomUUID().toString();
    Index action = new Index.Builder(source).build();

    // when
    builder.addAction(action);

    // then
    ExtendedBulk bulk = builder.build();
    assertEquals(1, introspector.items(bulk).size());

}
 

public void indexLocations(List<Location> locations) {
    if (locations != null && !locations.isEmpty()) {
        log.info("Indexing locations with Elasticsearch Jest....");

        Bulk.Builder builder = new Bulk.Builder();

        for (Location location : locations) {
            builder.addAction(new Index.Builder(location).index(LOCATION_INDEX_NAME).type(LOCATION_INDEX_TYPE).build());
        }

        Bulk bulk = builder.build();

        try {
            JestResult result = jestClient.execute(bulk);
            log.info("Bulk search success: " + result.isSucceeded());
        } catch (Exception e) {
            log.error(e.getMessage(), e);
        }

        log.info(String.format("Indexed %d documents", locations.size()));
    }
}
 

@Test
public void mappingTypeCanBeSet() {

    // given
    String expectedMappingType = UUID.randomUUID().toString();
    BatchOperations<Bulk> bulkOperations = new JestBulkOperations(expectedMappingType);

    String testPayload = "{ \"testfield\": \"testvalue\" }";
    StringItemSource itemSource = spy(new StringItemSource(testPayload));
    Index item = (Index) bulkOperations.createBatchItem("testIndex", itemSource);

    // when
    String type = item.getType();

    // then
    assertEquals(expectedMappingType, type);

}
 

@Test
public void defaultJestBulkOperationsSetsDefaultMappingType() {

    // given
    BatchOperations<Bulk> bulkOperations = new JestBulkOperations();

    String testPayload = "{ \"testfield\": \"testvalue\" }";
    StringItemSource itemSource = spy(new StringItemSource(testPayload));
    Index item = (Index) bulkOperations.createBatchItem("testIndex", itemSource);

    // when
    String type = item.getType();

    // then
    assertEquals("index", type);

}
 

@Test
public void bulkContainsAddedSourceItem() {

    // given
    BatchOperations<Bulk> bulkOperations = JestHttpObjectFactoryTest.createTestObjectFactoryBuilder().build().createBatchOperations();
    BatchBuilder<Bulk> batchBuilder = bulkOperations.createBatchBuilder();

    String testPayload = "{ \"testfield\": \"testvalue\" }";
    StringItemSource itemSource = spy(new StringItemSource(testPayload));
    Index item = (Index) bulkOperations.createBatchItem("testIndex", itemSource);

    // when
    batchBuilder.add(item);
    Bulk bulk = batchBuilder.build();

    // then
    verify(itemSource, times(2)).getSource();
    JestBatchIntrospector introspector = new JestBatchIntrospector();
    AbstractAction action = (AbstractAction) introspector.items(bulk).iterator().next();
    assertEquals(testPayload, introspector.itemIntrospector().getPayload(action));

}
 

@Test
public void bulkContainsAddedStringItem() {

    // given
    BatchOperations<Bulk> bulkOperations = JestHttpObjectFactoryTest.createTestObjectFactoryBuilder().build().createBatchOperations();
    BatchBuilder<Bulk> batchBuilder = bulkOperations.createBatchBuilder();

    String testPayload = "{ \"testfield\": \"testvalue\" }";
    Index item = (Index) bulkOperations.createBatchItem("testIndex", testPayload);

    // when
    batchBuilder.add(item);
    Bulk bulk = batchBuilder.build();

    // then
    JestBatchIntrospector introspector = new JestBatchIntrospector();
    AbstractAction action = (AbstractAction) introspector.items(bulk).iterator().next();
    assertEquals(testPayload, introspector.itemIntrospector().getPayload(action));

}
 

/**
 * Stores a "dataversion" record in the ES store.  There is only a single one of these.  The
 * return value of the add will include the version number of the entity.  This version
 * number is what we use to determine whether our cache is stale.
 */
protected void updateDataVersion() {
    DataVersionBean dv = new DataVersionBean();
    dv.setUpdatedOn(System.currentTimeMillis());
    Index index = new Index.Builder(dv).refresh(false)
            .index(getDefaultIndexName())
            .type("dataVersion").id("instance").build(); //$NON-NLS-1$ //$NON-NLS-2$
    getClient().executeAsync(index, new JestResultHandler<JestResult>() {
        @Override
        public void completed(JestResult result) {
            dataVersion = null;
        }
        @Override
        public void failed(Exception e) {
            dataVersion = null;
        }
    });
}
 

@Test
public void failureHandlerExecutesFailoverForEachBatchItemSeparately() {

    // given
    Builder builder = createTestObjectFactoryBuilder();
    ClientObjectFactory<JestClient, Bulk> config = builder.build();

    FailoverPolicy failoverPolicy = Mockito.spy(new NoopFailoverPolicy());

    String payload1 = "test1";
    String payload2 = "test2";
    Bulk bulk = new Bulk.Builder()
            .addAction(spy(new Index.Builder(payload1)).build())
            .addAction(spy(new Index.Builder(payload2)).build())
            .build();

    // when
    config.createFailureHandler(failoverPolicy).apply(bulk);

    // then
    ArgumentCaptor<FailedItemSource> captor = ArgumentCaptor.forClass(FailedItemSource.class);
    verify(failoverPolicy, times(2)).deliver(captor.capture());

    assertTrue(captor.getAllValues().get(0).getSource().equals(payload1));
    assertTrue(captor.getAllValues().get(1).getSource().equals(payload2));
}
 

@Test
public void builderAddCollectionAddsAllElements() {

    // given
    ExtendedBulk.Builder builder = new ExtendedBulk.Builder();
    BatchIntrospector<Bulk> introspector = new JestBatchIntrospector();

    String source = UUID.randomUUID().toString();
    Index action = new Index.Builder(source).build();

    int randomSize = new Random().nextInt(1000) + 10;
    Collection<BulkableAction> actions = new ArrayList<>(randomSize);
    for (int ii = 0; ii < randomSize; ii++) {
        actions.add(action);
    }

    // when
    builder.addAction(actions);

    // then
    ExtendedBulk bulk = builder.build();
    assertEquals(randomSize, introspector.items(bulk).size());

}
 

/**
  * Process the next item in the queue.
  */
 protected void processQueue() {
     try {
         Collection<RequestMetric> batch = new ArrayList<>(this.batchSize);
         RequestMetric rm = queue.take();
         batch.add(rm);
queue.drainTo(batch, this.batchSize - 1);

Builder builder = new Bulk.Builder();
for (RequestMetric metric : batch) {
          Index index = new Index.Builder(metric).refresh(false)
                  .index(getIndexName())
                  .type("request").build(); //$NON-NLS-1$
	builder.addAction(index);
}
         
         BulkResult result = getClient().execute(builder.build());
         if (!result.isSucceeded()) {
             logger.warn("Failed to add metric(s) to ES"); //$NON-NLS-1$
         }
     } catch (Exception e) {
         logger.warn("Error adding metric to ES"); //$NON-NLS-1$
         return;
     }
 }
 

@Override
public IEsItem saveIndex(IEsItem doc) throws Exception {
	Index.Builder builder = new Index.Builder(doc);
	if (doc.getIndex() != null) {
		builder.index(doc.getIndex());
	}
	if (doc.getType() != null) {
		builder.type(doc.getType());
	}
	if (doc.getId() != null) {
		builder.id(doc.getId());
	}

	DocumentResult result = _exec(builder.build());
	if (result != null) {
		doc.setIndex(result.getIndex());
		doc.setType(result.getType());
		doc.setId(result.getId());
		return doc;
	}
	return null;
}
 

@Override
public Function<Bulk, Boolean> createFailureHandler(FailoverPolicy failover) {
    return new Function<Bulk, Boolean>() {

        private final JestBatchIntrospector introspector = new JestBatchIntrospector();

        @Override
        public Boolean apply(Bulk bulk) {

            Collection items = introspector.items(bulk);

            LOG.warn(String.format("Batch of %s items failed. Redirecting to %s",
                    items.size(),
                    failover.getClass().getName()));

            items.forEach(item -> {
                Index failedAction = (Index) item;
                failover.deliver(failedItemOps.createItem(failedAction));
            });

            return true;
        }

    };
}
 

@Override
public void onCompleted(Run run, TaskListener listener) {
    loadConfig();

    final boolean validConfig = config != null && config.nonEmptyValues();
    if (validConfig) {
        final Build build = createBuild(run, listener);

        try {
            final Index index = new Index.Builder(build).index(config.getIndexName()).type(config.getTypeName()).build();

            final JestResult result = jestClient.execute(index);

            if (result.isSucceeded()) {
                LOG.fine("Sent build to Elasticsearch: " + build);
            } else {
                LOG.warning("Failed to index build, got error message: " + result.getErrorMessage());
            }
        } catch (Exception e) {
            LOG.log(Level.SEVERE, "Error when sending build data: " + build, e);
        }
    } else {
        LOG.fine("The configuration is not valid, can not index the build");
    }
}
 

/**
 * @see io.apiman.gateway.engine.IRegistry#publishApi(io.apiman.gateway.engine.beans.Api, io.apiman.gateway.engine.async.IAsyncResultHandler)
 */
@Override
public void publishApi(final Api api, final IAsyncResultHandler<Void> handler) {
    try {
        String id = getApiId(api);
        Index index = new Index.Builder(api).refresh(false)
                .index(getIndexName()).setParameter(Parameters.OP_TYPE, "index") //$NON-NLS-1$
                .type("api").id(id).build(); //$NON-NLS-1$
        JestResult result = getClient().execute(index);
        if (!result.isSucceeded()) {
            throw new IOException(result.getErrorMessage());
        } else {
            handler.handle(AsyncResultImpl.create((Void) null));
        }
    } catch (Exception e) {
        handler.handle(AsyncResultImpl.create(
                new PublishingException(Messages.i18n.format("ESRegistry.ErrorPublishingApi"), e),  //$NON-NLS-1$
                Void.class));
    }
}
 

private long doSync( List<DataMap> list,String table,String pk) throws IOException {
    long posi=0;
    //List<Map> list2=new ArrayList<>();
    Bulk.Builder bulk = new Bulk.Builder().defaultIndex(indexName).defaultType(Const.ES_TYPE);
    for(DataMap dataMap:list){
        long id=dataMap.getLong(pk);
        if(id>posi) posi=id;
        Map map=(convertMysql2Es(dataMap));
        logger.info("[full] {}={}",table,map);
        Index index = new Index.Builder(map).id(""+id).build();
        bulk.addAction(index);

    }


    BulkResult br = jest.getJestClient().execute(bulk.build());
    if(!br.isSucceeded()){
        logger.error("error={}, failItems={}",br.getErrorMessage(), JSON.toJSONString(br.getFailedItems()));
        //   br.getFailedItems().get(0).
        throw new RuntimeException("bulk error");
    }
    return  posi;


}
 

/**
 * The update index documents function will take as arguments the index name, document type, and a map of documents to update. The document map key is the
 * document id, and the value is the document as a JSON string.
 */
@Override
public final void updateIndexDocuments(String indexName, String documentType, Map<String, String> documentMap)
{
    LOGGER.info("Updating Elasticsearch index documents, indexName={}, documentType={}.", indexName, documentType);

    List<String> allIndices = getAliases(indexName);

    allIndices.forEach((index) -> {
        // Prepare a bulk request builder
        Bulk.Builder bulkBuilder = new Bulk.Builder();
        // For each document prepare an update request and add it to the bulk request builder
        documentMap.forEach((id, jsonString) -> {
            BulkableAction updateIndex = new Index.Builder(jsonString).index(index).type(documentType).id(id).build();
            bulkBuilder.addAction(updateIndex);
        });

        // Execute the bulk update request
        JestResult jestResult = jestClientHelper.execute(bulkBuilder.build());

        // If there are failures log them
        if (!jestResult.isSucceeded())
        {
            LOGGER.error("Bulk response error = {}", jestResult.getErrorMessage());
        }
    });
}
 

/**
* 批量新增数据
* @param indexName
* @param typeName
* @param objs
* @return
* @throws Exception
*/
public static boolean insertBatch(JestClient jestClient,String indexName, String typeName, List<Object> objs) throws Exception {  
    Bulk.Builder bulk = new Bulk.Builder().defaultIndex(indexName).defaultType(typeName);  
    for (Object obj : objs) {  
        Index index = new Index.Builder(obj).build();  
         bulk.addAction(index);  
    }  
    BulkResult br = jestClient.execute(bulk.build());  
    return br.isSucceeded();  
   }
 

/**
 * Updates a single entity.
 * @param type
 * @param id
 * @param source
 * @throws StorageException
 */
private void updateEntity(String type, String id, XContentBuilder source) throws StorageException {
    try {
        String doc = source.string();
        /* JestResult response = */esClient.execute(new Index.Builder(doc)
                .setParameter(Parameters.OP_TYPE, "index").index(getIndexName()).type(type).id(id).build()); //$NON-NLS-1$
    } catch (Exception e) {
        throw new StorageException(e);
    }
}
 

/**
 * @see io.apiman.gateway.engine.components.ICacheStoreComponent#put(java.lang.String, java.lang.Object, long)
 */
@Override
public <T> void put(String cacheKey, T jsonObject, long timeToLive) throws IOException {
    CacheEntry entry = new CacheEntry();
    entry.setData(null);
    entry.setExpiresOn(System.currentTimeMillis() + (timeToLive * 1000));
    entry.setHead(JSON_MAPPER.writeValueAsString(entry));
    Index index = new Index.Builder(entry).refresh(false).index(getIndexName())
            .type("cacheEntry").id(cacheKey).build(); //$NON-NLS-1$
    try {
        getClient().execute(index);
    } catch (Throwable e) {
    }
}
 

/**
 * @see io.apiman.gateway.engine.components.ICacheStoreComponent#putBinary(java.lang.String, java.lang.Object, long)
 */
@Override
public <T> ISignalWriteStream putBinary(final String cacheKey, final T jsonObject, final long timeToLive)
        throws IOException {
    final CacheEntry entry = new CacheEntry();
    entry.setExpiresOn(System.currentTimeMillis() + (timeToLive * 1000));
    entry.setHead(JSON_MAPPER.writeValueAsString(jsonObject));
    final IApimanBuffer data = bufferFactory.createBuffer();
    return new ISignalWriteStream() {
        boolean finished = false;
        boolean aborted = false;
        @Override
        public void abort(Throwable t) {
            finished = true;
            aborted = false;
        }
        @Override
        public boolean isFinished() {
            return finished;
        }
        @Override
        public void write(IApimanBuffer chunk) {
            data.append(chunk);
        }
        @Override
        public void end() {
            if (!aborted) {
                entry.setData(Base64.encodeBase64String(data.getBytes()));
                Index index = new Index.Builder(entry).refresh(false).index(getIndexName())
                        .type("cacheEntry").id(cacheKey).build(); //$NON-NLS-1$
                try {
                    getClient().execute(index);
                } catch (Throwable e) {
                }
            }
            finished = true;
        }
    };
}
 

/**
 * @see io.apiman.gateway.engine.IRegistry#registerClient(io.apiman.gateway.engine.beans.Client, io.apiman.gateway.engine.async.IAsyncResultHandler)
 */
@Override
public void registerClient(final Client client, final IAsyncResultHandler<Void> handler) {
    try {
        // Validate the client and populate the api map with apis found during validation.
        validateClient(client);

        String id = getClientId(client);
        Index index = new Index.Builder(client)
                .refresh(false)
                .index(getIndexName())
                .setParameter(Parameters.OP_TYPE, "index") //$NON-NLS-1$
                .type("client") //$NON-NLS-1$
                .id(id)
                .build();
        JestResult result = getClient().execute(index);
        if (!result.isSucceeded()) {
            throw new IOException(result.getErrorMessage());
        } else {
            handler.handle(AsyncResultImpl.create((Void) null));
        }
    } catch (IOException e) {
        handler.handle(AsyncResultImpl.create(
                new RegistrationException(Messages.i18n.format("ESRegistry.ErrorRegisteringClient"), e),  //$NON-NLS-1$
                Void.class));
    } catch (RuntimeException re) {
        handler.handle(AsyncResultImpl.create(re, Void.class));
    }
}
 

@Override
public void createIndexDocuments(String indexName, String documentType, Map<String, String> documentMap)
{
    LOGGER.info("Creating Elasticsearch index documents, indexName={}, documentType={}", indexName, documentType);

    List<String> allIndices = getAliases(indexName);

    allIndices.forEach((index) -> {
    // Prepare a bulk request builder
    //final BulkRequestBuilder bulkRequestBuilder = new BulkRequestBuilder(new ElasticsearchClientImpl(), BulkAction.INSTANCE);
    Bulk.Builder bulkBuilder = new Bulk.Builder();
    // For each document prepare an insert request and add it to the bulk request builder
    documentMap.forEach((id, jsonString) ->
    {
        BulkableAction createIndex = new Index.Builder(jsonString).index(index).type(documentType).id(id).build();
        bulkBuilder.addAction(createIndex);

    });

    JestResult jestResult = jestClientHelper.execute(bulkBuilder.build());

    // If there are failures log them
    if (!jestResult.isSucceeded())
    {
        LOGGER.error("Bulk response error = {}", jestResult.getErrorMessage());
    }
    });
}
 

/**
 * The index function will take as arguments indexName, documentType, id, json and add the document to the index.
 */
@Override
public final void createIndexDocument(String indexName, String documentType, String id, String json)
{
    Index index = new Index.Builder(json).index(indexName).type(documentType).id(id).build();
    JestResult jestResult = jestClientHelper.execute(index);
    LOGGER.info("Creating Index Document, indexName={}. successful is {}", indexName, jestResult.isSucceeded());
}
 

@Test
public void intropectorCanAccessActionPayload() {

    // given
    BatchItemIntrospector<AbstractDocumentTargetedAction<DocumentResult>> introspector = new JestActionIntrospector();
    String testPayload = "testPayload";
    AbstractDocumentTargetedAction<DocumentResult> action = new Index.Builder(testPayload).build();

    // when
    String payload = (String) introspector.getPayload(action);

    // then
    Assert.assertEquals(testPayload, payload);
}
 

private Bulk createTestBatch(String... payloads) {
    io.searchbox.core.Bulk.Builder builder = spy(new Bulk.Builder());
    for (String payload : payloads) {
        builder.addAction(spy(new Index.Builder(payload)).build());
    }
    return builder.build();
}
 

@Override
public Object createBatchItem(String indexName, ItemSource source) {
    if (source.getSource() instanceof String) {
        return new Index.Builder(source.getSource())
                .index(indexName)
                .type(mappingType)
                .build();
    }
    throw new ConfigurationException("Non String payloads are not supported by this factory. Make sure that proper ClientObjectFactory implementation is configured");
}
 

@Override
public Object createBatchItem(String indexName, Object source) {
    return new Index.Builder(source)
            .index(indexName)
            .type(mappingType)
            .build();
}
 

@Override
public void invoke(T document)  {
  documentBuffer.add(new Index.Builder(document.toString()).index(indexName).type(documentType).build());

  if (documentBuffer.size() >= batchSize || System.currentTimeMillis() - lastBufferFlush >= maxBufferTime) {
    try {
      flushDocumentBuffer();
    } catch (IOException e) {
      //if the request fails, that's fine, just retry on the next invocation
    }
  }
}
 

private void doSync(List<ConsumerRecord> records) throws Exception {

        Bulk.Builder bulk = new Bulk.Builder().defaultIndex(indexName).defaultType(Const.ES_TYPE);
        for (ConsumerRecord record : records) {
            logger.info("[incr] {}={}",indexName,record.value());
            Row row = JSON.parseObject(record.value(), Row.class);
            if(columnMap==null){
                columnMap=databaseService.getColumnMap(row.getDatabase(),row.getTable());
            }
            String id = record.key();
            if (row.getType().equalsIgnoreCase("insert") || (row.getType().equalsIgnoreCase("update"))) {
                LinkedHashMap<String, Object> data = row.getData();
                Map map = (convertKafka2Es(data));
                Index index = new Index.Builder(map).id(id).build();
                bulk.addAction(index);

            } else if (row.getType().equalsIgnoreCase("delete")) {
                Delete delete = new Delete.Builder(id).build();
                bulk.addAction(delete);
            } else {
                //
            }
        }


            BulkResult br = jest.getJestClient().execute(bulk.build());
            if (!br.isSucceeded()) {
                logger.error("error={}, failItems={}", br.getErrorMessage(), JSON.toJSONString(br.getFailedItems()));
                //   br.getFailedItems().get(0).
                throw new RuntimeException("bulk error");
            }


            //   buffer.add(record);

    }
 

/**
* 批量新增数据
* @param indexName
* @param typeName
* @param objs
* @return
* @throws Exception
*/
public static boolean insertBatch(JestClient jestClient,String indexName, String typeName, List<Object> objs) throws Exception {  
    Bulk.Builder bulk = new Bulk.Builder().defaultIndex(indexName).defaultType(typeName);  
    for (Object obj : objs) {  
        Index index = new Index.Builder(obj).build();  
         bulk.addAction(index);  
    }  
    BulkResult br = jestClient.execute(bulk.build());  
    return br.isSucceeded();  
   }