org.elasticsearch.action.index.IndexRequestBuilder Java Examples

/**
 * 非同期でドキュメントを登録する.
 * @param index インデックス名
 * @param type タイプ名
 * @param id ドキュメントのid
 * @param routingId routingId
 * @param data データ
 * @param opType 操作タイプ
 * @param version version番号
 * @return 非同期応答
 */
public ActionFuture<IndexResponse> asyncIndex(String index,
        String type,
        String id,
        String routingId,
        Map<String, Object> data,
        OpType opType,
        long version) {
    IndexRequestBuilder req = esTransportClient.prepareIndex(index, type, id).setSource(data).setOpType(opType)
            .setConsistencyLevel(WriteConsistencyLevel.DEFAULT).setRefresh(true);
    if (routingFlag) {
        req = req.setRouting(routingId);
    }
    if (version > -1) {
        req.setVersion(version);
    }

    ActionFuture<IndexResponse> ret = req.execute();
    EsRequestLogInfo logInfo = new EsRequestLogInfo(index, type, id, routingId, data, opType.toString(),
            version);
    this.fireEvent(Event.afterCreate, logInfo);

    return ret;
}
 

private static void remoteCheck(AbstractClient client, List<Object> builderList) throws ExecutionException, InterruptedException {
  for (Object builder : builderList) {
    BulkRequestBuilder bulkRequestBuilder = null;
    if (builder instanceof IndexRequestBuilder) {
      bulkRequestBuilder = client.prepareBulk().add((IndexRequestBuilder) builder);
    } else if (builder instanceof UpdateRequestBuilder) {
      bulkRequestBuilder = client.prepareBulk().add((UpdateRequestBuilder) builder);
    }  else if (builder instanceof DeleteRequestBuilder) {
      bulkRequestBuilder = client.prepareBulk().add((DeleteRequestBuilder) builder);
    } else {
      fail();
    }
    BulkResponse bulkItemResponses = bulkRequestBuilder.execute().get();
    assertFalse(Arrays.stream(bulkItemResponses.getItems()).anyMatch(BulkItemResponse::isFailed));
  }
}
 

private void populateIndexRequestBuilder(IndexRequestBuilder builder, IndexOptions options) {
    if (options != null) {
        if (options.getId() != null) builder.setId(options.getId());
        if (options.getRouting() != null) builder.setRouting(options.getRouting());
        if (options.getParent() != null) builder.setParent(options.getParent());
        if (options.getOpType() != null)
            builder.setOpType(DocWriteRequest.OpType.valueOf(options.getOpType().name()));
        if (options.getWaitForActiveShard() != null)
            builder.setWaitForActiveShards(options.getWaitForActiveShard());
        if (options.getRefreshPolicy() != null)
            builder.setRefreshPolicy(WriteRequest.RefreshPolicy.valueOf(options.getRefreshPolicy().name()));
        if (options.getVersion() != null) builder.setVersion(options.getVersion());
        if (options.getVersionType() != null) builder.setVersionType(options.getVersionType());
        if (options.getTimeout() != null) builder.setTimeout(options.getTimeout());
    }
}
 

protected IndexRequestBuilder createUpdateRequest(final EsAbstractEntity esEntity) {
    final IndexRequestBuilder builder =
            client.prepareIndex().setIndex(asEsIndex()).setId(esEntity.asDocMeta().id()).setSource(toSource(esEntity));
    final RequestOptionCall<IndexRequestBuilder> indexOption = esEntity.asDocMeta().indexOption();
    if (indexOption != null) {
        indexOption.callback(builder);
    }
    final Long seqNo = esEntity.asDocMeta().seqNo();
    if (seqNo != null && seqNo.longValue() != SequenceNumbers.UNASSIGNED_SEQ_NO) {
        esEntity.asDocMeta().seqNo(seqNo);
    }
    final Long primaryTerm = esEntity.asDocMeta().primaryTerm();
    if (primaryTerm != null && primaryTerm.longValue() != SequenceNumbers.UNASSIGNED_PRIMARY_TERM) {
        esEntity.asDocMeta().primaryTerm(primaryTerm);
    }
    return builder;
}
 

@Override
protected int delegateUpdate(final Entity entity, final UpdateOption<? extends ConditionBean> option) {
    final EsAbstractEntity esEntity = (EsAbstractEntity) entity;
    final IndexRequestBuilder builder = createUpdateRequest(esEntity);

    final IndexResponse response = builder.execute().actionGet(indexTimeout);
    final long seqNo = response.getSeqNo();
    if (seqNo != SequenceNumbers.UNASSIGNED_SEQ_NO) {
        esEntity.asDocMeta().seqNo(seqNo);
    }
    final long primaryTerm = response.getPrimaryTerm();
    if (primaryTerm != SequenceNumbers.UNASSIGNED_PRIMARY_TERM) {
        esEntity.asDocMeta().primaryTerm(primaryTerm);
    }

    return 1;
}
 

public boolean doStore(final AuditMessage msg) {

		if (Boolean.parseBoolean((String) HeaderHelper.getSafeFromHeader(threadPool.getThreadContext(), ConfigConstants.OPENDISTRO_SECURITY_CONF_REQUEST_HEADER))) {
			if (log.isTraceEnabled()) {
				log.trace("audit log of audit log will not be executed");
			}
			return true;
		}

		try (StoredContext ctx = threadPool.getThreadContext().stashContext()) {
			try {
				final IndexRequestBuilder irb = clientProvider.prepareIndex(getExpandedIndexName(indexPattern, index), type).setRefreshPolicy(RefreshPolicy.IMMEDIATE).setSource(msg.getAsMap());
				threadPool.getThreadContext().putHeader(ConfigConstants.OPENDISTRO_SECURITY_CONF_REQUEST_HEADER, "true");
				irb.setTimeout(TimeValue.timeValueMinutes(1));
				irb.execute().actionGet();
				return true;
			} catch (final Exception e) {
				log.error("Unable to index audit log {} due to {}", msg, e.toString(), e);
				return false;
			}
		}
	}
 

public boolean store(final String index, final Object obj) {
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    @SuppressWarnings("unchecked")
    final Map<String, Object> source = obj instanceof Map ? (Map<String, Object>) obj : BeanUtil.copyBeanToNewMap(obj);
    final String id = (String) source.remove(fessConfig.getIndexFieldId());
    source.remove(fessConfig.getIndexFieldVersion());
    final Number seqNo = (Number) source.remove(fessConfig.getIndexFieldSeqNo());
    final Number primaryTerm = (Number) source.remove(fessConfig.getIndexFieldPrimaryTerm());
    IndexResponse response;
    try {
        if (id == null) {
            // TODO throw Exception in next release
            // create
            response =
                    client.prepareIndex().setIndex(index).setSource(new DocMap(source)).setRefreshPolicy(RefreshPolicy.IMMEDIATE)
                            .setOpType(OpType.CREATE).execute().actionGet(fessConfig.getIndexIndexTimeout());
        } else {
            // create or update
            final IndexRequestBuilder builder =
                    client.prepareIndex().setIndex(index).setId(id).setSource(new DocMap(source))
                            .setRefreshPolicy(RefreshPolicy.IMMEDIATE).setOpType(OpType.INDEX);
            if (seqNo != null) {
                builder.setIfSeqNo(seqNo.longValue());
            }
            if (primaryTerm != null) {
                builder.setIfPrimaryTerm(primaryTerm.longValue());
            }
            response = builder.execute().actionGet(fessConfig.getIndexIndexTimeout());
        }
        final Result result = response.getResult();
        return result == Result.CREATED || result == Result.UPDATED;
    } catch (final ElasticsearchException e) {
        throw new FessEsClientException("Failed to store: " + obj, e);
    }
}
 

@Override
protected Client getTransportClient(Settings.Builder settingsBuilder, String xPackPath,
                                    String username, String password,
                                    List<InetSocketAddress> esHosts, ComponentLog log)
        throws MalformedURLException {
    final Client mockClient = mock(Client.class);
    BulkRequestBuilder bulkRequestBuilder = spy(new BulkRequestBuilder(mockClient, BulkAction.INSTANCE));
    if (exceptionToThrow != null) {
        doThrow(exceptionToThrow).when(bulkRequestBuilder).execute();
    } else {
        doReturn(new MockBulkRequestBuilderExecutor(responseHasFailures, esHosts.get(0))).when(bulkRequestBuilder).execute();
    }
    when(mockClient.prepareBulk()).thenReturn(bulkRequestBuilder);

    when(mockClient.prepareIndex(anyString(), anyString(), anyString())).thenAnswer(new Answer<IndexRequestBuilder>() {
        @Override
        public IndexRequestBuilder answer(InvocationOnMock invocationOnMock) throws Throwable {
            Object[] args = invocationOnMock.getArguments();
            String arg1 = (String) args[0];
            if (arg1.isEmpty()) {
                throw new NoNodeAvailableException("Needs index");
            }
            String arg2 = (String) args[1];
            if (arg2.isEmpty()) {
                throw new NoNodeAvailableException("Needs doc type");
            } else {
                IndexRequestBuilder indexRequestBuilder = new IndexRequestBuilder(mockClient, IndexAction.INSTANCE);
                return indexRequestBuilder;
            }
        }
    });

    return mockClient;
}
 

/**
 * 批量保存
 *
 * @param batchList
 * @return
 */
public boolean batchSave(List<IndexRequestBuilder> batchList) {
    if (batchList.size() > 0) {
        BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
        for (IndexRequestBuilder requestBuilder : batchList) {
            bulkRequestBuilder.add(requestBuilder);
        }
        return !bulkRequestBuilder.get().hasFailures();
    } else {
        return false;
    }
}
 

@Override
protected int delegateInsert(final Entity entity, final InsertOption<? extends ConditionBean> option) {
    final EsAbstractEntity esEntity = (EsAbstractEntity) entity;
    IndexRequestBuilder builder = createInsertRequest(esEntity);

    final IndexResponse response = builder.execute().actionGet(indexTimeout);
    esEntity.asDocMeta().id(response.getId());
    return response.getResult() == Result.CREATED ? 1 : 0;
}
 

@Override
public void closeHistory(String documentId) {
  ElasticsearchDocumentHistory dh = getCachedHistoryIfPresent(documentId);

  if (dh == null) {
    LOGGER.warn(
        "Attempt to close a document {} which is not in cache, thus can't be persisted",
        documentId);
    return;
  }

  try {
    byte[] source = mapper.writeValueAsBytes(new ESHistory(documentId, dh.getAllHistory()));

    new IndexRequestBuilder(elasticsearch.getClient(), IndexAction.INSTANCE)
        .setIndex(documentId)
        .setIndex(esIndex)
        .setType(esType)
        .setId(documentId)
        .setSource(source, XContentType.JSON)
        .get();

  } catch (JsonProcessingException e) {
    LOGGER.warn("Unable to convert history to source, so can't be persisted {}", documentId, e);
  }

  super.closeHistory(documentId);
}
 

/**
 * Creates and prepares an {@link IndexRequestBuilder} from the supplied
 * {@link Client} via delegation to the subclass-hook template methods
 * {@link #getIndexName(String, long)} and
 * {@link #prepareIndexRequest(IndexRequestBuilder, String, String, Event)}
 */
@Override
public IndexRequestBuilder createIndexRequest(Client client,
      String indexPrefix, String indexType, Event event) throws IOException {
  IndexRequestBuilder request = prepareIndex(client);
  String realIndexPrefix = BucketPath.escapeString(indexPrefix, event.getHeaders());
  String realIndexType = BucketPath.escapeString(indexType, event.getHeaders());

  TimestampedEvent timestampedEvent = new TimestampedEvent(event);
  long timestamp = timestampedEvent.getTimestamp();

  String indexName = getIndexName(realIndexPrefix, timestamp);
  prepareIndexRequest(request, indexName, realIndexType, timestampedEvent);
  return request;
}
 

/**
 * Bulk save of the entities.
 *
 * @param type index type
 * @param docs metacat documents
 */
private void bulkSaveToIndex(final String type, final List<ElasticSearchDoc> docs, final String index) {
    if (docs != null && !docs.isEmpty()) {
        try {

            RETRY_ES_PUBLISH.call(() -> {
                final BulkRequestBuilder bulkRequest = client.prepareBulk();
                for (ElasticSearchDoc doc : docs) {
                    final IndexRequestBuilder indexRequestBuilder = prepareIndexRequest(index, type, doc);
                    if (indexRequestBuilder != null) {
                        bulkRequest.add(indexRequestBuilder);
                    }
                }

                if (bulkRequest.numberOfActions() > 0) {
                    final BulkResponse bulkResponse = bulkRequest.execute().actionGet(esBulkCallTimeout);
                    log.info("Bulk saving metadata of index {} type {} with size {}.",
                        index, type, docs.size());
                    if (bulkResponse.hasFailures()) {
                        for (BulkItemResponse item : bulkResponse.getItems()) {
                            if (item.isFailed()) {
                                handleException("ElasticSearchUtil.bulkSaveToIndex.index", type, item.getId(),
                                    item.getFailure().getCause(), Metrics.CounterElasticSearchSave.getMetricName());
                            }
                        }
                    }
                }
                return null;
            });
        } catch (Exception e) {
            final List<String> docIds = docs.stream().map(ElasticSearchDoc::getId).collect(Collectors.toList());
            handleException("ElasticSearchUtil.bulkSaveToIndex", type, docIds, e,
                Metrics.CounterElasticSearchBulkSave.getMetricName());
        }
    }
}
 

private static void reindexData(IndicesAdminClient indicesAdminClient) {
    //查询原来的所有数据,TimeValue是需要保存的时长
    SearchResponse searchResponse = client.prepareSearch(ALIX_NAME).setTypes(INDEX_TYPE).setQuery(QueryBuilders.matchAllQuery()).
            setSearchType(SearchType.SCAN).setScroll(new TimeValue(20000))
            .setSize(100).execute().actionGet();
    //当前id
         String scrollId = searchResponse.getScrollId();
        while(StringUtils.isNotEmpty(scrollId)) {
            BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
            SearchResponse scrollSearchResponse = client.prepareSearchScroll(scrollId).setScroll(new TimeValue(20000)).execute().actionGet();
            SearchHits hits = scrollSearchResponse.getHits();
            if(hits.getHits().length>0) {
                for (SearchHit searchHitFields : hits.getHits()) {
                    IndexRequestBuilder indexRequestBuilder = client.prepareIndex(INDEX_NAME_v2, INDEX_TYPE).setId(searchHitFields.getId()).setSource(searchHitFields.getSource()).setOpType(IndexRequest.OpType.INDEX);
                    bulkRequestBuilder.add(indexRequestBuilder);
                }
                BulkResponse bulkResponse = bulkRequestBuilder.execute().actionGet();
                if (bulkResponse.hasFailures()) {
                    log.error("reindex失败 : {}", bulkResponse.buildFailureMessage());
                } else {
                    log.info("reindex {}条成功:", hits.getHits().length);
                }
            }else{
                break;
            }
        }
}
 

@Before
public void setupFactory() throws Exception {
  serializer = new FakeEventSerializer();
  factory = new EventSerializerIndexRequestBuilderFactory(serializer) {
    @Override
    IndexRequestBuilder prepareIndex(Client client) {
      return new IndexRequestBuilder(FAKE_CLIENT);
    }
  };
}
 

/**
 * @param row
 * @param requestBuilder
 */
private void addSourceFromJsonString( Object[] row, IndexRequestBuilder requestBuilder ) throws KettleStepException {
  Object jsonString = row[jsonFieldIdx];
  if ( jsonString instanceof byte[] ) {
    requestBuilder.setSource( (byte[]) jsonString, XContentType.JSON );
  } else if ( jsonString instanceof String ) {
    requestBuilder.setSource( (String) jsonString, XContentType.JSON );
  } else {
    throw new KettleStepException( BaseMessages.getString( "ElasticSearchBulk.Error.NoJsonFieldFormat" ) );
  }
}
 

protected IndexRequestBuilder createInsertRequest(final EsAbstractEntity esEntity) {
    final IndexRequestBuilder builder = client.prepareIndex().setIndex(asEsIndex()).setSource(toSource(esEntity));
    final String id = esEntity.asDocMeta().id();
    if (id != null) {
        builder.setId(id);
    }
    final RequestOptionCall<IndexRequestBuilder> indexOption = esEntity.asDocMeta().indexOption();
    if (indexOption != null) {
        indexOption.callback(builder);
    }
    return builder;
}
 

public IndexResponse createDocument(String index, String type, String id, String source) {
    return execute(new Callable<IndexResponse>() {

        @Override
        public IndexResponse call() throws Exception {
            LOGGER.trace("create document: '{}/{}/{}' source: '{}'", index, type, id, source);
            IndexRequestBuilder builder = client.prepareIndex(index, type, id).setSource(source, XContentType.JSON);
            IndexResponse response = builder.get();
            return response;
        }
    });
}
 

public void insertOrUpdate(DataConfig entity, RequestOptionCall<IndexRequestBuilder> opLambda) {
    entity.asDocMeta().indexOption(opLambda);
    doInsertOrUpdate(entity, null, null);
}
 

protected abstract IndexRequestBuilder completeIndexRequestBuilder(IndexRequestBuilder requestBuilder,
                                        String resourceName, EntityContentProducer ecp,
                                        boolean includeContent)
throws IOException;
 

public int[] batchUpdate(List<CrawlingInfoParam> list, RequestOptionCall<BulkRequestBuilder> call,
        RequestOptionCall<IndexRequestBuilder> entityCall) {
    return doBatchUpdate(new BulkList<>(list, call, entityCall), null);
}
 

public void update(RoleType entity, RequestOptionCall<IndexRequestBuilder> opLambda) {
    entity.asDocMeta().indexOption(opLambda);
    doUpdate(entity, null);
}
 

public void insert(ElevateWordToLabel entity, RequestOptionCall<IndexRequestBuilder> opLambda) {
    entity.asDocMeta().indexOption(opLambda);
    doInsert(entity, null);
}
 

public void update(ThumbnailQueue entity, RequestOptionCall<IndexRequestBuilder> opLambda) {
    entity.asDocMeta().indexOption(opLambda);
    doUpdate(entity, null);
}
 

public int[] batchDelete(List<ScheduledJob> list, RequestOptionCall<BulkRequestBuilder> call,
        RequestOptionCall<IndexRequestBuilder> entityCall) {
    return doBatchDelete(new BulkList<>(list, call, entityCall), null);
}
 

public void insertOrUpdate(RelatedQuery entity, RequestOptionCall<IndexRequestBuilder> opLambda) {
    entity.asDocMeta().indexOption(opLambda);
    doInsertOrUpdate(entity, null, null);
}
 

@VisibleForTesting
IndexRequestBuilder prepareIndex(Client client) {
  return client.prepareIndex();
}
 

public void insert(RelatedContent entity, RequestOptionCall<IndexRequestBuilder> opLambda) {
    entity.asDocMeta().indexOption(opLambda);
    doInsert(entity, null);
}
 

public int[] batchInsert(List<BoostDocumentRule> list, RequestOptionCall<BulkRequestBuilder> call,
        RequestOptionCall<IndexRequestBuilder> entityCall) {
    return doBatchInsert(new BulkList<>(list, call, entityCall), null);
}
 

public int[] batchUpdate(List<CrawlingInfo> list, RequestOptionCall<BulkRequestBuilder> call,
        RequestOptionCall<IndexRequestBuilder> entityCall) {
    return doBatchUpdate(new BulkList<>(list, call, entityCall), null);
}