Java Code Examples for org.elasticsearch.action.index.IndexRequestBuilder#setSource()

/**
 * 存储到ES
 * 
 * @param appUUID
 * @param span
 * @param bulkRequest
 */
private void pushSpanToBulkRequest(String appUUID, String appGroup, Span span, BulkRequestBuilder bulkRequest) {

    /**
     * 保证不会重复
     */
    String uuid = EncodeHelper.encodeMD5(span.toString());

    /**
     * 获取当前正在使用的index名称
     */
    String currentIndex = indexMgr.prepareIndex();

    IndexRequestBuilder irb = client.getClient().prepareIndex(currentIndex, InvokeChainIndexMgr.IVC_Table, uuid);

    Map<String, Object> m = span.toMap();

    m.put("appuuid", appUUID);
    m.put("appgroup", appGroup);

    irb.setSource(m);

    bulkRequest.add(irb);
}
 

/**
 * 获取新增请求
 *
 * @param esObject 更新请求参数
 * @return UpdateRequestBuilder 更新请求
 */
private IndexRequestBuilder getIndexRequest(SaveESObject esObject) {
    String dataId = getId(esObject.getUkMap());

    byte[] dataBytes = null;
    try {
        dataBytes = OBJECT_MAPPER.writeValueAsBytes(esObject.getDataMap());
    } catch (JsonProcessingException e) {
        // never hapened
        SearchLogger.error("", e);
    }

    IndexRequestBuilder indexRequestBuilder = transportClient.prepareIndex().setIndex(esObject.getIndexName())
            .setType(esObject.getTypeName());
    if (StringUtils.isNotBlank(dataId)) {
        indexRequestBuilder.setId(dataId);
    }
    // TODO 替换
    // indexRequestBuilder.setSource(esObject.getDataMap());
    indexRequestBuilder.setSource(dataBytes, XContentType.JSON);
    return indexRequestBuilder;
}
 

/**
 * 
 * @param appUUID
 * @param appGroup
 * @param span
 * @param bulkRequest
 */
private void pushDataToBulkRequest(JavaThreadObject jto, BulkRequestBuilder bulkRequest) {

    /**
     * 保证不会重复
     */
    String uuid = EncodeHelper.encodeMD5(jto.toString());

    /**
     * 获取当前正在使用的index名称
     */
    String currentIndex = indexMgr.prepareIndex();

    IndexRequestBuilder irb = client.getClient().prepareIndex(currentIndex, ThreadAnalysisIndexMgr.JTA_TABLE, uuid);

    Map<String, Object> m = jto.toMap();

    irb.setSource(m);

    bulkRequest.add(irb);
}
 

public Optional<BulkResult> indexData(ElasticDataPointer targetDataPointer, SearchHit[] hits) {
  BulkRequestBuilder bulkRequest = createBulkRequestBuilder();

  for (SearchHit hit : hits) {
    Map<String, Object> source = hit.getSource();

    IndexRequestBuilder requestBuilder = prepareIndex(targetDataPointer.getIndexName(), targetDataPointer
        .getTypeName(), hit.getId(), source, hit.getIndex());
    if (hit.getFields().get("_ttl") != null) {
      requestBuilder.setTTL(hit.getFields().get("_ttl").value());
    }
    if (hit.getFields().get("_routing") != null) {
      requestBuilder.setRouting(hit.getFields().get("_routing").value());
    }
    requestBuilder.setSource(source);
    bulkRequest.add(requestBuilder);
  }
  return executeBulk(hits.length, bulkRequest);
}
 

/**
 * @param row
 * @param requestBuilder
 */
private void addSourceFromJsonString( Object[] row, IndexRequestBuilder requestBuilder ) throws KettleStepException {
  Object jsonString = row[jsonFieldIdx];
  if ( jsonString instanceof byte[] ) {
    requestBuilder.setSource( (byte[]) jsonString, XContentType.JSON );
  } else if ( jsonString instanceof String ) {
    requestBuilder.setSource( (String) jsonString, XContentType.JSON );
  } else {
    throw new KettleStepException( BaseMessages.getString( "ElasticSearchBulk.Error.NoJsonFieldFormat" ) );
  }
}
 

@Override
public void write(NutchDocument doc) throws IOException {
  String id = (String)doc.getFieldValue("url");
  String type = doc.getDocumentMeta().get("type");
  if (type == null) type = "doc";
  IndexRequestBuilder request = client.prepareIndex(defaultIndex, type, id);

  Map<String, Object> source = new HashMap<String, Object>();

  // Loop through all fields of this doc
  for (String fieldName : doc.getFieldNames()) {
    if (doc.getField(fieldName).getValues().size() > 1) {
      source.put(fieldName, doc.getFieldValue(fieldName));
      // Loop through the values to keep track of the size of this document
      for (Object value : doc.getField(fieldName).getValues()) {
        bulkLength += value.toString().length();
      }
    } else {
      source.put(fieldName, doc.getFieldValue(fieldName));
      bulkLength += doc.getFieldValue(fieldName).toString().length();
    }
  }
  request.setSource(source);

  // Add this indexing request to a bulk request
  bulk.add(request);
  indexedDocs++;
  bulkDocs++;

  if (bulkDocs >= maxBulkDocs || bulkLength >= maxBulkLength) {
    LOG.info("Processing bulk request [docs = " + bulkDocs + ", length = "
            + bulkLength + ", total docs = " + indexedDocs
            + ", last doc in bulk = '" + id + "']");
    // Flush the bulk of indexing requests
    createNewBulk = true;
    commit();
  }
}
 

/**
 *
 * @param resourceName
 * @param ecp
 * @return
 */
protected IndexRequestBuilder prepareIndex(String resourceName, EntityContentProducer ecp, boolean includeContent)
        throws IOException, NoContentException {
    IndexRequestBuilder requestBuilder = newIndexRequestBuilder(resourceName, ecp, includeContent);
    final XContentBuilder requestContentSource = buildIndexRequestContentSource(resourceName, ecp, includeContent);
    requestBuilder = requestBuilder.setSource(requestContentSource);
    return completeIndexRequestBuilder(requestBuilder, resourceName, ecp, includeContent);
}
 

private IndexRequestBuilder getIndexRequestBuilderForAProductGroup(
		final ProductGroup productGroup,
		final ElasticSearchIndexConfig config, final String indexName)
		throws IOException {
	final XContentBuilder contentBuilder = getXContentBuilderForAProductGroup(productGroup);
	logger.debug("Generated XContentBuilder for document id {} is {}",
			new Object[] { productGroup.getId(),
					contentBuilder.prettyPrint().string() });
	final IndexRequestBuilder indexRequestBuilder = searchClientService
			.getClient().prepareIndex(indexName,
					config.getGroupDocumentType(),
					String.valueOf(productGroup.getId()));
	indexRequestBuilder.setSource(contentBuilder);
	return indexRequestBuilder;
}
 

private IndexRequestBuilder getIndexRequestBuilderForAProduct(
		final Product product, final String indexName,
		final String documentType) throws IOException {
	final XContentBuilder contentBuilder = getXContentBuilderForAProduct(product);
	final IndexRequestBuilder indexRequestBuilder = searchClientService
			.getClient().prepareIndex(indexName, documentType,
					String.valueOf(product.getId()));
	indexRequestBuilder.setSource(contentBuilder);
	return indexRequestBuilder;
}
 

public boolean doIndex() throws Exception {

		try {

			synchronized (bulk_set) {
				if (client == null)
					throw new Exception("client is null");

				BulkRequestBuilder bulkRequest = client.prepareBulk();

				Iterator<JSONObject> iterator = bulk_set.iterator();

				while (iterator.hasNext()) {
					JSONObject setElement = iterator.next();

					IndexRequestBuilder a = client.prepareIndex(_index_name,
							_document_name);
					a.setSource(setElement.toString());
					bulkRequest.add(a);

				}

				_LOG.trace("[OpenSOC] Performing bulk load of size: "
						+ bulkRequest.numberOfActions());

				BulkResponse resp = bulkRequest.execute().actionGet();
				_LOG.trace("[OpenSOC] Received bulk response: "
						+ resp.toString());
				bulk_set.clear();
			}

			return true;
		}

		catch (Exception e) {
			e.printStackTrace();
			return false;
		}
	}
 

/**
 *
 * @param resourceName
 * @param ecp
 * @return
 */
protected IndexRequestBuilder prepareIndex(String resourceName, EntityContentProducer ecp, boolean includeContent)
        throws IOException, NoContentException {
    IndexRequestBuilder requestBuilder = newIndexRequestBuilder(resourceName, ecp, includeContent);
    final XContentBuilder requestContentSource = buildIndexRequestContentSource(resourceName, ecp, includeContent);
    requestBuilder = requestBuilder.setSource(requestContentSource);
    return completeIndexRequestBuilder(requestBuilder, resourceName, ecp, includeContent);
}
 

/**
 * 存储到ES
 * 
 * @param appUUID
 * @param appGroup
 * @param span
 * @param bulkRequest
 * @param protocolType
 */
private void pushSpanToBulkRequest(String appUUID, String appGroup, SlowOperSpan span,
        BulkRequestBuilder bulkRequest, String protocolType) {

    /**
     * 保证不会重复(其实是防止重复读取时数据重复)
     */
    String uuid = EncodeHelper.encodeMD5(span.toString());

    /**
     * 获取当前正在使用的index名称
     */
    String currentIndex = indexMgr.prepareIndex();
    /**
     * 准备对应type
     */
    indexMgr.prepareIndexType(currentIndex, protocolType);

    IndexRequestBuilder irb = client.getClient().prepareIndex(currentIndex, protocolType, uuid);

    Map<String, Object> m = span.toMap();

    // 暂时保留这两个属性
    m.put("appuuid", appUUID);
    m.put("appgroup", appGroup);

    irb.setSource(m);

    bulkRequest.add(irb);
}
 

@Override
protected IndexRequestBuilder setSource(IndexRequestBuilder indexRequestBuilder, T tuple)
{
  return indexRequestBuilder.setSource(tuple);
}
 

/**
 * Emits records to elasticsearch.
 * 1. Adds each record to a bulk index request, conditionally adding version, ttl or create if they were set in the
 * transformer.
 * 2. Executes the bulk request, returning any record specific failures to be retried by the connector library
 * pipeline, unless
 * outlined below.
 * 
 * Record specific failures (noted in the failure.getMessage() string)
 * - DocumentAlreadyExistsException means the record has create set to true, but a document already existed at the
 * specific index/type/id.
 * - VersionConflictEngineException means the record has a specific version number that did not match what existed
 * in elasticsearch.
 * To guarantee in order processing by the connector, when putting data use the same partition key for objects going
 * to the same
 * index/type/id and set sequence number for ordering.
 * - In either case, the emitter will assume that the record would fail again in the future and thus will not return
 * the record to
 * be retried.
 * 
 * Bulk request failures
 * - NoNodeAvailableException means the TransportClient could not connect to the cluster.
 * - A general Exception catches any other unexpected behavior.
 * - In either case the emitter will continue making attempts until the issue has been resolved. This is to ensure
 * that no data
 * loss occurs and simplifies restarting the application once issues have been fixed.
 */
@Override
public List<ElasticsearchObject> emit(UnmodifiableBuffer<ElasticsearchObject> buffer) throws IOException {
    List<ElasticsearchObject> records = buffer.getRecords();
    if (records.isEmpty()) {
        return Collections.emptyList();
    }

    BulkRequestBuilder bulkRequest = elasticsearchClient.prepareBulk();
    for (ElasticsearchObject record : records) {
        IndexRequestBuilder indexRequestBuilder =
                elasticsearchClient.prepareIndex(record.getIndex(), record.getType(), record.getId());
        indexRequestBuilder.setSource(record.getSource());
        Long version = record.getVersion();
        if (version != null) {
            indexRequestBuilder.setVersion(version);
        }
        Long ttl = record.getTtl();
        if (ttl != null) {
            indexRequestBuilder.setTTL(ttl);
        }
        Boolean create = record.getCreate();
        if (create != null) {
            indexRequestBuilder.setCreate(create);
        }
        bulkRequest.add(indexRequestBuilder);
    }

    while (true) {
        try {
            BulkResponse bulkResponse = bulkRequest.execute().actionGet();

            BulkItemResponse[] responses = bulkResponse.getItems();
            List<ElasticsearchObject> failures = new ArrayList<ElasticsearchObject>();
            int numberOfSkippedRecords = 0;
            for (int i = 0; i < responses.length; i++) {
                if (responses[i].isFailed()) {
                    LOG.error("Record failed with message: " + responses[i].getFailureMessage());
                    Failure failure = responses[i].getFailure();
                    if (failure.getMessage().contains("DocumentAlreadyExistsException")
                            || failure.getMessage().contains("VersionConflictEngineException")) {
                        numberOfSkippedRecords++;
                    } else {
                        failures.add(records.get(i));
                    }
                }
            }
            LOG.info("Emitted " + (records.size() - failures.size() - numberOfSkippedRecords)
                    + " records to Elasticsearch");
            if (!failures.isEmpty()) {
                printClusterStatus();
                LOG.warn("Returning " + failures.size() + " records as failed");
            }
            return failures;
        } catch (NoNodeAvailableException nnae) {
            LOG.error("No nodes found at " + elasticsearchEndpoint + ":" + elasticsearchPort + ". Retrying in "
                    + BACKOFF_PERIOD + " milliseconds", nnae);
            sleep(BACKOFF_PERIOD);
        } catch (Exception e) {
            LOG.error("ElasticsearchEmitter threw an unexpected exception ", e);
            sleep(BACKOFF_PERIOD);
        }
    }

}
 

/**
 * creates a set of index requests based on a set of explicit VALUES  
 * @param insert
 * @param index
 * @return
 * @throws SQLException
 */
@SuppressWarnings("unchecked")
private int insertFromValues(String sql, Insert insert, String index, int maxRequestsPerBulk) throws SQLException {
	Heading heading = new Heading();
	QueryState state = new BasicQueryState(sql, heading, this.props);
	List<Object> values = updateParser.parse(insert, state);
	if(state.hasException()) throw state.getException();
	if(heading.hasLabel("_index") || heading.hasLabel("_type")) throw new SQLException("Not possible to set _index and _type fields");

	String[] indexAndType = this.getIndexAndType(insert.getTarget().toString(), sql, "into\\s+", "\\s+values", index);
	index = indexAndType[0];
	String type = indexAndType[1];
	
	if(values.size() % heading.getColumnCount() != 0) throw new SQLException("Number of columns does not match number of values for one of the inserts");
	
	List<IndexRequestBuilder> indexReqs = new ArrayList<IndexRequestBuilder>();
	int indexCount = 0;
	int valueIdx = 0;
	while(valueIdx < values.size()){
		HashMap<String, Object> fieldValues = new HashMap<String, Object>();
		String id = null;
		for(Column col : heading.columns()){
			Object value = values.get(valueIdx);
			valueIdx++;
			
			if(col.getColumn().equals("_id")){
				id = value.toString();
				continue;
			}
			
			if(col.getColumn().indexOf('.') == -1) {
				fieldValues.put(col.getColumn(), value);
				continue;
			}
				
			// create nested object
			Map<String, Object> map = fieldValues; 
			String[] objectDef = col.getColumn().split("\\.");
			for(int k=0; k<objectDef.length; k++){
				String key = objectDef[k];
				if(k == objectDef.length-1) map.put(key, value);
				else{
					if(!map.containsKey(key)) map.put(key, new HashMap<String, Object>());
						map = (Map<String, Object>)map.get(key);
				}
			}
		}
		
		// create index request
		IndexRequestBuilder indexReq = client.prepareIndex().setIndex(index).setType(type);
		if(id != null) indexReq.setId(id);
		indexReq.setSource(fieldValues);
		indexReqs.add(indexReq);
		if(indexReqs.size() >= maxRequestsPerBulk){
			indexCount += this.execute(indexReqs, maxRequestsPerBulk);
			indexReqs.clear();
		}
	}
	if(indexReqs.size() > 0)  indexCount += this.execute(indexReqs, maxRequestsPerBulk);
	return indexCount;
}
 

public static void main(String[] args) {

        TransportClient client = new PreBuiltTransportClient(Settings.EMPTY);

        client.addTransportAddress(new InetSocketTransportAddress(new InetSocketAddress("127.0.0.1", 9300)));

        IndexRequestBuilder irb = client.prepareIndex("uav_test_db", "uav_test_table");

        Map<String, Object> item = new HashMap<String, Object>();

        item.put("name", "zz");
        item.put("age", 1);

        irb.setSource(item);

        IndexResponse ir = irb.get();

        System.out.println(ir.status());

        client.close();
    }
 

/**
 * pushLogLineToBulkRequest
 * 
 * @param mdf
 * @param appid
 * @param bulkRequest
 * @param logData
 */
@SuppressWarnings({ "rawtypes", "unchecked" })
private void pushLogLineToBulkRequest(MonitorDataFrame mdf, String appid, String ipport,
        BulkRequestBuilder bulkRequest, Map logData) {

    /**
     * Step 1: get log file name
     */
    String logid = (String) logData.get("id");

    File f = new File(logid);

    String logFileName = f.getName().replace('.', '_');

    /**
     * Step 2: read log file lines
     */
    Map<String, Object> fields = (Map<String, Object>) logData.get("values");

    List<Map> lines = (List<Map>) fields.get("content");

    for (Map line : lines) {

        long ts = DataConvertHelper.toLong(line.get("_timestamp"), -1);

        line.remove("_timestamp");
        line.put("l_timestamp", ts);

        long lnum = DataConvertHelper.toLong(line.get("_lnum"), -1);

        line.remove("_lnum");
        line.put("l_num", lnum);

        /**
         * 如果没有规则设置，使用了全行抓取，则索引Type=日志文件名+"_def"
         */
        String logFileType = logFileName;
        StringBuilder uuidStr = new StringBuilder();
        uuidStr.append(ipport).append(mdf.getServerId()).append("-").append(appid).append("-").append(logid)
                .append("-").append(lnum);
        if (line.containsKey("content")) {
            logFileType += "_def";
            uuidStr.append("-").append(line.get("content"));
        }
        /**
         * 如果设置了规则，则应该使用索引Type=日志文件名+"_"+<规则名>
         */
        else {
            // TODO: not implement yet
        }

        /**
         * 保证不重复：IP+SvrID+AppID+LogFileName+lineNum+日志内容（def下为content）
         */
        String uuid = EncodeHelper.encodeMD5(uuidStr.toString());

        // 准备index，如果不存在，就创建
        String currentIndex = indexMgr.prepareIndex();

        // 检查type是否存在，不存在就创建
        indexMgr.prepareIndexType(currentIndex, logFileType.toLowerCase());

        IndexRequestBuilder irb = client.getClient().prepareIndex(currentIndex, logFileType.toLowerCase(), uuid);

        /**
         * 用于区分不同机器上的应用实例
         */
        line.put("appid", appid);
        line.put("ipport", ipport);

        irb.setSource(line);

        bulkRequest.add(irb);
    }
}
 

private IndexRequestBuilder getIndexRequestBuilderForAProduct(Product product, ElasticSearchIndexConfig config) throws IOException
{
    XContentBuilder contentBuilder = getXContentBuilderForAProduct(product);
    
    IndexRequestBuilder indexRequestBuilder = searchClientService.getClient().prepareIndex(config.getIndexAliasName(), config.getDocumentType(), String.valueOf(product.getId()));

    indexRequestBuilder.setSource(contentBuilder);

    return indexRequestBuilder;
}
 

private IndexRequestBuilder getIndexRequestBuilderForAProductProperty(Product product, ProductProperty productProperty, ElasticSearchIndexConfig config) throws IOException
{
    XContentBuilder contentBuilder = getXContentBuilderForAProductProperty(productProperty);
    
    String documentId = String.valueOf(product.getId()) + String.valueOf(productProperty.getId()) + "0000";
    logger.debug("Generated XContentBuilder for document id {} is {}", new Object[]{documentId, contentBuilder.prettyPrint().string()});

    IndexRequestBuilder indexRequestBuilder = searchClientService.getClient().prepareIndex(config.getIndexAliasName(), config.getPropertiesDocumentType(), documentId);

    indexRequestBuilder.setSource(contentBuilder);

    return indexRequestBuilder;
}
 

private IndexRequestBuilder getIndexRequestBuilderForAProductGroup(ProductGroup productGroup, ElasticSearchIndexConfig config) throws IOException
{
    XContentBuilder contentBuilder = getXContentBuilderForAProductGroup(productGroup);
    
    logger.debug("Generated XContentBuilder for document id {} is {}", new Object[]{productGroup.getId(), contentBuilder.prettyPrint().string()});

    IndexRequestBuilder indexRequestBuilder = searchClientService.getClient().prepareIndex(config.getIndexAliasName(), config.getGroupDocumentType(), String.valueOf(productGroup.getId()));

    indexRequestBuilder.setSource(contentBuilder);

    return indexRequestBuilder;
}