org.elasticsearch.client.transport.TransportClient Java Examples

@Override
protected void populateData(TransportClient tc) {

    try {
        tc.admin().indices().create(new CreateIndexRequest("logs").mapping("_doc", FileHelper.loadFile("dlsfls/masked_field_mapping.json"), XContentType.JSON)).actionGet();


        byte[] data = FileHelper.loadFile("dlsfls/logs_bulk_data.json").getBytes(StandardCharsets.UTF_8);
        BulkRequest br = new BulkRequest().add(data, 0, data.length, XContentType.JSON).setRefreshPolicy(RefreshPolicy.IMMEDIATE);
        if(tc.bulk(br).actionGet().hasFailures()) {
            Assert.fail("bulk import failed");
        }
        Thread.sleep(1000);

    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail(e.toString());
    }

}
 

private String[] getIndexes() {

		Settings settings = Settings.builder().put("cluster.name", "cluster1").build();
		try (TransportClient client = new PreBuiltTransportClient(settings)) {
			client.addTransportAddress(
					new TransportAddress(InetAddress.getByName("localhost"), embeddedElastic.getTransportTcpPort()));

			return client.admin()
					.indices()
					.getIndex(new GetIndexRequest())
					.actionGet()
					.getIndices();
		} catch (UnknownHostException e) {
			throw new IllegalStateException(e);
		}

	}
 

@Test
public void failureHandlerExecutesFailoverForEachBatchItemSeparately() {

    // given
    Builder builder = createTestObjectFactoryBuilder();
    ClientObjectFactory<TransportClient, BulkRequest> config = builder.build();

    FailoverPolicy failoverPolicy = spy(new NoopFailoverPolicy());

    String payload1 = "test1";
    String payload2 = "test2";
    BulkRequest bulk = new BulkRequest()
            .add(spy(new IndexRequest().source(payload1, XContentType.CBOR)))
            .add(spy(new IndexRequest().source(payload2, XContentType.CBOR)));

    // when
    config.createFailureHandler(failoverPolicy).apply(bulk);

    // then
    ArgumentCaptor<String> captor = ArgumentCaptor.forClass(String.class);
    verify(failoverPolicy, times(2)).deliver(captor.capture());

    assertTrue(captor.getAllValues().contains(payload1));
    assertTrue(captor.getAllValues().contains(payload2));
}
 

@Override
protected void populateData(TransportClient tc) {



    LocalDateTime yesterday = LocalDateTime.now(ZoneId.of("UTC")).minusDays(1);
    LocalDateTime today = LocalDateTime.now(ZoneId.of("UTC"));
    LocalDateTime tomorrow = LocalDateTime.now(ZoneId.of("UTC")).plusDays(1);
    DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy/MM/dd");
    
    tc.index(new IndexRequest("logstash").type("_doc").id("1").setRefreshPolicy(RefreshPolicy.IMMEDIATE)
            .source("{\"@timestamp\": \""+formatter.format(yesterday)+"\"}", XContentType.JSON)).actionGet();
    tc.index(new IndexRequest("logstash").type("_doc").id("2").setRefreshPolicy(RefreshPolicy.IMMEDIATE)
            .source("{\"@timestamp\": \""+formatter.format(today)+"\"}", XContentType.JSON)).actionGet();
    tc.index(new IndexRequest("logstash").type("_doc").id("3").setRefreshPolicy(RefreshPolicy.IMMEDIATE)
            .source("{\"@timestamp\": \""+formatter.format(tomorrow)+"\"}", XContentType.JSON)).actionGet();
}
 

@Test
public void configReturnsACopyOfServerUrisList() {

    // given
    Builder builder = createTestObjectFactoryBuilder();
    builder.withServerUris("http://localhost:9200;http://localhost:9201;http://localhost:9202");
    ClientObjectFactory<TransportClient, BulkRequest> config = builder.build();

    // when
    Collection<String> serverUrisList = config.getServerList();
    serverUrisList.add("test");

    // then
    assertNotEquals(serverUrisList.size(), config.getServerList().size());

}
 

@Override
public Client get()
{
    try {
        Settings settings = Settings.builder().put("cluster.name", clusterName)
                .put("client.transport.sniff", true).build();

        TransportClient client = new PreBuiltTransportClient(settings);
        for (String ip : hosts.split(",")) {
            client.addTransportAddress(
                    new TransportAddress(InetAddress.getByName(ip.split(":")[0]),
                            Integer.parseInt(ip.split(":")[1])));
        }
        return client;
    }
    catch (IOException e) {
        throw new PrestoException(UNEXPECTED_ES_ERROR, "Failed to get connection to Elasticsearch", e);
    }
}
 

@Test
public void failureHandlerExecutesFailoverForEachBatchItemSeparately() {

    // given
    Builder builder = createTestObjectFactoryBuilder();
    ClientObjectFactory<TransportClient, BulkRequest> config = builder.build();

    FailoverPolicy failoverPolicy = spy(new NoopFailoverPolicy());

    String payload1 = "test1";
    String payload2 = "test2";
    BulkRequest bulk = new BulkRequest()
            .add(spy(new IndexRequest().source(payload1)))
            .add(spy(new IndexRequest().source(payload2)));

    // when
    config.createFailureHandler(failoverPolicy).apply(bulk);

    // then
    ArgumentCaptor<String> captor = ArgumentCaptor.forClass(String.class);
    verify(failoverPolicy, times(2)).deliver((String) captor.capture());

    assertTrue(captor.getAllValues().contains(payload1));
    assertTrue(captor.getAllValues().contains(payload2));
}
 

public Client createClient() {
    final AtomicReference<Client> elasticsearchClient = new AtomicReference<>();
    await().atMost(30, TimeUnit.SECONDS).pollDelay(1, TimeUnit.SECONDS).until(() -> {
        Client c = new TransportClient(ImmutableSettings.settingsBuilder().put("cluster.name", elasticsearchClusterName).build()).addTransportAddress(new InetSocketTransportAddress(getIpAddress(), 9300));
        try {
            c.admin().cluster().health(Requests.clusterHealthRequest("_all")).actionGet();
        } catch (ElasticsearchException e) {
            c.close();
            return false;
        }
        elasticsearchClient.set(c);
        return true;
    });
    assertEquals(elasticsearchClusterName, elasticsearchClient.get().admin().cluster().health(Requests.clusterHealthRequest("_all")).actionGet().getClusterName());
    return elasticsearchClient.get();
}
 

/**
 * 多字段查询
 */
public static void multisearch() {
    try {
        Settings settings = Settings.settingsBuilder().put("cluster.name", "elasticsearch1").build();
        TransportClient transportClient = TransportClient.builder().
                settings(settings).build().addTransportAddress(
                new InetSocketTransportAddress(InetAddress.getByName("172.16.2.93"), 9300));
        SearchRequestBuilder searchRequestBuilder = transportClient.prepareSearch("service2","clients");
        SearchResponse searchResponse = searchRequestBuilder.
                setQuery(QueryBuilders.boolQuery()
                        .should(QueryBuilders.termQuery("id","5"))
                        .should(QueryBuilders.prefixQuery("content","oracle")))
                .setFrom(0).setSize(100).setExplain(true).execute().actionGet();
        SearchHits searchHits = searchResponse.getHits();
        System.out.println();
        System.out.println("Total Hits is " + searchHits.totalHits());
        System.out.println();
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 

/**
 * 批量删除
 *
 * @param transportClient
 */
private static void batchDelete(TransportClient transportClient) throws IOException {
	BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();

	DeleteRequestBuilder deleteRequestBuilder1 = transportClient.prepareDelete("product_index", "product", "1");
	DeleteRequestBuilder deleteRequestBuilder2 = transportClient.prepareDelete("product_index", "product", "2");
	DeleteRequestBuilder deleteRequestBuilder3 = transportClient.prepareDelete("product_index", "product", "3");

	bulkRequestBuilder.add(deleteRequestBuilder1);
	bulkRequestBuilder.add(deleteRequestBuilder2);
	bulkRequestBuilder.add(deleteRequestBuilder3);

	BulkResponse bulkResponse = bulkRequestBuilder.get();
	for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) {
		logger.info("--------------------------------version= " + bulkItemResponse.getVersion());
	}

}
 

public TransportClient getClient()   {
    if (esClient == null) {
        synchronized (this) {
            if (esClient == null) {
                try {
                    //判断配置
                    Preconditions.checkNotNull(clusterName, "es 服务clusterName未配置");
                    Preconditions.checkNotNull(addresses, "es 服务ip未配置");
                    //Preconditions.checkArgument(esPort > 0, "es 服务服务port未配置");
                    //设置集群的名字
                    Settings settings = Settings.settingsBuilder().put("client.node", true).put("cluster.name", clusterName).put("client.transport.sniff", sniff).build();
                    //Settings settings = Settings.settingsBuilder().put("client.transport.sniff", sniff).build();
                    //创建集群client并添加集群节点地址
                    esClient = TransportClient.builder().settings(settings).build();
                    for (String address : addresses) {
                        String[] inetAddress = address.split(":");
                        esClient.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(inetAddress[0]), new Integer(inetAddress[1])));
                    }
                  }catch (Exception e){
                     LOGGER.error("客户端连接初始化异常",e);
                }
            }
        }
    }
    return esClient;
}
 

protected ShellNativeClient newTransportClient(TransportAddress... addresses) {

        Settings settings = ImmutableSettings.settingsBuilder().put("client.transport.ignore_cluster_name", true).build();
        org.elasticsearch.client.transport.TransportClient client = new TransportClient(settings).addTransportAddresses(addresses);

        //if no connected node we can already close the (useless) client
        if (client.connectedNodes().size() == 0) {
            client.close();
            return null;
        }

        AbstractClient<TransportClient, JsonInput, JsonOutput> shellClient = clientWrapper.wrapEsTransportClient(client);
        resourceRegistry.registerResource(shellClient);
        ShellNativeClient shellNativeClient = clientWrapper.wrapShellClient(shellClient);
        clientScopeSynchronizerRunner.startSynchronizer(shellNativeClient);
        return shellNativeClient;
    }
 

public TransportBasedClient(Properties props) throws UnknownHostException {
  final String host =
      props.getProperty(ElasticsearchInterpreter.ELASTICSEARCH_HOST);
  final int port = Integer.parseInt(
      props.getProperty(ElasticsearchInterpreter.ELASTICSEARCH_PORT));
  final String clusterName =
      props.getProperty(ElasticsearchInterpreter.ELASTICSEARCH_CLUSTER_NAME);

  final Settings settings = Settings.settingsBuilder()
      .put("cluster.name", clusterName)
      .put(props)
      .build();

  client = TransportClient.builder().settings(settings).build()
      .addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(host), port));
}
 

@Override
public void init(Map<String, String> props) {
    String host = props.get(PropConst.HOST);
    if (host == null) {
        throw new IllegalArgumentException("Host does not specified");
    }
    String port = props.get(PropConst.PORT);
    if (port == null) {
        throw new IllegalArgumentException("Port does not specified");
    }
    String clusterName = props.get(PropConst.CLUSTER_NAME);
    if (clusterName == null) {
        throw new IllegalArgumentException("Cluster name does not specified");
    }
    client = (new TransportClient(ImmutableSettings.settingsBuilder().put("cluster.name", clusterName).build()))
            .addTransportAddress(new InetSocketTransportAddress(host, Integer.valueOf(port)));
}
 

private Client initTransportClient() throws UnknownHostException {
    TransportClient client = null;
    try {
        Settings settings = Settings.builder()
                                    .put("cluster.name", "docker-node")
                                    .put("client.transport.sniff", false)
                                    .build();

        client = new PreBuiltTransportClient(settings).addTransportAddress(new InetSocketTransportAddress(InetAddress
            .getByName(host), 9300));
    } catch (UnknownHostException e) {
        logger.error("create client error", e);
        throw e;
    }
    return client;
}
 

/**
    * 使用elasticsearch实现类时才触发
    *
    * @return
    */
@Bean
   @ConditionalOnBean(value = EsLogServiceImpl.class)
public TransportClient getESClient() {
	// 设置集群名字
	Settings settings = Settings.builder().put("cluster.name", this.clusterName).build();
	TransportClient client = new PreBuiltTransportClient(settings);
	try {
		// 读取的ip列表是以逗号分隔的
		for (String clusterNode : this.clusterNodes.split(",")) {
			String ip = clusterNode.split(":")[0];
			String port = clusterNode.split(":")[1];
			((TransportClient) client)
					.addTransportAddress(new TransportAddress(InetAddress.getByName(ip), Integer.parseInt(port)));
		}
	} catch (UnknownHostException e) {
		e.printStackTrace();
	}

	return client;
}
 

@Test
public void clientIsInitializedOnlyOnce() {

    // given
    BulkProcessorObjectFactory factory = spy(createTestObjectFactoryBuilder().build());

    BulkProcessorObjectFactory.InsecureTransportClientProvider clientProvider =
            new BulkProcessorObjectFactory.InsecureTransportClientProvider();

    when(factory.getClientProvider()).thenReturn(spy(clientProvider));

    // when
    TransportClient client1 = factory.createClient();
    TransportClient client2 = factory.createClient();

    // then
    verify(factory, times(1)).getClientProvider();
    assertEquals(client1, client2);

}
 

/**
 * @see org.elasticsearch.transport.TcpTransport#TCP_CONNECT_TIMEOUT
 */
public AbstractClient esClient() throws Exception {
  // https://discuss.elastic.co/t/getting-availableprocessors-is-already-set-to-1-rejecting-1-illegalstateexception-exception/103082
  System.setProperty("es.set.netty.runtime.available.processors", "false");

  TransportClient client = new PreBuiltXPackTransportClient(settings());
  for (String clusterNode : getClusterNodes()) {
    String hostName = substringBeforeLast(clusterNode, COLON);
    String port = substringAfterLast(clusterNode, COLON);
    Assert.hasText(hostName, "[Assertion failed] missing host name in 'clusterNodes'");
    Assert.hasText(port, "[Assertion failed] missing port in 'clusterNodes'");
    logger.info("Adding transport node : {}, timeout in 30s", clusterNode);
    client.addTransportAddress(
        new InetSocketTransportAddress(InetAddress.getByName(hostName), Integer.valueOf(port)));
  }
  return client;
}
 

@Bean
public TransportClient getTransportClient() throws Exception {
    Settings settings = Settings.builder().put("cluster.name", clusterName)
            .put("client.transport.sniff", true)
            .build();
    transportClient = new PreBuiltTransportClient(settings)
            .addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(host), Integer.valueOf(port)));
    logger.info("elasticsearch transportClient 连接成功");
    return transportClient;
}
 

@Bean
public TransportClient transportClient(Settings settings) {
    TransportClient client = TransportClient.builder().settings(settings).build();
    for (String ip : this.esProperties.getIps().split(Constants.COMMA)) {
        try {
            client.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(ip), this.esProperties.getPort()));
        } catch (UnknownHostException e) {
            LOGGER.error("es集群主机名错误, ip: {}", ip);
        }
    }
    return client;
}
 

public Client getEsClient(Map<String, Integer> esAddresses,Map<String, String> esConfig){
    Settings settings = Settings.builder().put(esConfig).put("client.transport.sniff", true).build();
    TransportClient transportClient = new PreBuiltTransportClient(settings);
    for (Map.Entry<String, Integer> esIpAndPort: esAddresses.entrySet()) {
        transportClient.addTransportAddress(new InetSocketTransportAddress(new InetSocketAddress(esIpAndPort.getKey(), esIpAndPort.getValue())));
    }
    List<DiscoveryNode> discoveryNodes = transportClient.connectedNodes();
    if (discoveryNodes.isEmpty()) {
        throw new RuntimeException("Cannot connect to any elasticsearch nodes");
    }
   return transportClient;
}
 

@Override
protected ElasticsearchSinkBase<Tuple2<Integer, String>, TransportClient> createElasticsearchSink(
		int bulkFlushMaxActions,
		String clusterName,
		List<InetSocketAddress> transportAddresses,
		ElasticsearchSinkFunction<Tuple2<Integer, String>> elasticsearchSinkFunction) {

	return new ElasticsearchSink<>(
			Collections.unmodifiableMap(createUserConfig(bulkFlushMaxActions, clusterName)),
			transportAddresses,
			elasticsearchSinkFunction);
}
 

@Override
protected ElasticsearchSinkBase<Tuple2<Integer, String>, TransportClient> createElasticsearchSinkForNode(
		int bulkFlushMaxActions,
		String clusterName,
		ElasticsearchSinkFunction<Tuple2<Integer, String>> elasticsearchSinkFunction,
		String ipAddress) throws Exception {

	List<InetSocketAddress> transports = new ArrayList<>();
	transports.add(new InetSocketAddress(InetAddress.getByName(ipAddress), 9300));

	return new ElasticsearchSink<>(
			Collections.unmodifiableMap(createUserConfig(bulkFlushMaxActions, clusterName)),
			transports,
			elasticsearchSinkFunction);
}
 

public Client startup() throws UnknownHostException {
	org.elasticsearch.common.settings.Settings settings = org.elasticsearch.common.settings.Settings
			.settingsBuilder().put("cluster.name", cluster).build();
	// on startup
	client = TransportClient.builder().settings(settings).build()
			.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(host), 9300));

	return client;
}
 

@SuppressWarnings({"unchecked", "resource"})
public static void main(String[] args) throws IOException {
	// 先构建client，两个参数分别是：cluster.name 固定参数代表后面参数的含义，集群名称
	// client.transport.sniff 表示设置自动探查集群的集群节点
	Settings settings = Settings.builder()
			.put("cluster.name", "youmeek-cluster")
			.put("client.transport.sniff", true)
			.build();

	//单个节点的写法
	TransportClient transportClient = new PreBuiltTransportClient(settings).addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("192.168.1.127"), 9300));

	//======================================================

	create(transportClient);
	batchCreate(transportClient);
	batchUpdate(transportClient);
	batchDelete(transportClient);
	update(transportClient);
	query(transportClient);
	queryByMatchOneParam(transportClient);
	queryByMatchMoreParam(transportClient);
	queryByTerm(transportClient);
	queryByPrefix(transportClient);
	queryByBool(transportClient);
	queryMore(transportClient);
	queryByMultiGet(transportClient);
	queryByScroll(transportClient);
	queryByTemplate(transportClient);
	delete(transportClient);
	aggregate(transportClient);

	//======================================================

	transportClient.close();
}
 

public EsStore(String host, int port) {
  String clusterName = Configuration.getProperties().getString("es_cluster_name", "soundwave");
  Settings settings = ImmutableSettings.settingsBuilder().put("cluster.name", clusterName)
      .put("client.transport.sniff", false).build();
  esClient = new TransportClient(settings)
      .addTransportAddress(
          new InetSocketTransportAddress(host, port));
}
 

public boolean initializeConnection(String ip, int port,
		String cluster_name, String index_name, String document_name,
		int bulk_size, String date_format) {

	_LOG.info("Initializing ESBulkAdapter...");

	try {
		httpclient = new DefaultHttpClient();
		String alias_link = "http://" + ip + ":" + 9200 + "/_aliases";
		post = new HttpPost(alias_link);

		_index_name = index_name;
		_document_name = document_name;

		_bulk_size = bulk_size - 1;
		

		dateFormat = new SimpleDateFormat(date_format);
		
		element_count = 0;
		running_index_postfix = "NONE";

		Settings settings = ImmutableSettings.settingsBuilder()
				.put("cluster.name", cluster_name).build();
		client = new TransportClient(settings)
				.addTransportAddress(new InetSocketTransportAddress(ip,
						port));

		bulkRequest = client.prepareBulk();

		return true;
	} catch (Exception e) {
		e.printStackTrace();
		return false;
	}
}
 

@Override
protected ElasticsearchSinkBase<Tuple2<Integer, String>, TransportClient> createElasticsearchSinkForEmbeddedNode(
		int bulkFlushMaxActions,
		String clusterName,
		ElasticsearchSinkFunction<Tuple2<Integer, String>> elasticsearchSinkFunction) throws Exception {

	return createElasticsearchSinkForNode(
			bulkFlushMaxActions, clusterName, elasticsearchSinkFunction, "127.0.0.1");
}
 

/**
 * 创建索引
 *
 * @param indexName
 *
 * @return
 */
public boolean createIndex(String indexName) {
    TransportClient client = esClientFactory.getClient();
    CreateIndexResponse response = null;
    // 如果存在返回true
    if (client.admin().indices().prepareExists(esClientFactory.getIndexs(indexName)).get().isExists()) {
        return true;
    } else {
        response = client.admin().indices().prepareCreate(esClientFactory.getIndexs(indexName)).execute().actionGet();
    }
    return response.isAcknowledged();
}
 

/**
 * 批量更新
 *
 * @param transportClient
 */
private static void batchUpdate(TransportClient transportClient) throws IOException {
	BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();

	UpdateRequestBuilder updateRequestBuilder1 = transportClient.prepareUpdate("product_index", "product", "1")
			.setDoc(XContentFactory.jsonBuilder()
					.startObject()
					.field("product_name", "更新后的商品名称1")
					.endObject());

	UpdateRequestBuilder updateRequestBuilder2 = transportClient.prepareUpdate("product_index", "product", "2")
			.setDoc(XContentFactory.jsonBuilder()
					.startObject()
					.field("product_name", "更新后的商品名称2")
					.endObject());

	UpdateRequestBuilder updateRequestBuilder3 = transportClient.prepareUpdate("product_index", "product", "3")
			.setDoc(XContentFactory.jsonBuilder()
					.startObject()
					.field("product_name", "更新后的商品名称3")
					.endObject());


	bulkRequestBuilder.add(updateRequestBuilder1);
	bulkRequestBuilder.add(updateRequestBuilder2);
	bulkRequestBuilder.add(updateRequestBuilder3);

	BulkResponse bulkResponse = bulkRequestBuilder.get();
	for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) {
		logger.info("--------------------------------version= " + bulkItemResponse.getVersion());
	}
}