org.elasticsearch.client.sniff.Sniffer Java Examples

public void init() {
    RestClientBuilder restClientBuilder = RestClient.builder(parseHttpHost(esHosts)
            .toArray(new HttpHost[0]));

    if (this.settings != null && this.settings.size() > 0 &&
            !Strings.isNullOrEmpty(this.settings.get("username"))
            && !Strings.isNullOrEmpty(this.settings.get("password"))) {
        String userName = this.settings.get("username");
        String password = this.settings.get("password");
        final CredentialsProvider credentialsProvider =
                new BasicCredentialsProvider();
        credentialsProvider.setCredentials(AuthScope.ANY,
                new UsernamePasswordCredentials(userName, password));
        restClientBuilder.setHttpClientConfigCallback(httpAsyncClientBuilder -> {
            httpAsyncClientBuilder.disableAuthCaching();
            return httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
        });
    }
    restHighLevelClient = new RestHighLevelClient(restClientBuilder);
    this.restLowLevelClient = restHighLevelClient.getLowLevelClient();
    if (sniff) {
        sniffer = Sniffer.builder(restLowLevelClient).setSniffIntervalMillis(5 * 60 * 1000).build();
    }
}
 

@Override
public void start() {
    prefs = new PreferencesReader(AlarmLogTableApp.class, "/alarm_logging_preferences.properties");
    try {
        client = new RestHighLevelClient(
                RestClient.builder(new HttpHost(prefs.get("es_host"), Integer.valueOf(prefs.get("es_port")))));
        if (prefs.get("es_sniff").equals("true")) {
            sniffer = Sniffer.builder(client.getLowLevelClient()).build();
            logger.log(Level.INFO, "ES Sniff feature is enabled");
        }
    } catch (Exception e) {
        logger.log(Level.WARNING, "Failed to properly create the elastic rest client to: " + prefs.get("es_host")
                + ":" + prefs.get("es_port"), e);
    }

}
 

private void addSniffer(HttpHost[] hosts) {
  if (conf.clientSniff) {
    switch (hosts[0].getSchemeName()) {
      case "http":
        sniffer = Sniffer.builder(restClient).build();
        break;
      case "https":
        HostsSniffer hostsSniffer = new ElasticsearchHostsSniffer(
            restClient,
            ElasticsearchHostsSniffer.DEFAULT_SNIFF_REQUEST_TIMEOUT,
            ElasticsearchHostsSniffer.Scheme.HTTPS
        );
        sniffer = Sniffer.builder(restClient).setHostsSniffer(hostsSniffer).build();
        break;
      default:
        // unsupported scheme. do nothing.
    }
  }
}
 

private RestHighLevelClient initClient(List<HttpHost> httpHosts, boolean useSsl, ClusterMode mode, String token) {
    RestHighLevelClient restHighLevelClient;
    RestClientBuilder restClientBuilder = RestClient.builder(httpHosts.toArray(new HttpHost[0]))
            .setNodeSelector(NodeSelector.SKIP_DEDICATED_MASTERS)
            .setRequestConfigCallback(builder -> builder.setConnectTimeout(50000)
                    .setSocketTimeout(600000));
    if (useSsl) {
        restClientBuilder
                .setHttpClientConfigCallback(httpAsyncClientBuilder -> {
                    httpAsyncClientBuilder.setSSLHostnameVerifier((s, sslSession) -> true);
                    SSLContext sslContext;
                    try {
                        sslContext = SSLContext.getInstance("TLS");
                        sslContext.init(null, TRUST_ALL_CERTS, new java.security.SecureRandom());
                    } catch (NoSuchAlgorithmException | KeyManagementException e) {
                        throw new RuntimeException(e.getMessage());
                    }
                    httpAsyncClientBuilder.setSSLContext(sslContext);
                    String basicToken = Base64.getEncoder().encodeToString(token.getBytes());
                    httpAsyncClientBuilder.setDefaultHeaders(Collections.singletonList(new BasicHeader("Authorization", "Basic " + basicToken)));
                    return httpAsyncClientBuilder;
                });
    }
    if (ClusterMode.CLUSTER.equals(mode)) {
        SniffOnFailureListener sniffOnFailureListener = new SniffOnFailureListener();
        restClientBuilder.setFailureListener(sniffOnFailureListener);
        restHighLevelClient = new RestHighLevelClient(restClientBuilder);
        NodesSniffer nodesSniffer = new ElasticsearchNodesSniffer(restHighLevelClient.getLowLevelClient(),
                ElasticsearchNodesSniffer.DEFAULT_SNIFF_REQUEST_TIMEOUT,
                useSsl ? ElasticsearchNodesSniffer.Scheme.HTTPS : ElasticsearchNodesSniffer.Scheme.HTTP);
        Sniffer sniffer = Sniffer.builder(restHighLevelClient.getLowLevelClient()).setSniffIntervalMillis(5000).setNodesSniffer(nodesSniffer).build();
        sniffOnFailureListener.setSniffer(sniffer);
    } else {
        restHighLevelClient = new RestHighLevelClient(restClientBuilder);
    }
    return restHighLevelClient;
}
 

@GET
@Path("/sniffer")
@Produces(MediaType.TEXT_PLAIN)
public String testSniffer() throws IOException, InterruptedException {
    try (RestClient restClient = createRestClient()) {
        Sniffer sniffer = Sniffer.builder(restClient).setSniffIntervalMillis(5).build();

        // Wait for a few iterations of the sniffer
        Thread.sleep(20);

        sniffer.close();

        return "OK";
    }
}
 

@Autowired
public EsRestClient(JkesSearchProperties jkesProperties) {
    SniffOnFailureListener sniffOnFailureListener = new SniffOnFailureListener();
    Header[] defaultHeaders = {new BasicHeader("Content-Type", "application/json")};

    String[] urls = jkesProperties.getEs().getServers().split("\\s*,");
    HttpHost[] hosts = new HttpHost[urls.length];
    for (int i = 0; i < urls.length; i++) {
        hosts[i] = HttpHost.create(urls[i]);
    }

    RestClient restClient = RestClient.builder(hosts)
            .setRequestConfigCallback(requestConfigBuilder -> {
                return requestConfigBuilder.setConnectTimeout(5000) // default 1s
                        .setSocketTimeout(60000); // defaults to 30 seconds
            }).setHttpClientConfigCallback(httpClientBuilder -> {
                return httpClientBuilder.setDefaultIOReactorConfig(
                        IOReactorConfig.custom().setIoThreadCount(2).build()); // because only used for admin, so not necessary to hold many worker threads
            })
            .setMaxRetryTimeoutMillis(60000) // defaults to 30 seconds
            .setDefaultHeaders(defaultHeaders)
            .setFailureListener(sniffOnFailureListener)
            .build();

    Sniffer sniffer = Sniffer.builder(restClient).build();
    sniffOnFailureListener.setSniffer(sniffer);

    this.sniffer = sniffer;
    this.restClient = restClient;
}
 

@Inject
public EsRestClient(JkesProperties jkesProperties) {
    SniffOnFailureListener sniffOnFailureListener = new SniffOnFailureListener();
    Header[] defaultHeaders = {new BasicHeader("Content-Type", "application/json")};

    String[] urls = jkesProperties.getEsBootstrapServers().split("\\s*,");
    HttpHost[] hosts = new HttpHost[urls.length];
    for (int i = 0; i < urls.length; i++) {
        hosts[i] = HttpHost.create(urls[i]);
    }

    RestClient restClient = RestClient.builder(hosts)
            .setRequestConfigCallback(requestConfigBuilder -> {
                return requestConfigBuilder.setConnectTimeout(5000) // default 1s
                        .setSocketTimeout(60000); // defaults to 30 seconds
            }).setHttpClientConfigCallback(httpClientBuilder -> {
                return httpClientBuilder.setDefaultIOReactorConfig(
                        IOReactorConfig.custom().setIoThreadCount(2).build()); // because only used for admin, so not necessary to hold many worker threads
            })
            .setMaxRetryTimeoutMillis(60000) // defaults to 30 seconds
            .setDefaultHeaders(defaultHeaders)
            .setFailureListener(sniffOnFailureListener)
            .build();

    Sniffer sniffer = Sniffer.builder(restClient).build();
    sniffOnFailureListener.setSniffer(sniffer);

    this.sniffer = sniffer;
    this.restClient = restClient;
}
 

private Cluster createCluster(List<String> hosts, boolean enableSniffer) {
    HttpHost[] nodes = hosts.stream().map(HttpHost::create).toArray(HttpHost[]::new);
    Header[] defaultHeaders = new Header[]{new BasicHeader(HEADER_CONTENT_TYPE_KEY, DEFAULT_HEADER_CONTENT_TYPE)};

    RestClient client = RestClient.builder(nodes)
            .setDefaultHeaders(defaultHeaders)
            .setFailureListener(new LoggingFailureListener())
            .build();

    if (enableSniffer) {
        return new Cluster(client, Sniffer.builder(client).build());
    } else {
        return new Cluster(client);
    }
}
 

private void initESClient() throws NumberFormatException,
        UnknownHostException {

    List<HttpHost> httpHostList = hosts.stream().map(hostString -> {
        return HttpHost.create(hostString);
    }).collect(toList());
    List<HttpHost> clusterHosts = unmodifiableList(httpHostList);

    if (config.containsKey("user") && config.containsKey("password")) {
        String user = config.get("user").toString();
        String password = config.get("password").toString();
        final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        credentialsProvider.setCredentials(AuthScope.ANY,
                new UsernamePasswordCredentials(user, password));

        restClient = RestClient.builder(clusterHosts.toArray(new HttpHost[clusterHosts.size()]))
                .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
                    @Override
                    public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
                        return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
                    }
                }).build();
    } else {
        restClient = RestClient.builder(clusterHosts.toArray(new HttpHost[clusterHosts.size()]))
                .build();
    }
    if (this.isSniff) {
        sniffer = Sniffer.builder(restClient).build();
    }

}
 

public static ElasticSearchConnection getConnection(Map stormConf,
        String boltType, BulkProcessor.Listener listener) {

    String flushIntervalString = ConfUtils.getString(stormConf,
            "es." + boltType + ".flushInterval", "5s");

    TimeValue flushInterval = TimeValue.parseTimeValue(flushIntervalString,
            TimeValue.timeValueSeconds(5), "flushInterval");

    int bulkActions = ConfUtils.getInt(stormConf,
            "es." + boltType + ".bulkActions", 50);

    int concurrentRequests = ConfUtils.getInt(stormConf,
            "es." + boltType + ".concurrentRequests", 1);

    RestHighLevelClient client = getClient(stormConf, boltType);

    boolean sniff = ConfUtils.getBoolean(stormConf,
            "es." + boltType + ".sniff", true);
    Sniffer sniffer = null;
    if (sniff) {
        sniffer = Sniffer.builder(client.getLowLevelClient()).build();
    }

    BulkProcessor bulkProcessor = BulkProcessor
            .builder((request, bulkListener) -> client.bulkAsync(request,
                    RequestOptions.DEFAULT, bulkListener), listener)
            .setFlushInterval(flushInterval).setBulkActions(bulkActions)
            .setConcurrentRequests(concurrentRequests).build();

    return new ElasticSearchConnection(client, bulkProcessor, sniffer);
}
 

private RestHighLevelClient createEsClient() {
  HttpHost[] hosts = new HttpHost[config.esHosts.length];
  int i = 0;
  for (String host : config.esHosts) {
    try {
      URL url = new URL(host);
      hosts[i] = new HttpHost(url.getHost(), url.getPort(), url.getProtocol());
      i++;
    } catch (MalformedURLException e) {
      throw new IllegalArgumentException(e.getMessage(), e);
    }
  }

  SniffOnFailureListener sniffOnFailureListener =
    new SniffOnFailureListener();

  RestClientBuilder builder =
      RestClient.builder(hosts)
          .setRequestConfigCallback(
              requestConfigBuilder ->
                  requestConfigBuilder
                      .setConnectTimeout(config.esConnectTimeout)
                      .setSocketTimeout(config.esSocketTimeout))
          .setMaxRetryTimeoutMillis(config.esSocketTimeout)
          .setNodeSelector(NodeSelector.SKIP_DEDICATED_MASTERS)
          .setFailureListener(sniffOnFailureListener);

  RestHighLevelClient highLevelClient = new RestHighLevelClient(builder);

  esSniffer =
    Sniffer.builder(highLevelClient.getLowLevelClient())
      .setSniffIntervalMillis(config.esSniffInterval)
      .setSniffAfterFailureDelayMillis(config.esSniffAfterFailureDelay)
      .build();
  sniffOnFailureListener.setSniffer(esSniffer);

  return highLevelClient;
}
 

private static BulkProcessor setUpBulkProcessor(ElasticsearchConfiguration elasticsearch) throws IOException, GarmadonEsException {
    final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();

    LogFailureListener sniffOnFailureListener = new LogFailureListener();
    RestClientBuilder restClientBuilder = RestClient.builder(
        new HttpHost(elasticsearch.getHost(), elasticsearch.getPort(), "http")
    )
        .setFailureListener(sniffOnFailureListener)
        .setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder
            .setConnectTimeout(CONNECTION_TIMEOUT_MS)
            .setSocketTimeout(SOCKET_TIMEOUT_MS)
            .setContentCompressionEnabled(true))
        .setMaxRetryTimeoutMillis(2 * SOCKET_TIMEOUT_MS);

    if (elasticsearch.getUser() != null) {
        credentialsProvider.setCredentials(AuthScope.ANY,
            new UsernamePasswordCredentials(elasticsearch.getUser(), elasticsearch.getPassword()));

        restClientBuilder
            .setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder
                .setDefaultCredentialsProvider(credentialsProvider));
    }

    //setup es client
    RestHighLevelClient esClient = new RestHighLevelClient(restClientBuilder);

    putGarmadonTemplate(esClient, elasticsearch);
    putGarmadonLifecyclePolicy(esClient, elasticsearch);

    Sniffer sniffer = Sniffer.builder(esClient.getLowLevelClient()).build();
    sniffOnFailureListener.setSniffer(sniffer);

    BiConsumer<BulkRequest, ActionListener<BulkResponse>> bulkConsumer =
        (request, bulkListener) -> esClient.bulkAsync(request, RequestOptions.DEFAULT, bulkListener);

    return BulkProcessor.builder(bulkConsumer, new ElasticSearchListener())
        .setBulkActions(elasticsearch.getBulkActions())
        .setBulkSize(new ByteSizeValue(elasticsearch.getBulkSizeMB(), ByteSizeUnit.MB))
        .setFlushInterval(TimeValue.timeValueSeconds(elasticsearch.getBulkFlushIntervalSec()))
        .setConcurrentRequests(elasticsearch.getBulkConcurrent())
        .setBackoffPolicy(
            BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(100), NB_RETRIES)
        )
        .build();
}
 

private ElasticSearchConnection(RestHighLevelClient c, BulkProcessor p,
        Sniffer s) {
    processor = p;
    client = c;
    sniffer = s;
}
 

/**
 * Default constructor for the cluster
 *
 * @param client  rest client that provides access to the cluster
 * @param sniffer sniffer that notices differences in the cluster architecture
 */
public Cluster(RestClient client, Sniffer sniffer) {
    this.client = client;
    this.sniffer = sniffer;
}
 

/**
 * Obtain the sniffer for this cluster
 *
 * @return sniffer
 */
public Sniffer getSniffer() {
    return sniffer;
}