org.elasticsearch.client.RestClient Java Examples

private void verifyAnomaly(
    String datasetName,
    int intervalMinutes,
    int trainTestSplit,
    int shingleSize,
    double minPrecision,
    double minRecall,
    double maxError
) throws Exception {

    RestClient client = client();

    String dataFileName = String.format("data/%s.data", datasetName);
    String labelFileName = String.format("data/%s.label", datasetName);

    List<JsonObject> data = getData(dataFileName);
    List<Entry<Instant, Instant>> anomalies = getAnomalyWindows(labelFileName);

    indexTrainData(datasetName, data, trainTestSplit, client);
    String detectorId = createDetector(datasetName, intervalMinutes, client);
    startDetector(detectorId, data, trainTestSplit, shingleSize, intervalMinutes, client);

    indexTestData(data, datasetName, trainTestSplit, client);
    double[] testResults = getTestResults(detectorId, data, trainTestSplit, intervalMinutes, anomalies, client);
    verifyTestResults(testResults, anomalies, minPrecision, minRecall, maxError);
}
 

/**
 * Create an index in Elasticsearch. If necessary, this function should check whether a new index
 * is required.
 *
 * @return true if a new index has been created, false otherwise
 */
public boolean createIndex() {
  RestClient client = esrResource.getClient();

  try {
    Response r = client.performRequest("HEAD", "/" + index);

    if (r.getStatusLine().getStatusCode() != 200) {
      client.performRequest("PUT", "/" + index);

      return true;
    }
  } catch (IOException ioe) {
    getMonitor().error("Unable to create index", ioe);
  }

  return false;
}
 

/**
 * Create an alias if needed
 * @param client Client to use
 * @param alias Alias name
 * @param index Index name
 * @throws Exception When alias can not be set
 */
public static void createAlias(RestClient client, String alias, String index) throws Exception {
    logger.trace("createAlias({},{})", alias, index);

    assert client != null;
    assert alias != null;
    assert index != null;

    Request request = new Request("POST", "/_aliases/");
    request.setJsonEntity("{\"actions\":[{\"add\":{\"index\":\"" + index +"\",\"alias\":\"" + alias +"\"}}]}");
    Response response = client.performRequest(request);

    if (response.getStatusLine().getStatusCode() != 200) {
        logger.warn("Could not create alias [{}] on index [{}]", alias, index);
        throw new Exception("Could not create alias ["+alias+"] on index ["+index+"].");
    }
    logger.trace("/createAlias({},{})", alias, index);
}
 

private static RestClient connect(Map<String, Integer> coordinates,
    Map<String, String> userConfig) {
    Objects.requireNonNull(coordinates, "coordinates");
    Preconditions.checkArgument(! coordinates.isEmpty(), "no ES coordinates specified");
    final Set<HttpHost> set = new LinkedHashSet<>();
    for (Map.Entry<String, Integer> entry : coordinates.entrySet()) {
        set.add(new HttpHost(entry.getKey(), entry.getValue()));
    }

    final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
    credentialsProvider.setCredentials(AuthScope.ANY,
        new UsernamePasswordCredentials(userConfig.getOrDefault("esUser", "none"),
            userConfig.getOrDefault("esPass", "none")));

    return RestClient.builder(set.toArray(new HttpHost[0]))
        .setHttpClientConfigCallback(httpClientBuilder ->
            httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider))
        .setMaxRetryTimeoutMillis(300000).build();
}
 

/**
 * Registers an instance of the high level client for Elasticsearch.
 *
 * @return An instance of Elasticsearch's high level rest client
 */
@Bean
public RestHighLevelClient restHighLevelClient() {
    RestClientBuilder builder = RestClient.builder(new HttpHost(configProps.getHost(), configProps.getPort(), configProps.getScheme()));
    RestClientBuilder.HttpClientConfigCallback httpClientConfigCallback = httpAsyncClientBuilder -> {
        httpAsyncClientBuilder
                .setMaxConnTotal(configProps.getRestClientMaxConnectionsTotal())
                .setMaxConnPerRoute(configProps.getRestClientMaxConnectionsPerRoute());

        if (null != configProps.getUser() && !configProps.getUser().isEmpty()) {
            CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
            credentialsProvider.setCredentials(AuthScope.ANY,
                    new UsernamePasswordCredentials(configProps.getUser(), configProps.getPassword()));
            httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
        }

        return httpAsyncClientBuilder;
    };

    builder.setHttpClientConfigCallback(httpClientConfigCallback);
    builder.setMaxRetryTimeoutMillis(configProps.getRestClientMaxRetryTimeoutMillis());

    //return new RestHighLevelClient(builder.build());
    return new RestHighLevelClient(builder);
}
 

@Test
public void canIndexPolicy() {
    PolicyDocument policyDocument = new PolicyDocument(
            "111-111",
            LocalDate.of(2019, 1, 1),
            LocalDate.of(2019, 12, 31),
            "John Smith",
            "SAFE_HOUSE",
            BigDecimal.valueOf(1000),
            "m.smith"
    );

    PolicyElasticRepository repository = new PolicyElasticRepository(
            new RestHighLevelClient(RestClient.builder(new HttpHost("localhost", el.getHttpPort(), "http"))),
            new JsonConverter(objectMapper())
    );

    repository.save(policyDocument);

    PolicyDocument saved = repository.findByNumber("111-111");

    assertNotNull(saved);
}
 

@Bean
@ConditionalOnMissingBean
public RestHighLevelClient restHighLevelClient() {

    List<String> clusterNodes = elasticsearchProperties.getClusterNodes();
    clusterNodes.forEach(node -> {
        try {
            String[] parts = StringUtils.split(node, ":");
            Assert.notNull(parts, "Must defined");
            Assert.state(parts.length == 2, "Must be defined as 'host:port'");
            httpHosts.add(new HttpHost(parts[0], Integer.parseInt(parts[1]), elasticsearchProperties.getSchema()));
        } catch (Exception e) {
            throw new IllegalStateException("Invalid ES nodes " + "property '" + node + "'", e);
        }
    });
    RestClientBuilder builder = RestClient.builder(httpHosts.toArray(new HttpHost[0]));

    return getRestHighLevelClient(builder, elasticsearchProperties);
}
 

@Before
public void setUp() throws Exception {
    /**
     * 如果es集群安装了x-pack插件则以此种方式连接集群
     * 1. java客户端的方式是以tcp协议在9300端口上进行通信
     * 2. http客户端的方式是以http协议在9200端口上进行通信
     */
    Settings settings = Settings.builder(). put("xpack.security.user", "elastic:changeme").build();
    client = new PreBuiltXPackTransportClient(settings)
            .addTransportAddress(new TransportAddress(InetAddress.getByName("localhost"),9200));
    final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
    credentialsProvider.setCredentials(AuthScope.ANY,
            new UsernamePasswordCredentials("elastic", "changeme"));
    restClient = RestClient.builder(new HttpHost("localhost",9200,"http"))
            .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
                @Override
                public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
                    return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
                }
            }).build();
}
 

@Override public Schema create(SchemaPlus parentSchema, String name,
    Map<String, Object> operand) {

  final Map map = (Map) operand;

  final ObjectMapper mapper = new ObjectMapper();
  mapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true);

  try {
    final Map<String, Integer> coordinates =
        mapper.readValue((String) map.get("coordinates"),
            new TypeReference<Map<String, Integer>>() { });

    final RestClient client = connect(coordinates);

    final Map<String, String> userConfig =
        mapper.readValue((String) map.get("userConfig"),
            new TypeReference<Map<String, String>>() { });

    final String index = (String) map.get("index");
    Preconditions.checkArgument(index != null, "index is missing in configuration");
    return new ElasticsearchSchema(client, new ObjectMapper(), index);
  } catch (IOException e) {
    throw new RuntimeException("Cannot parse values from json", e);
  }
}
 

@Override
public void setup() {
  RestClientBuilder restClientBuilder =
      RestClient.builder(new HttpHost(properties.getHost(), properties.getPort(), properties.getScheme()));

  if (StringUtils.isNotEmpty(properties.getUsername()) && StringUtils.isNotEmpty(properties.getPassword())) {
    UsernamePasswordCredentials credentials =
        new UsernamePasswordCredentials(properties.getUsername(), properties.getPassword());

    CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
    credentialsProvider.setCredentials(AuthScope.ANY, credentials);

    restClientBuilder.setHttpClientConfigCallback(
        httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
  } else {
    if (StringUtils.isNotEmpty(properties.getUsername()) || StringUtils.isNotEmpty(properties.getPassword())) {
      log.warn("Both username and password must be configured to setup ES authentication.");
    }
  }

  client = new RestHighLevelClient(restClientBuilder);
}
 

/**
 * Creates an ElasticsearchTable.
 * @param client low-level ES rest client
 * @param mapper Jackson API
 * @param indexName elastic search index
 * @param typeName elastic searh index type
 */
ElasticsearchTable(RestClient client, ObjectMapper mapper, String indexName, String typeName) {
  super(Object[].class);
  this.restClient = Objects.requireNonNull(client, "client");
  try {
    this.version = detectVersion(client, mapper);
  } catch (IOException e) {
    final String message = String.format(Locale.ROOT, "Couldn't detect ES version "
        + "for %s/%s", indexName, typeName);
    throw new UncheckedIOException(message, e);
  }
  this.indexName = Objects.requireNonNull(indexName, "indexName");
  this.typeName = Objects.requireNonNull(typeName, "typeName");
  this.mapper = Objects.requireNonNull(mapper, "mapper");

}
 

/**
 * Update settings in Elasticsearch
 * @param client Elasticsearch client
 * @param index Index name
 * @param settings Settings if any, null if no update settings
 * @throws Exception if the elasticsearch API call is failing
 */
private static void updateIndexWithSettingsInElasticsearch(RestClient client, String index, String settings) throws Exception {
	logger.trace("updateIndex([{}])", index);

	assert client != null;
	assert index != null;


	if (settings != null) {
		logger.trace("Found update settings for index [{}]: [{}]", index, settings);
		logger.debug("updating settings for index [{}]", index);
           Request request = new Request("PUT", "/" + index + "/_settings");
           request.setJsonEntity(settings);
		client.performRequest(request);
	}

	logger.trace("/updateIndex([{}])", index);
}
 

private Map<String, Object> getDetectionResult(String detectorId, Instant begin, Instant end, RestClient client) {
    try {
        Request request = new Request("POST", String.format("/_opendistro/_anomaly_detection/detectors/%s/_run", detectorId));
        request
            .setJsonEntity(
                String.format(Locale.ROOT, "{ \"period_start\": %d, \"period_end\": %d }", begin.toEpochMilli(), end.toEpochMilli())
            );
        return entityAsMap(client.performRequest(request));
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 

private RestHighLevelClient createEsClient() {
  HttpHost[] hosts = new HttpHost[config.esHosts.length];
  int i = 0;
  for (String host : config.esHosts) {
    try {
      URL url = new URL(host);
      hosts[i] = new HttpHost(url.getHost(), url.getPort(), url.getProtocol());
      i++;
    } catch (MalformedURLException e) {
      throw new IllegalArgumentException(e.getMessage(), e);
    }
  }

  SniffOnFailureListener sniffOnFailureListener =
    new SniffOnFailureListener();

  RestClientBuilder builder =
      RestClient.builder(hosts)
          .setRequestConfigCallback(
              requestConfigBuilder ->
                  requestConfigBuilder
                      .setConnectTimeout(config.esConnectTimeout)
                      .setSocketTimeout(config.esSocketTimeout))
          .setMaxRetryTimeoutMillis(config.esSocketTimeout)
          .setNodeSelector(NodeSelector.SKIP_DEDICATED_MASTERS)
          .setFailureListener(sniffOnFailureListener);

  RestHighLevelClient highLevelClient = new RestHighLevelClient(builder);

  esSniffer =
    Sniffer.builder(highLevelClient.getLowLevelClient())
      .setSniffIntervalMillis(config.esSniffInterval)
      .setSniffAfterFailureDelayMillis(config.esSniffAfterFailureDelay)
      .build();
  sniffOnFailureListener.setSniffer(esSniffer);

  return highLevelClient;
}
 

private static void test2(final RestClient client, final HttpEntity entity) {
  final Request request = new Request("PUT", "/twitter/tweet/2");
  request.setEntity(entity);

  client.performRequestAsync(request, new ResponseListener() {
    @Override
    public void onSuccess(final Response response) {
    }

    @Override
    public void onFailure(final Exception exception) {
    }
  });
}
 

private static void runRestClient() throws IOException, InterruptedException {
  try (final RestClient restClient = RestClient.builder(new HttpHost("localhost", HTTP_PORT, "http")).build()) {
    final HttpEntity entity = new NStringEntity(
      "{\n" +
      "    \"user\": \"user\",\n" +
      "    \"post_date\": \"2009-11-15T14:12:12\",\n" +
      "    \"message\": \"trying out Elasticsearch\"\n" +
      "}", ContentType.APPLICATION_JSON);

    final Request request1 = new Request("PUT", "/twitter/tweet/1");
    request1.setEntity(entity);

    final Response indexResponse = restClient.performRequest(request1);
    System.out.println(indexResponse);

    final Request request2 = new Request("PUT", "/twitter/tweet/2");
    request2.setEntity(entity);

    final CountDownLatch latch = new CountDownLatch(1);
    restClient.performRequestAsync(request2, new ResponseListener() {
      @Override
      public void onSuccess(final Response response) {
        latch.countDown();
      }

      @Override
      public void onFailure(final Exception e) {
        latch.countDown();
      }
    });

    latch.await(30, TimeUnit.SECONDS);
  }
}
 

private void initESClient() throws NumberFormatException,
        UnknownHostException {

    List<HttpHost> httpHostList = hosts.stream().map(hostString -> {
        return HttpHost.create(hostString);
    }).collect(toList());
    List<HttpHost> clusterHosts = unmodifiableList(httpHostList);

    if (config.containsKey("user") && config.containsKey("password")) {
        String user = config.get("user").toString();
        String password = config.get("password").toString();
        final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        credentialsProvider.setCredentials(AuthScope.ANY,
                new UsernamePasswordCredentials(user, password));

        restClient = RestClient.builder(clusterHosts.toArray(new HttpHost[clusterHosts.size()]))
                .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
                    @Override
                    public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
                        return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
                    }
                }).build();
    } else {
        restClient = RestClient.builder(clusterHosts.toArray(new HttpHost[clusterHosts.size()]))
                .build();
    }
    if (this.isSniff) {
        sniffer = Sniffer.builder(restClient).build();
    }

}
 

@Autowired
public IndexService(RestClient client) {
    this.client = client;

    if (!initialized) {
        CreateIndexIfNotExist();
        initialized = true;
    }
}
 

private RestClient getRestClient() {
	if (restClient == null) {
		String esHost = config.getElasticSearch().getDevIngestHost();
		int esPort = config.getElasticSearch().getDevIngestPort();
		RestClientBuilder builder = RestClient.builder(new HttpHost(esHost, esPort));
		RequestConfigCallback requestConfigCallback = requestConfigBuilder -> requestConfigBuilder
				.setConnectionRequestTimeout(0);
		builder.setRequestConfigCallback(requestConfigCallback);
		restClient = builder.build();
	}
	return restClient;
}
 

ElasticsearchOps(RestClient restClient, String index, ObjectMapper mapper, int scrollSize) {
  this.transport = new RxJavaTransport(restClient);
  this.mapper = Objects.requireNonNull(mapper, "mapper");
  this.index = Objects.requireNonNull(index, "index");
  Preconditions.checkArgument(scrollSize > 0, "Invalid scrollSize: %s", scrollSize);
  this.scrollSize = scrollSize;
  IndexOps ops = new IndexOps(restClient, mapper, index);
  // cache mapping and version
  this.mapping = ops.mapping().blockingGet();
  this.version = ops.version().blockingGet();
}
 

/**
 * Gets the rest client.
 *
 * @return the rest client
 */
private RestClient getRestClient() {
	if (restClient == null) {
		restClient = RestClient.builder(new HttpHost(updateESHost, updateESPort)).build();
	}
	return restClient;
}
 

ElasticsearchTransport(final RestClient restClient,
                       final ObjectMapper mapper,
                       final String indexName,
                       final int fetchSize) {
  this.mapper = Objects.requireNonNull(mapper, "mapper");
  this.restClient = Objects.requireNonNull(restClient, "restClient");
  this.indexName = Objects.requireNonNull(indexName, "indexName");
  this.fetchSize = fetchSize;
  this.version = version(); // cache version
  this.mapping = fetchAndCreateMapping(); // cache mapping
}
 

public MetricStoreImpl(@Value("${elasticsearch.host}") String host,
                       @Value("${elasticsearch.port}") int port,
                       @Value("${elasticsearch.scheme:http}") String scheme,
                       @Value("${elasticsearch.user:}") String user,
                       @Value("${elasticsearch.password:}") String password) {
    HttpHost httpHost = new HttpHost(host, port, scheme);
    RestClientBuilder builder = RestClient.builder(httpHost);
    if (!user.isEmpty() && !password.isEmpty()) {
        String encodedAuth = buildBasicAuthString(user, password);
        Header[] requestHeaders = new Header[]{
            new BasicHeader(org.apache.http.HttpHeaders.AUTHORIZATION,
                encodedAuth)};
        builder.setDefaultHeaders(requestHeaders);
    }
    this.client = builder.build();
    HttpHeaders responseHeaders = new HttpHeaders();
    responseHeaders.setContentType(MediaType.APPLICATION_JSON);
    this.responseHeaders = responseHeaders;
    String urlBase = String.format("/%s/%s", INDEX, TYPE);
    this.urlGet = urlBase.concat("/_search?filter_path=hits.hits._source");
    this.urlPost = urlBase.concat("/_bulk");
    this.urlDelete = urlBase.concat("/_delete_by_query");
    this.indexMetaData = String.format(
        "{ \"index\" : { \"_index\" : " +
            "\"%s\",\"_type\" : \"%s\" } }%n",
        INDEX,
        TYPE);
    this.mapper = new ObjectMapper();
}
 

public Visualizer(Logger logger, List<String> hosts, List<Integer> ports) {
    jsonParser = new JsonParser();
    gson = new GsonBuilder().serializeNulls().create();
    this.logger = logger;

    HttpHost[] httpHosts = new HttpHost[hosts.size()];
    for (int i=0;i< hosts.size();i++){
        HttpHost host = new HttpHost(hosts.get(i), ports.get(i),"http");
        httpHosts[i] = host;
    }
    esClient = RestClient.builder(httpHosts).build();
}
 

ElasticsearchIOTestCommon(
    ConnectionConfiguration connectionConfiguration, RestClient restClient, boolean useAsITests) {
  this.connectionConfiguration = connectionConfiguration;
  this.restClient = restClient;
  this.numDocs = useAsITests ? NUM_DOCS_ITESTS : NUM_DOCS_UTESTS;
  this.useAsITests = useAsITests;
}
 

@BeforeClass
public static void setUp() throws Exception {
  ElasticsearchBaseIT.setUp();

  // Create index and add data
  HttpHost host = new HttpHost("127.0.0.1", esHttpPort);
  restClient = RestClient.builder(host).build();

  HttpEntity entity = new StringEntity(
      "{\"mappings\":{\"tweet\":{\"properties\":{\"message\":{\"type\":\"text\"},\"timestamp\":{\"type\":\"date\"}}}}}"
  );
  restClient.performRequest(
      "PUT",
      "/twitter",
      new HashMap<>(),
      entity
  );

  final int numTestDocs = 100;
  for (int i = 1; i <= numTestDocs; i++) {
    restClient.performRequest("PUT", "/twitter/tweet/" + i, new HashMap<>(), makeTweet());
  }

  await().atMost(5, SECONDS).until(() -> dataIsAvailable(restClient, numTestDocs));

  LOG.info("Finished setup");
}
 

public RestElasticClient(RestClient client, RestClient proxyClient, boolean enableRunAs) {
    this.client = client;
    this.proxyClient = proxyClient;
    this.mapper = new ObjectMapper();
    this.mapper.setDateFormat(DATE_FORMAT);
    this.enableRunAs = enableRunAs;
}
 

@Override
public void onConstruct(EnhancedInstance objInst, Object[] allArguments) {
    RestClientBuilder restClientBuilder = (RestClientBuilder) (allArguments[0]);
    RestClient restClient = restClientBuilder.build();

    RestClientEnhanceInfo restClientEnhanceInfo = new RestClientEnhanceInfo();
    List<Node> nodeList = restClient.getNodes();
    for (Node node : nodeList) {
        restClientEnhanceInfo.addHttpHost(node.getHost());
    }

    objInst.setSkyWalkingDynamicField(restClientEnhanceInfo);
}
 

public static RestHighLevelClient createESClient(final PropertiesProvider propertiesProvider) {
    System.setProperty("es.set.netty.runtime.available.processors", "false");

    String indexAddress = propertiesProvider.get(INDEX_ADDRESS_PROP).orElse(DEFAULT_ADDRESS);

    RestHighLevelClient client = new RestHighLevelClient(RestClient.builder(create(indexAddress)).setRequestConfigCallback(
            requestConfigBuilder -> requestConfigBuilder
                .setConnectTimeout(5000)
                .setSocketTimeout(60000)).
            setMaxRetryTimeoutMillis(50000)); // listener t/o cf https://github.com/ICIJ/datashare/issues/462
    String clusterName = propertiesProvider.get(CLUSTER_PROP).orElse(ES_CLUSTER_NAME);
    return client;
}
 

private void indexTestData(List<JsonObject> data, String datasetName, int trainTestSplit, RestClient client) throws Exception {
    data.stream().skip(trainTestSplit).forEach(r -> {
        try {
            Request req = new Request("POST", String.format("/%s/_doc/", datasetName));
            req.setJsonEntity(r.toString());
            client.performRequest(req);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    });
    Thread.sleep(1_000);
}