org.elasticsearch.action.admin.indices.stats.IndexStats Java Examples

The following examples show how to use org.elasticsearch.action.admin.indices.stats.IndexStats. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: PrometheusMetricsCollector.java    From elasticsearch-prometheus-exporter with Apache License 2.0 6 votes vote down vote up 
private void updatePerIndexMetrics(ClusterHealthResponse chr, IndicesStatsResponse isr) {

        if (chr != null && isr != null) {
            for (Map.Entry<String, IndexStats> entry : isr.getIndices().entrySet()) {
                String indexName = entry.getKey();
                ClusterIndexHealth cih = chr.getIndices().get(indexName);
                catalog.setClusterGauge("index_status", cih.getStatus().value(), indexName);
                catalog.setClusterGauge("index_replicas_number", cih.getNumberOfReplicas(), indexName);
                catalog.setClusterGauge("index_shards_number", cih.getActiveShards(), "active", indexName);
                catalog.setClusterGauge("index_shards_number", cih.getNumberOfShards(), "shards", indexName);
                catalog.setClusterGauge("index_shards_number", cih.getActivePrimaryShards(), "active_primary", indexName);
                catalog.setClusterGauge("index_shards_number", cih.getInitializingShards(), "initializing", indexName);
                catalog.setClusterGauge("index_shards_number", cih.getRelocatingShards(), "relocating", indexName);
                catalog.setClusterGauge("index_shards_number", cih.getUnassignedShards(), "unassigned", indexName);
                IndexStats indexStats = entry.getValue();
                updatePerIndexContextMetrics(indexName, "total", indexStats.getTotal());
                updatePerIndexContextMetrics(indexName, "primaries", indexStats.getPrimaries());
            }
        }
    }
Example #2
Source File: ElasticsearchIndexManager.java    From Raigad with Apache License 2.0 6 votes vote down vote up 
void preCreateIndex(Client client, IndexMetadata indexMetadata, DateTime dateTime) throws UnsupportedAutoIndexException {
    logger.info("Pre-creating indices for {}*", indexMetadata.getIndexNamePattern());

    IndicesStatsResponse indicesStatsResponse = getIndicesStatsResponse(client);
    Map<String, IndexStats> indexStatsMap = indicesStatsResponse.getIndices();

    if (indexStatsMap == null || indexStatsMap.isEmpty()) {
        logger.info("No existing indices, no need to pre-create");
        return;
    }

    indexStatsMap.keySet().stream()
        .filter(indexName -> indexMetadata.getIndexNameFilter().filter(indexName))
        .findFirst()
        .ifPresent(indexName -> {
            try {
                createIndex(client, indexMetadata.getIndexNameToPreCreate(dateTime));
            } catch (UnsupportedAutoIndexException e) {
                logger.error("Invalid index metadata: " + indexMetadata.toString(), e);
            }
        });
}
Example #3
Source File: TestElasticsearchIndexManager.java    From Raigad with Apache License 2.0 6 votes vote down vote up 
@Test
public void testRunIndexManagement_NotActionable_NoIndex() throws Exception {
    String serializedIndexMetadata = "[{\"retentionType\": \"yearly\", \"retentionPeriod\": 20}]";
    when(config.getIndexMetadata()).thenReturn(serializedIndexMetadata);

    Map<String, IndexStats> indexStats = new HashMap<>();
    indexStats.put("nf_errors_log2018", new IndexStats("nf_errors_log2018", new ShardStats[]{}));

    IndicesStatsResponse indicesStatsResponse = mock(IndicesStatsResponse.class);
    when(indicesStatsResponse.getIndices()).thenReturn(indexStats);

    doReturn(indicesStatsResponse).when(elasticsearchIndexManager).getIndicesStatsResponse(elasticsearchClient);

    elasticsearchIndexManager.runIndexManagement();

    verify(elasticsearchIndexManager, times(0)).checkIndexRetention(any(Client.class), anySet(), any(IndexMetadata.class), any(DateTime.class));
    verify(elasticsearchIndexManager, times(0)).preCreateIndex(any(Client.class), any(IndexMetadata.class), any(DateTime.class));
}
Example #4
Source File: TestElasticsearchIndexManager.java    From Raigad with Apache License 2.0 6 votes vote down vote up 
@Test
public void testRunIndexManagement_NotActionable_NoRetentionPeriod() throws Exception {
    String serializedIndexMetadata = "[{\"retentionType\": \"yearly\", \"indexName\": \"nf_errors_log\"}]";
    when(config.getIndexMetadata()).thenReturn(serializedIndexMetadata);

    Map<String, IndexStats> indexStats = new HashMap<>();
    indexStats.put("nf_errors_log2018", new IndexStats("nf_errors_log2018", new ShardStats[]{}));

    IndicesStatsResponse indicesStatsResponse = mock(IndicesStatsResponse.class);
    when(indicesStatsResponse.getIndices()).thenReturn(indexStats);

    doReturn(indicesStatsResponse).when(elasticsearchIndexManager).getIndicesStatsResponse(elasticsearchClient);

    elasticsearchIndexManager.runIndexManagement();

    verify(elasticsearchIndexManager, times(0)).checkIndexRetention(any(Client.class), anySet(), any(IndexMetadata.class), any(DateTime.class));
    verify(elasticsearchIndexManager, times(0)).preCreateIndex(any(Client.class), any(IndexMetadata.class), any(DateTime.class));
}
Example #5
Source File: TestElasticsearchIndexManager.java    From Raigad with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCheckIndexRetention_Overlapping() throws Exception {
    String serializedIndexMetadata = "[{\"preCreate\": false, \"retentionType\": \"hourly\", \"retentionPeriod\": 2, \"indexName\": \"nf_errors_log\"}," +
            "{\"preCreate\": false, \"retentionType\": \"yearly\", \"retentionPeriod\": 3, \"indexName\": \"nf_errors_log201712\"}]";
    List<IndexMetadata> indexMetadataList = IndexUtils.parseIndexMetadata(serializedIndexMetadata);

    Map<String, IndexStats> indexStats = new HashMap<>();
    indexStats.put("nf_errors_log2017121110", new IndexStats("nf_errors_log2017121110", new ShardStats[]{}));
    indexStats.put("nf_errors_log2017121111", new IndexStats("nf_errors_log2017121111", new ShardStats[]{}));
    indexStats.put("nf_errors_log2017121112", new IndexStats("nf_errors_log2017121112", new ShardStats[]{}));
    indexStats.put("nf_errors_log2017121113", new IndexStats("nf_errors_log2017121113", new ShardStats[]{}));
    indexStats.put("nf_errors_log2017121114", new IndexStats("nf_errors_log2017121114", new ShardStats[]{}));

    IndicesStatsResponse indicesStatsResponse = mock(IndicesStatsResponse.class);
    when(indicesStatsResponse.getIndices()).thenReturn(indexStats);

    doReturn(indicesStatsResponse).when(elasticsearchIndexManager).getIndicesStatsResponse(elasticsearchClient);

    elasticsearchIndexManager.runIndexManagement(elasticsearchClient, indexMetadataList, new DateTime("2017-12-11T13:30Z"));

    verify(elasticsearchIndexManager, times(2)).checkIndexRetention(any(Client.class), anySet(), any(IndexMetadata.class), any(DateTime.class));
}
Example #6
Source File: ElasticSearchHealthCheckJob.java    From soundwave with Apache License 2.0 5 votes vote down vote up 
private void logIndexStats(Map<String, IndexStats> indexStatsMap) {
  for (IndexStats stat : indexStatsMap.values()) {
    String indexKey = "esindex_" + stat.getIndex();
    Stats.setGauge(StatsUtil.getStatsName(indexKey, "primaryDocCount"),
        stat.getPrimaries().getDocs().getCount());
    Stats.setGauge(StatsUtil.getStatsName(indexKey, "totalDocCount"),
        stat.getTotal().getDocs().getCount());
    Stats.setGauge(StatsUtil.getStatsName(indexKey, "shardsCount"),
        stat.getShards().length);
    Stats.setGauge(StatsUtil.getStatsName(indexKey, "primaryStoreSizeMB"),
        stat.getPrimaries().getStore().getSize().getMbFrac());
    Stats.setGauge(StatsUtil.getStatsName(indexKey, "totalStoreSizeMB"),
        stat.getTotal().getStore().getSize().getMbFrac());
  }
}
Example #7
Source File: ElasticsearchIndexManager.java    From Raigad with Apache License 2.0 5 votes vote down vote up 
void runIndexManagement(Client esTransportClient, List<IndexMetadata> indexMetadataList, DateTime dateTime) {
    // Find all the indices
    IndicesStatsResponse indicesStatsResponse = getIndicesStatsResponse(esTransportClient);
    Map<String, IndexStats> indexStatsMap = indicesStatsResponse.getIndices();

    if (indexStatsMap == null || indexStatsMap.isEmpty()) {
        logger.info("Cluster is empty, no indices found");
        return;
    }

    for (IndexMetadata indexMetadata : indexMetadataList) {
        if (!indexMetadata.isActionable()) {
            logger.warn(String.format("Index metadata %s is not actionable, skipping", indexMetadata));
            continue;
        }

        try {
            checkIndexRetention(esTransportClient, indexStatsMap.keySet(), indexMetadata, dateTime);

            if (indexMetadata.isPreCreate()) {
                preCreateIndex(esTransportClient, indexMetadata, dateTime);
            }
        } catch (Exception e) {
            logger.error("Caught an exception while building index metadata information from configuration property", e);
            return;
        }
    }
}
Example #8
Source File: TestElasticsearchIndexManager.java    From Raigad with Apache License 2.0 5 votes vote down vote up 
@Test
public void testRunIndexManagement() throws Exception {
    String serializedIndexMetadata = "[{\"retentionType\": \"yearly\", \"retentionPeriod\": 3, \"indexName\": \"nf_errors_log\"}]";
    when(config.getIndexMetadata()).thenReturn(serializedIndexMetadata);

    Map<String, IndexStats> indexStats = new HashMap<>();
    indexStats.put("nf_errors_log2018", new IndexStats("nf_errors_log2018", new ShardStats[]{}));
    indexStats.put("nf_errors_log2017", new IndexStats("nf_errors_log2017", new ShardStats[]{}));
    indexStats.put("nf_errors_log2016", new IndexStats("nf_errors_log2016", new ShardStats[]{}));
    indexStats.put("nf_errors_log2015", new IndexStats("nf_errors_log2015", new ShardStats[]{}));
    indexStats.put("nf_errors_log2014", new IndexStats("nf_errors_log2014", new ShardStats[]{}));
    indexStats.put("nf_errors_log2013", new IndexStats("nf_errors_log2013", new ShardStats[]{}));
    indexStats.put("nf_errors_log2012", new IndexStats("nf_errors_log2012", new ShardStats[]{}));

    IndicesStatsResponse indicesStatsResponse = mock(IndicesStatsResponse.class);
    when(indicesStatsResponse.getIndices()).thenReturn(indexStats);

    doReturn(indicesStatsResponse).when(elasticsearchIndexManager).getIndicesStatsResponse(elasticsearchClient);

    elasticsearchIndexManager.runIndexManagement();

    verify(elasticsearchIndexManager, times(1)).checkIndexRetention(any(Client.class), anySet(), any(IndexMetadata.class), any(DateTime.class));

    verify(elasticsearchIndexManager, times(1)).deleteIndices(any(Client.class), eq("nf_errors_log2012"), eq(AUTO_CREATE_INDEX_TIMEOUT));
    verify(elasticsearchIndexManager, times(1)).deleteIndices(any(Client.class), eq("nf_errors_log2013"), eq(AUTO_CREATE_INDEX_TIMEOUT));

    verify(elasticsearchIndexManager, times(0)).preCreateIndex(any(Client.class), any(IndexMetadata.class), any(DateTime.class));
}
Example #9
Source File: EsInstanceStore.java    From soundwave with Apache License 2.0 4 votes vote down vote up 
public Map<String, IndexStats> getIndexStats() throws Exception {
  IndicesStatsResponse resp = esClient.admin().indices().prepareStats().all().get();
  return resp.getIndices();
}
Example #10
Source File: IndexShardStatsResponse.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up 
public IndexStats getIndex(String index) {
    return getIndices().get(index);
}
Example #11
Source File: IndexShardStatsResponse.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up 
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
    String level = params.param("level", "indices");
    boolean isLevelValid = "indices".equalsIgnoreCase(level) || "shards".equalsIgnoreCase(level) || "cluster".equalsIgnoreCase(level);
    if (!isLevelValid) {
        return builder;
    }

    if ("indices".equalsIgnoreCase(level) || "shards".equalsIgnoreCase(level)) {
        builder.startObject(Fields.INDICES);
        for (IndexStats indexStats : getIndices().values()) {
            builder.startObject(indexStats.getIndex(), XContentBuilder.FieldCaseConversion.NONE);

            builder.startObject("primaries");
            indexStats.getPrimaries().toXContent(builder, params);
            builder.endObject();

            builder.startObject("total");
            indexStats.getTotal().toXContent(builder, params);
            builder.endObject();

            if ("shards".equalsIgnoreCase(level)) {
                builder.startObject(Fields.SHARDS);
                for (IndexShardStats indexShardStats : indexStats) {
                    builder.startArray(Integer.toString(indexShardStats.getShardId().id()));
                    for (ShardStats shardStats : indexShardStats) {
                        builder.startObject();
                        shardStats.toXContent(builder, params);
                        builder.endObject();
                    }
                    builder.endArray();
                }
                builder.endObject();
            }
            builder.endObject();
        }
        builder.endObject();
    }

    return builder;
}