org.elasticsearch.action.admin.indices.stats.ShardStats Java Examples

@Override
protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeRequest) {
    NodeInfo nodeInfo = nodeService.info(false, true, false, true, false, true, false, true);
    NodeStats nodeStats = nodeService.stats(CommonStatsFlags.NONE, true, true, true, false, true, false, false, false, false);
    List<ShardStats> shardsStats = new ArrayList<>();
    for (IndexService indexService : indicesService) {
        for (IndexShard indexShard : indexService) {
            if (indexShard.routingEntry() != null && indexShard.routingEntry().active()) {
                // only report on fully started shards
                shardsStats.add(new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indexShard, SHARD_STATS_FLAGS), indexShard.commitStats()));
            }
        }
    }

    ClusterHealthStatus clusterStatus = null;
    if (clusterService.state().nodes().localNodeMaster()) {
        clusterStatus = new ClusterStateHealth(clusterService.state()).getStatus();
    }

    return new ClusterStatsNodeResponse(nodeInfo.getNode(), clusterStatus, nodeInfo, nodeStats, shardsStats.toArray(new ShardStats[shardsStats.size()]));

}
 

static void buildShardLevelInfo(ESLogger logger, ShardStats[] stats, HashMap<String, Long> newShardSizes, HashMap<ShardRouting, String> newShardRoutingToDataPath, ClusterState state) {
    MetaData meta = state.getMetaData();
    for (ShardStats s : stats) {
        IndexMetaData indexMeta = meta.index(s.getShardRouting().index());
        Settings indexSettings = indexMeta == null ? null : indexMeta.getSettings();
        newShardRoutingToDataPath.put(s.getShardRouting(), s.getDataPath());
        long size = s.getStats().getStore().sizeInBytes();
        String sid = ClusterInfo.shardIdentifierFromRouting(s.getShardRouting());
        if (logger.isTraceEnabled()) {
            logger.trace("shard: {} size: {}", sid, size);
        }
        if (indexSettings != null && IndexMetaData.isIndexUsingShadowReplicas(indexSettings)) {
            // Shards on a shared filesystem should be considered of size 0
            if (logger.isTraceEnabled()) {
                logger.trace("shard: {} is using shadow replicas and will be treated as size 0", sid);
            }
            size = 0;
        }
        newShardSizes.put(sid, size);
    }
}
 

@Test
public void testCheckIndexRetention_Overlapping() throws Exception {
    String serializedIndexMetadata = "[{\"preCreate\": false, \"retentionType\": \"hourly\", \"retentionPeriod\": 2, \"indexName\": \"nf_errors_log\"}," +
            "{\"preCreate\": false, \"retentionType\": \"yearly\", \"retentionPeriod\": 3, \"indexName\": \"nf_errors_log201712\"}]";
    List<IndexMetadata> indexMetadataList = IndexUtils.parseIndexMetadata(serializedIndexMetadata);

    Map<String, IndexStats> indexStats = new HashMap<>();
    indexStats.put("nf_errors_log2017121110", new IndexStats("nf_errors_log2017121110", new ShardStats[]{}));
    indexStats.put("nf_errors_log2017121111", new IndexStats("nf_errors_log2017121111", new ShardStats[]{}));
    indexStats.put("nf_errors_log2017121112", new IndexStats("nf_errors_log2017121112", new ShardStats[]{}));
    indexStats.put("nf_errors_log2017121113", new IndexStats("nf_errors_log2017121113", new ShardStats[]{}));
    indexStats.put("nf_errors_log2017121114", new IndexStats("nf_errors_log2017121114", new ShardStats[]{}));

    IndicesStatsResponse indicesStatsResponse = mock(IndicesStatsResponse.class);
    when(indicesStatsResponse.getIndices()).thenReturn(indexStats);

    doReturn(indicesStatsResponse).when(elasticsearchIndexManager).getIndicesStatsResponse(elasticsearchClient);

    elasticsearchIndexManager.runIndexManagement(elasticsearchClient, indexMetadataList, new DateTime("2017-12-11T13:30Z"));

    verify(elasticsearchIndexManager, times(2)).checkIndexRetention(any(Client.class), anySet(), any(IndexMetadata.class), any(DateTime.class));
}
 

@Override
public void writeTo(StreamOutput out) throws IOException {
    super.writeTo(out);
    if (clusterStatus == null) {
        out.writeBoolean(false);
    } else {
        out.writeBoolean(true);
        out.writeByte(clusterStatus.value());
    }
    nodeInfo.writeTo(out);
    nodeStats.writeTo(out);
    out.writeVInt(shardsStats.length);
    for (ShardStats ss : shardsStats) {
        ss.writeTo(out);
    }
}
 

private Map<Index, CommonStats> createStatsByIndex() {
    Map<Index, CommonStats> statsMap = Maps.newHashMap();
    for (Map.Entry<Index, List<IndexShardStats>> entry : statsByShard.entrySet()) {
        if (!statsMap.containsKey(entry.getKey())) {
            statsMap.put(entry.getKey(), new CommonStats());
        }

        for (IndexShardStats indexShardStats : entry.getValue()) {
            for (ShardStats shardStats : indexShardStats.getShards()) {
                statsMap.get(entry.getKey()).add(shardStats.getStats());
            }
        }
    }

    return statsMap;
}
 

@Test
public void testRunIndexManagement_NotActionable_NoRetentionPeriod() throws Exception {
    String serializedIndexMetadata = "[{\"retentionType\": \"yearly\", \"indexName\": \"nf_errors_log\"}]";
    when(config.getIndexMetadata()).thenReturn(serializedIndexMetadata);

    Map<String, IndexStats> indexStats = new HashMap<>();
    indexStats.put("nf_errors_log2018", new IndexStats("nf_errors_log2018", new ShardStats[]{}));

    IndicesStatsResponse indicesStatsResponse = mock(IndicesStatsResponse.class);
    when(indicesStatsResponse.getIndices()).thenReturn(indexStats);

    doReturn(indicesStatsResponse).when(elasticsearchIndexManager).getIndicesStatsResponse(elasticsearchClient);

    elasticsearchIndexManager.runIndexManagement();

    verify(elasticsearchIndexManager, times(0)).checkIndexRetention(any(Client.class), anySet(), any(IndexMetadata.class), any(DateTime.class));
    verify(elasticsearchIndexManager, times(0)).preCreateIndex(any(Client.class), any(IndexMetadata.class), any(DateTime.class));
}
 

@Test
public void testRunIndexManagement_NotActionable_NoIndex() throws Exception {
    String serializedIndexMetadata = "[{\"retentionType\": \"yearly\", \"retentionPeriod\": 20}]";
    when(config.getIndexMetadata()).thenReturn(serializedIndexMetadata);

    Map<String, IndexStats> indexStats = new HashMap<>();
    indexStats.put("nf_errors_log2018", new IndexStats("nf_errors_log2018", new ShardStats[]{}));

    IndicesStatsResponse indicesStatsResponse = mock(IndicesStatsResponse.class);
    when(indicesStatsResponse.getIndices()).thenReturn(indexStats);

    doReturn(indicesStatsResponse).when(elasticsearchIndexManager).getIndicesStatsResponse(elasticsearchClient);

    elasticsearchIndexManager.runIndexManagement();

    verify(elasticsearchIndexManager, times(0)).checkIndexRetention(any(Client.class), anySet(), any(IndexMetadata.class), any(DateTime.class));
    verify(elasticsearchIndexManager, times(0)).preCreateIndex(any(Client.class), any(IndexMetadata.class), any(DateTime.class));
}
 

@Override
protected ShardStats shardOperation(IndexShardStatsRequest request, ShardRouting shardRouting) {
    IndexService indexService = indicesService.indexServiceSafe(shardRouting.shardId().getIndex());
    IndexShard indexShard = indexService.shardSafe(shardRouting.shardId().id());
    // if we don't have the routing entry yet, we need it stats wise, we treat it as if the shard is not ready yet
    if (indexShard.routingEntry() == null) {
        throw new ShardNotFoundException(indexShard.shardId());
    }

    if (!indexShard.state().equals(IndexShardState.STARTED)) {
        throw new ElasticsearchException(indexShard.shardId().toString() + " state is " + indexShard.state() + ", not started");
    }
    
    CommonStatsFlags flags = new CommonStatsFlags().clear();

    if (request.dl()) {
        flags.set(CommonStatsFlags.Flag.DL);
    }

    return new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indexShard, flags), indexShard.commitStats());
}
 

@Override
public void readFrom(StreamInput in) throws IOException {
    super.readFrom(in);
    clusterStatus = null;
    if (in.readBoolean()) {
        clusterStatus = ClusterHealthStatus.fromValue(in.readByte());
    }
    this.nodeInfo = NodeInfo.readNodeInfo(in);
    this.nodeStats = NodeStats.readNodeStats(in);
    int size = in.readVInt();
    shardsStats = new ShardStats[size];
    for (size--; size >= 0; size--) {
        shardsStats[size] = ShardStats.readShardStats(in);
    }
}
 

public ClusterStatsNodeResponse(DiscoveryNode node, @Nullable ClusterHealthStatus clusterStatus, NodeInfo nodeInfo, NodeStats nodeStats, ShardStats[] shardsStats) {
    super(node);
    this.nodeInfo = nodeInfo;
    this.nodeStats = nodeStats;
    this.shardsStats = shardsStats;
    this.clusterStatus = clusterStatus;
}
 

public NodeIndicesStats(CommonStats oldStats, Map<Index, List<IndexShardStats>> statsByShard) {
    //this.stats = stats;
    this.statsByShard = statsByShard;

    // make a total common stats from old ones and current ones
    this.stats = oldStats;
    for (List<IndexShardStats> shardStatsList : statsByShard.values()) {
        for (IndexShardStats indexShardStats : shardStatsList) {
            for (ShardStats shardStats : indexShardStats.getShards()) {
                stats.add(shardStats.getStats());
            }
        }
    }
}
 

@Test
public void testRunIndexManagement() throws Exception {
    String serializedIndexMetadata = "[{\"retentionType\": \"yearly\", \"retentionPeriod\": 3, \"indexName\": \"nf_errors_log\"}]";
    when(config.getIndexMetadata()).thenReturn(serializedIndexMetadata);

    Map<String, IndexStats> indexStats = new HashMap<>();
    indexStats.put("nf_errors_log2018", new IndexStats("nf_errors_log2018", new ShardStats[]{}));
    indexStats.put("nf_errors_log2017", new IndexStats("nf_errors_log2017", new ShardStats[]{}));
    indexStats.put("nf_errors_log2016", new IndexStats("nf_errors_log2016", new ShardStats[]{}));
    indexStats.put("nf_errors_log2015", new IndexStats("nf_errors_log2015", new ShardStats[]{}));
    indexStats.put("nf_errors_log2014", new IndexStats("nf_errors_log2014", new ShardStats[]{}));
    indexStats.put("nf_errors_log2013", new IndexStats("nf_errors_log2013", new ShardStats[]{}));
    indexStats.put("nf_errors_log2012", new IndexStats("nf_errors_log2012", new ShardStats[]{}));

    IndicesStatsResponse indicesStatsResponse = mock(IndicesStatsResponse.class);
    when(indicesStatsResponse.getIndices()).thenReturn(indexStats);

    doReturn(indicesStatsResponse).when(elasticsearchIndexManager).getIndicesStatsResponse(elasticsearchClient);

    elasticsearchIndexManager.runIndexManagement();

    verify(elasticsearchIndexManager, times(1)).checkIndexRetention(any(Client.class), anySet(), any(IndexMetadata.class), any(DateTime.class));

    verify(elasticsearchIndexManager, times(1)).deleteIndices(any(Client.class), eq("nf_errors_log2012"), eq(AUTO_CREATE_INDEX_TIMEOUT));
    verify(elasticsearchIndexManager, times(1)).deleteIndices(any(Client.class), eq("nf_errors_log2013"), eq(AUTO_CREATE_INDEX_TIMEOUT));

    verify(elasticsearchIndexManager, times(0)).preCreateIndex(any(Client.class), any(IndexMetadata.class), any(DateTime.class));
}
 

@Override
public void writeTo(StreamOutput out) throws IOException {
    super.writeTo(out);
    out.writeVInt(shards.length);
    for (ShardStats shard : shards) {
        shard.writeTo(out);
    }
}
 

@Override
public void readFrom(StreamInput in) throws IOException {
    super.readFrom(in);
    shards = new ShardStats[in.readVInt()];
    for (int i = 0; i < shards.length; i++) {
        shards[i] = ShardStats.readShardStats(in);
    }
}
 

static void buildShardLevelInfo(Logger logger, ShardStats[] stats, ImmutableOpenMap.Builder<String, Long> newShardSizes,
                                ImmutableOpenMap.Builder<ShardRouting, String> newShardRoutingToDataPath, ClusterState state) {
    for (ShardStats s : stats) {
        newShardRoutingToDataPath.put(s.getShardRouting(), s.getDataPath());
        long size = s.getStats().getStore().sizeInBytes();
        String sid = ClusterInfo.shardIdentifierFromRouting(s.getShardRouting());
        if (logger.isTraceEnabled()) {
            logger.trace("shard: {} size: {}", sid, size);
        }
        newShardSizes.put(sid, size);
    }
}
 

public ImmutableMap<ShardRouting, ShardStats> asMap() {
    if (shardStatsMap == null) {
        ImmutableMap.Builder<ShardRouting, ShardStats> mb = ImmutableMap.builder();
        for (ShardStats ss : shards) {
            mb.put(ss.getShardRouting(), ss);
        }

        shardStatsMap = mb.build();
    }
    return shardStatsMap;
}
 

@Test
public void testMappingNewFieldsTimeoutDoesntAffectCheckpoints() throws Exception {
    InternalTestCluster internalCluster = internalCluster();
    internalCluster.startNodes(3,
                               Settings.builder()
                                   .put(MappingUpdatedAction.INDICES_MAPPING_DYNAMIC_TIMEOUT_SETTING.getKey(),
                                        "1ms")
                                   .build()
    );
    ensureStableCluster(3);

    logger.info("creating table t with 1 shards and 1 replica");
    execute("create table t (id int primary key, x object(dynamic)) clustered into 1 shards with " +
            "(number_of_replicas = 1, \"routing.allocation.exclude._name\" = '" + internalCluster().getMasterName()
            + "', \"write.wait_for_active_shards\" = 1)");
    ensureGreen();
    execute("insert into t values (?, ?)", new Object[]{1, Map.of("first field", "first value")});

    ServiceDisruptionScheme disruption = new BlockMasterServiceOnMaster(random());
    setDisruptionScheme(disruption);

    disruption.startDisrupting();

    try {
        execute("insert into t values (?, ?), (?, ?), (?, ?)",
                new Object[]{
                    2, Map.of("2nd field", "2nd value"),
                    3, Map.of("3rd field", "3rd value"),
                    4, Map.of("4th field", "4th value"),
                });
    } catch (Exception e) {
        // failure is acceptable
    }

    disruption.stopDisrupting();

    String indexName = toIndexName(sqlExecutor.getCurrentSchema(), "t", null);
    assertBusy(() -> {
        IndicesStatsResponse stats = client().admin().indices().prepareStats(indexName).clear().get();
        for (ShardStats shardStats : stats.getShards()) {
            assertThat(shardStats.getShardRouting().toString(),
                       shardStats.getSeqNoStats().getGlobalCheckpoint(), equalTo(shardStats.getSeqNoStats().getLocalCheckpoint()));
        }
    });
}
 

public NodeIndicesStats stats(boolean includePrevious, CommonStatsFlags flags) {
    CommonStats oldStats = new CommonStats(flags);

    if (includePrevious) {
        Flag[] setFlags = flags.getFlags();
        for (Flag flag : setFlags) {
            switch (flag) {
                case Get:
                    oldStats.get.add(oldShardsStats.getStats);
                    break;
                case Indexing:
                    oldStats.indexing.add(oldShardsStats.indexingStats);
                    break;
                case Search:
                    oldStats.search.add(oldShardsStats.searchStats);
                    break;
                case Merge:
                    oldStats.merge.add(oldShardsStats.mergeStats);
                    break;
                case Refresh:
                    oldStats.refresh.add(oldShardsStats.refreshStats);
                    break;
                case Recovery:
                    oldStats.recoveryStats.add(oldShardsStats.recoveryStats);
                    break;
                case Flush:
                    oldStats.flush.add(oldShardsStats.flushStats);
                    break;
            }
        }
    }

    Map<Index, List<IndexShardStats>> statsByShard = Maps.newHashMap();
    for (IndexServiceInjectorPair value : indices.values()) {
        IndexService indexService = value.getIndexService();
        for (IndexShard indexShard : indexService) {
            try {
                if (indexShard.routingEntry() == null) {
                    continue;
                }
                IndexShardStats indexShardStats = new IndexShardStats(indexShard.shardId(), new ShardStats[] { new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indexShard, flags), indexShard.commitStats()) });
                if (!statsByShard.containsKey(indexService.index())) {
                    statsByShard.put(indexService.index(), arrayAsArrayList(indexShardStats));
                } else {
                    statsByShard.get(indexService.index()).add(indexShardStats);
                }
            } catch (IllegalIndexShardStateException e) {
                // we can safely ignore illegal state on ones that are closing for example
                logger.trace("{} ignoring shard stats", e, indexShard.shardId());
            }
        }
    }
    return new NodeIndicesStats(oldStats, statsByShard);
}
 

public ShardStats[] shardsStats() {
    return this.shardsStats;
}
 

@Override
public void run() {
    try {
        if (isPrimaryInRouting) {
            ActionFuture<IndexShardStatsResponse>  result = indexShardStatsAction.execute(new IndexShardStatsRequest(shardId.index().getName(), shardId.id()));
            IndexShardStatsResponse response = result.actionGet(10000);
            if (response.getTotalShards() == response.getSuccessfulShards() && response.getTotalShards() > 0) {
                DLSN miniumDlsn = new DLSN(Long.MAX_VALUE, 0, 0);
                boolean hasInvalidShardStats = false;
                for (ShardStats shardStats : response.getShards()) {
                    boolean isInvalid = true;
                    CommonStats commonStats = shardStats.getStats();
                    if (commonStats != null) {
                        DLStats dlStats = commonStats.getDLStats();
                        if (dlStats != null && !dlStats.getCommitDlsn().equals(DLSN.InvalidDLSN)) {
                            isInvalid = false;
                            if (dlStats.getCommitDlsn().compareTo(miniumDlsn) < 0) {
                                miniumDlsn = dlStats.getCommitDlsn();
                            }
                        } else {
                            logger.debug("dl [{}] status is invalid", dlStats);
                        }
                    } else {
                        logger.debug("common stats is null");
                    }
                    if (isInvalid) {
                        hasInvalidShardStats = true;
                        break;
                    }
                }
                if (!hasInvalidShardStats) {
                    if (!miniumDlsn.equals(DLSN.InitialDLSN)) {
                        logger.info("try to truncate log before [{}]", miniumDlsn);
                        dlTranslog.truncateLogBeforeDLSN(miniumDlsn);
                    }
                }
            } else {
                logger.warn("some shards failed to get stats: total shards [{}], shards give reponse [{}], failure exceptions [{}], maybe some shard is abnormal so skip truncate log", 
                        response.getTotalShards(), 
                        response.getSuccessfulShards(),
                        response.getShardFailures().length > 0 ? response.getShardFailures()[0] : "");
            }
        }
    } catch (Throwable t) {
        logger.error("catch exception when sleep to next lease check time", t);
    } finally {
        if (!isClosed.get()) {
            // check in 1 minutes
            threadPool.schedule(TimeValue.timeValueMinutes(1), ThreadPool.Names.GENERIC, this);
        }
    }
}
 

@Override
protected IndexShardStatsResponse newResponse(IndexShardStatsRequest request, int totalShards, int successfulShards, int failedShards, List<ShardStats> responses, List<ShardOperationFailedException> shardFailures, ClusterState clusterState) {
    return new IndexShardStatsResponse(responses.toArray(new ShardStats[responses.size()]), totalShards, successfulShards, failedShards, shardFailures);
}
 

@Override
protected ShardStats readShardResult(StreamInput in) throws IOException {
    return ShardStats.readShardStats(in);
}
 

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
    String level = params.param("level", "indices");
    boolean isLevelValid = "indices".equalsIgnoreCase(level) || "shards".equalsIgnoreCase(level) || "cluster".equalsIgnoreCase(level);
    if (!isLevelValid) {
        return builder;
    }

    if ("indices".equalsIgnoreCase(level) || "shards".equalsIgnoreCase(level)) {
        builder.startObject(Fields.INDICES);
        for (IndexStats indexStats : getIndices().values()) {
            builder.startObject(indexStats.getIndex(), XContentBuilder.FieldCaseConversion.NONE);

            builder.startObject("primaries");
            indexStats.getPrimaries().toXContent(builder, params);
            builder.endObject();

            builder.startObject("total");
            indexStats.getTotal().toXContent(builder, params);
            builder.endObject();

            if ("shards".equalsIgnoreCase(level)) {
                builder.startObject(Fields.SHARDS);
                for (IndexShardStats indexShardStats : indexStats) {
                    builder.startArray(Integer.toString(indexShardStats.getShardId().id()));
                    for (ShardStats shardStats : indexShardStats) {
                        builder.startObject();
                        shardStats.toXContent(builder, params);
                        builder.endObject();
                    }
                    builder.endArray();
                }
                builder.endObject();
            }
            builder.endObject();
        }
        builder.endObject();
    }

    return builder;
}
 

public ShardStats getAt(int position) {
    return shards[position];
}
 

public ShardStats[] getShards() {
    return this.shards;
}
 

IndexShardStatsResponse(ShardStats[] shards, int totalShards, int successfulShards, int failedShards, List<ShardOperationFailedException> shardFailures) {
    super(totalShards, successfulShards, failedShards, shardFailures);
    this.shards = shards;
}