Java Code Examples for org.elasticsearch.cluster.metadata.MetaData#index()

/**
 * {@inheritDoc}
 */
@Override
public void initializeSnapshot(SnapshotId snapshotId, List<String> indices, MetaData metaData) {
    if (readOnly()) {
        throw new RepositoryException(this.repositoryName, "cannot create snapshot in a readonly repository");
    }
    try {
        if (snapshotFormat.exists(snapshotsBlobContainer, snapshotId.getSnapshot()) ||
                snapshotLegacyFormat.exists(snapshotsBlobContainer, snapshotId.getSnapshot())) {
            throw new InvalidSnapshotNameException(snapshotId, "snapshot with such name already exists");
        }
        // Write Global MetaData
        globalMetaDataFormat.write(metaData, snapshotsBlobContainer, snapshotId.getSnapshot());
        for (String index : indices) {
            final IndexMetaData indexMetaData = metaData.index(index);
            final BlobPath indexPath = basePath().add("indices").add(index);
            final BlobContainer indexMetaDataBlobContainer = blobStore().blobContainer(indexPath);
            indexMetaDataFormat.write(indexMetaData, indexMetaDataBlobContainer, snapshotId.getSnapshot());
        }
    } catch (IOException ex) {
        throw new SnapshotCreationException(snapshotId, ex);
    }
}
 

/**
 * Loads the current meta state for each index in the new cluster state and checks if it has to be persisted.
 * Each index state that should be written to disk will be returned. This is only run for data only nodes.
 * It will return only the states for indices that actually have a shard allocated on the current node.
 *
 * @param previouslyWrittenIndices    A list of indices for which the state was already written before
 * @param potentiallyUnwrittenIndices The list of indices for which state should potentially be written
 * @param previousMetaData            The last meta data we know of. meta data for all indices in previouslyWrittenIndices list is persisted now
 * @param newMetaData                 The new metadata
 * @return iterable over all indices states that should be written to disk
 */
public static Iterable<GatewayMetaState.IndexMetaWriteInfo> resolveStatesToBeWritten(ImmutableSet<String> previouslyWrittenIndices, Set<String> potentiallyUnwrittenIndices, MetaData previousMetaData, MetaData newMetaData) {
    List<GatewayMetaState.IndexMetaWriteInfo> indicesToWrite = new ArrayList<>();
    for (String index : potentiallyUnwrittenIndices) {
        IndexMetaData newIndexMetaData = newMetaData.index(index);
        IndexMetaData previousIndexMetaData = previousMetaData == null ? null : previousMetaData.index(index);
        String writeReason = null;
        if (previouslyWrittenIndices.contains(index) == false || previousIndexMetaData == null) {
            writeReason = "freshly created";
        } else if (previousIndexMetaData.getVersion() != newIndexMetaData.getVersion()) {
            writeReason = "version changed from [" + previousIndexMetaData.getVersion() + "] to [" + newIndexMetaData.getVersion() + "]";
        }
        if (writeReason != null) {
            indicesToWrite.add(new GatewayMetaState.IndexMetaWriteInfo(newIndexMetaData, previousIndexMetaData, writeReason));
        }
    }
    return indicesToWrite;
}
 

static void buildShardLevelInfo(ESLogger logger, ShardStats[] stats, HashMap<String, Long> newShardSizes, HashMap<ShardRouting, String> newShardRoutingToDataPath, ClusterState state) {
    MetaData meta = state.getMetaData();
    for (ShardStats s : stats) {
        IndexMetaData indexMeta = meta.index(s.getShardRouting().index());
        Settings indexSettings = indexMeta == null ? null : indexMeta.getSettings();
        newShardRoutingToDataPath.put(s.getShardRouting(), s.getDataPath());
        long size = s.getStats().getStore().sizeInBytes();
        String sid = ClusterInfo.shardIdentifierFromRouting(s.getShardRouting());
        if (logger.isTraceEnabled()) {
            logger.trace("shard: {} size: {}", sid, size);
        }
        if (indexSettings != null && IndexMetaData.isIndexUsingShadowReplicas(indexSettings)) {
            // Shards on a shared filesystem should be considered of size 0
            if (logger.isTraceEnabled()) {
                logger.trace("shard: {} is using shadow replicas and will be treated as size 0", sid);
            }
            size = 0;
        }
        newShardSizes.put(sid, size);
    }
}
 

@Override
protected Tuple<IndexResponse, IndexRequest> shardOperationOnPrimary(MetaData metaData, IndexRequest request) throws Throwable {

    // validate, if routing is required, that we got routing
    IndexMetaData indexMetaData = metaData.index(request.shardId().getIndex());
    MappingMetaData mappingMd = indexMetaData.mappingOrDefault(request.type());
    if (mappingMd != null && mappingMd.routing().required()) {
        if (request.routing() == null) {
            throw new RoutingMissingException(request.shardId().getIndex(), request.type(), request.id());
        }
    }

    IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
    IndexShard indexShard = indexService.shardSafe(request.shardId().id());
    indexShard.checkDiskSpace(fsService);
    final WriteResult<IndexResponse> result = executeIndexRequestOnPrimary(null, request, indexShard, mappingUpdatedAction);
    final IndexResponse response = result.response;
    final Translog.Location location = result.location;
    processAfterWrite(request.refresh(), indexShard, location);
    return new Tuple<>(response, request);
}
 

@Test
public void testPathAndBlobPath() throws Exception {
    Path blobs = createTempDir("blobs");
    execute("create table t1 (x int) clustered into 1 shards with (number_of_replicas = 0)");
    execute("create blob table b1 clustered into 1 shards with (number_of_replicas = 0)");
    execute("create blob table b2 " +
            "clustered into 1 shards with (number_of_replicas = 0, blobs_path = '" + blobs.toString() + "')");
    ensureYellow();

    execute("select path, blob_path from sys.shards where table_name in ('t1', 'b1', 'b2') " +
            "order by table_name asc");
    // b1
    // path + /blobs == blob_path without custom blob path
    assertThat(response.rows()[0][0] + resolveCanonicalString("/blobs"), is(response.rows()[0][1]));

    ClusterService clusterService = internalCluster().getInstance(ClusterService.class);
    MetaData metaData = clusterService.state().getMetaData();
    IndexMetaData index = metaData.index(".blob_b2");
    String indexUUID = index.getIndexUUID();

    // b2
    String b2Path = (String) response.rows()[1][0];
    assertThat(b2Path, containsString(resolveCanonicalString("/nodes/")));
    assertThat(b2Path, endsWith(resolveCanonicalString("/indices/" + indexUUID + "/0")));

    String b2BlobPath = (String) response.rows()[1][1];
    assertThat(b2BlobPath, containsString(resolveCanonicalString("/nodes/")));
    assertThat(b2BlobPath, endsWith(resolveCanonicalString("/indices/" + indexUUID + "/0/blobs")));
    // t1
    assertThat(response.rows()[2][1], nullValue());
}
 

/**
 * Returns the shards to purge, i.e. the local started primary shards that have ttl enabled and disable_purge to false
 */
private List<IndexShard> getShardsToPurge() {
    List<IndexShard> shardsToPurge = new ArrayList<>();
    MetaData metaData = clusterService.state().metaData();
    for (IndexService indexService : indicesService) {
        // check the value of disable_purge for this index
        IndexMetaData indexMetaData = metaData.index(indexService.index().name());
        if (indexMetaData == null) {
            continue;
        }
        boolean disablePurge = indexMetaData.getSettings().getAsBoolean(INDEX_TTL_DISABLE_PURGE, false);
        if (disablePurge) {
            continue;
        }

        // check if ttl is enabled for at least one type of this index
        boolean hasTTLEnabled = false;
        for (String type : indexService.mapperService().types()) {
            DocumentMapper documentType = indexService.mapperService().documentMapper(type);
            if (documentType.TTLFieldMapper().enabled()) {
                hasTTLEnabled = true;
                break;
            }
        }
        if (hasTTLEnabled) {
            for (IndexShard indexShard : indexService) {
                if (indexShard.state() == IndexShardState.STARTED && indexShard.routingEntry().primary() && indexShard.routingEntry().started()) {
                    shardsToPurge.add(indexShard);
                }
            }
        }
    }
    return shardsToPurge;
}
 

/**
 * checks if metaData contains a particular index and
 * invalidates its aliases if so
 */
@VisibleForTesting
void invalidateFromIndex(Index index, MetaData metaData) {
    IndexMetaData indexMetaData = metaData.index(index);
    if (indexMetaData != null) {
        invalidateAliases(indexMetaData.getAliases());
    }
}
 

static Set<IndexMetaData> indexMetaDataSetFromIndexNames(MetaData metaData,
                                                         String[] indices,
                                                         IndexMetaData.State state) {
    Set<IndexMetaData> indicesMetaData = new HashSet<>();
    for (String indexName : indices) {
        IndexMetaData indexMetaData = metaData.index(indexName);
        if (indexMetaData != null && indexMetaData.getState() != state) {
            indicesMetaData.add(indexMetaData);
        }
    }
    return indicesMetaData;
}
 

@Override
public ClusterState execute(ClusterState currentState, SwapAndDropIndexRequest request) throws Exception {
    final MetaData metaData = currentState.getMetaData();
    final ClusterBlocks.Builder blocksBuilder = ClusterBlocks.builder().blocks(currentState.blocks());
    final MetaData.Builder mdBuilder = MetaData.builder(metaData);
    final RoutingTable.Builder routingBuilder = RoutingTable.builder(currentState.routingTable());

    String sourceIndexName = request.source();
    String targetIndexName = request.target();

    mdBuilder.remove(sourceIndexName);
    mdBuilder.remove(targetIndexName);
    routingBuilder.remove(sourceIndexName);
    routingBuilder.remove(targetIndexName);
    blocksBuilder.removeIndexBlocks(sourceIndexName);
    blocksBuilder.removeIndexBlocks(targetIndexName);

    IndexMetaData sourceIndex = metaData.index(sourceIndexName);
    if (sourceIndex == null) {
        throw new IllegalArgumentException("Source index must exist: " + sourceIndexName);
    }

    IndexMetaData newIndexMetaData = IndexMetaData.builder(sourceIndex).index(targetIndexName).build();
    mdBuilder.put(newIndexMetaData, true);
    routingBuilder.addAsFromCloseToOpen(newIndexMetaData);
    blocksBuilder.addBlocks(newIndexMetaData);

    return allocationService.reroute(
        ClusterState.builder(currentState)
            .metaData(mdBuilder)
            .routingTable(routingBuilder.build())
            .blocks(blocksBuilder)
            .build(),
        "swap and drop index"
    );
}
 

private static Settings forTable(MetaData metaData, RelationName relationName) {
    IndexMetaData indexMetaData = metaData.index(relationName.indexNameOrAlias());
    if (indexMetaData == null) {
        throw new IndexNotFoundException(relationName.indexNameOrAlias());
    }
    return indexMetaData.getSettings();
}
 

/**
 * Returns the expected shard size for the given shard or the default value provided if not enough information are available
 * to estimate the shards size.
 */
public static long getExpectedShardSize(ShardRouting shard,
                                        long defaultValue,
                                        ClusterInfo clusterInfo,
                                        MetaData metaData,
                                        RoutingTable routingTable) {
    final IndexMetaData indexMetaData = metaData.getIndexSafe(shard.index());
    if (indexMetaData.getResizeSourceIndex() != null && shard.active() == false &&
        shard.recoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS) {
        // in the shrink index case we sum up the source index shards since we basically make a copy of the shard in
        // the worst case
        long targetShardSize = 0;
        final Index mergeSourceIndex = indexMetaData.getResizeSourceIndex();
        final IndexMetaData sourceIndexMeta = metaData.index(mergeSourceIndex);
        if (sourceIndexMeta != null) {
            final Set<ShardId> shardIds = IndexMetaData.selectRecoverFromShards(shard.id(),
                sourceIndexMeta, indexMetaData.getNumberOfShards());
            for (IndexShardRoutingTable shardRoutingTable : routingTable.index(mergeSourceIndex.getName())) {
                if (shardIds.contains(shardRoutingTable.shardId())) {
                    targetShardSize += clusterInfo.getShardSize(shardRoutingTable.primaryShard(), 0);
                }
            }
        }
        return targetShardSize == 0 ? defaultValue : targetShardSize;
    } else {
        return clusterInfo.getShardSize(shard, defaultValue);
    }
}
 

/**
 * Cleans dangling indices if they are already allocated on the provided meta data.
 */
void cleanupAllocatedDangledIndices(MetaData metaData) {
    for (Index index : danglingIndices.keySet()) {
        final IndexMetaData indexMetaData = metaData.index(index);
        if (indexMetaData != null && indexMetaData.getIndex().getName().equals(index.getName())) {
            if (indexMetaData.getIndex().getUUID().equals(index.getUUID()) == false) {
                LOGGER.warn("[{}] can not be imported as a dangling index, as there is already another index " +
                    "with the same name but a different uuid. local index will be ignored (but not deleted)", index);
            } else {
                LOGGER.debug("[{}] no longer dangling (created), removing from dangling list", index);
            }
            danglingIndices.remove(index);
        }
    }
}
 

private boolean addFailureIfIndexIsUnavailable(DocumentRequest request, BulkRequest bulkRequest, AtomicArray<BulkItemResponse> responses, int idx,
                                          final ConcreteIndices concreteIndices,
                                          final MetaData metaData) {
    String concreteIndex = concreteIndices.getConcreteIndex(request.index());
    Exception unavailableException = null;
    if (concreteIndex == null) {
        try {
            concreteIndex = concreteIndices.resolveIfAbsent(request);
        } catch (IndexClosedException | IndexNotFoundException ex) {
            // Fix for issue where bulk request references an index that
            // cannot be auto-created see issue #8125
            unavailableException = ex;
        }
    }
    if (unavailableException == null) {
        IndexMetaData indexMetaData = metaData.index(concreteIndex);
        if (indexMetaData.getState() == IndexMetaData.State.CLOSE) {
            unavailableException = new IndexClosedException(new Index(metaData.index(request.index()).getIndex()));
        }
    }
    if (unavailableException != null) {
        BulkItemResponse.Failure failure = new BulkItemResponse.Failure(request.index(), request.type(), request.id(),
                unavailableException);
        String operationType = "unknown";
        if (request instanceof IndexRequest) {
            operationType = "index";
        } else if (request instanceof DeleteRequest) {
            operationType = "delete";
        } else if (request instanceof UpdateRequest) {
            operationType = "update";
        }
        BulkItemResponse bulkItemResponse = new BulkItemResponse(idx, operationType, failure);
        responses.set(idx, bulkItemResponse);
        // make sure the request gets never processed again
        bulkRequest.requests.set(idx, null);
        return true;
    }
    return false;
}
 

/**
 * Returns <code>true</code> iff the {@link IndexMetaData} for a given index
 * has changed between the previous cluster state and the new cluster state.
 * Note that this is an object reference equality test, not an equals test.
 */
public boolean indexMetaDataChanged(IndexMetaData current) {
    MetaData previousMetaData = previousState.metaData();
    if (previousMetaData == null) {
        return true;
    }
    IndexMetaData previousIndexMetaData = previousMetaData.index(current.getIndex());
    // no need to check on version, since disco modules will make sure to use the
    // same instance if its a version match
    if (previousIndexMetaData == current) {
        return false;
    }
    return true;
}
 

public static void updateLeftDelayOfUnassignedShards(RoutingAllocation allocation, Settings settings) {
    for (ShardRouting shardRouting : allocation.routingNodes().unassigned()) {
        final MetaData metaData = allocation.metaData();
        final IndexMetaData indexMetaData = metaData.index(shardRouting.index());
        shardRouting.unassignedInfo().updateDelay(allocation.getCurrentNanoTime(), settings, indexMetaData.getSettings());
    }
}
 

public void validate(RoutingTableValidation validation, MetaData metaData) {
    if (!metaData.hasIndex(index())) {
        validation.addIndexFailure(index(), "Exists in routing does not exists in metadata");
        return;
    }
    IndexMetaData indexMetaData = metaData.index(index());
    for (String failure : validate(indexMetaData)) {
        validation.addIndexFailure(index, failure);
    }

}
 

/**
 * Returns list of {@link IndexMetaDataAction} for each relevant index.
 * For each relevant index there are 3 options:
 * <ol>
 * <li>
 * {@link KeepPreviousGeneration} - index metadata is already stored to disk and index metadata version is not changed, no
 * action is required.
 * </li>
 * <li>
 * {@link WriteNewIndexMetaData} - there is no index metadata on disk and index metadata for this index should be written.
 * </li>
 * <li>
 * {@link WriteChangedIndexMetaData} - index metadata is already on disk, but index metadata version has changed. Updated
 * index metadata should be written to disk.
 * </li>
 * </ol>
 *
 * @param previouslyWrittenIndices A list of indices for which the state was already written before
 * @param relevantIndices          The list of indices for which state should potentially be written
 * @param previousMetaData         The last meta data we know of
 * @param newMetaData              The new metadata
 * @return list of {@link IndexMetaDataAction} for each relevant index.
 */
public static List<IndexMetaDataAction> resolveIndexMetaDataActions(Map<Index, Long> previouslyWrittenIndices,
                                                                    Set<Index> relevantIndices,
                                                                    MetaData previousMetaData,
                                                                    MetaData newMetaData) {
    List<IndexMetaDataAction> actions = new ArrayList<>();
    for (Index index : relevantIndices) {
        IndexMetaData newIndexMetaData = newMetaData.getIndexSafe(index);
        IndexMetaData previousIndexMetaData = previousMetaData == null ? null : previousMetaData.index(index);

        if (previouslyWrittenIndices.containsKey(index) == false || previousIndexMetaData == null) {
            actions.add(new WriteNewIndexMetaData(newIndexMetaData));
        } else if (previousIndexMetaData.getVersion() != newIndexMetaData.getVersion()) {
            actions.add(new WriteChangedIndexMetaData(previousIndexMetaData, newIndexMetaData));
        } else {
            actions.add(new KeepPreviousGeneration(index, previouslyWrittenIndices.get(index)));
        }
    }
    return actions;
}