Java Code Examples for org.elasticsearch.cluster.metadata.IndexMetaData#Builder

private void addIndexMetaDataToClusterState(IndexMetaData.Builder imdBuilder) throws Exception {
    CompletableFuture<Boolean> processed = new CompletableFuture<>();
    ClusterState currentState = clusterService.state();
    ClusterState newState = ClusterState.builder(currentState)
        .metaData(MetaData.builder(currentState.metaData())
            .put(imdBuilder))
        .build();
    clusterService.addListener(event -> {
        if (event.state() == newState) {
            processed.complete(true);
        }
    });
    clusterService.getClusterApplierService()
        .onNewClusterState("test", () -> newState, (source, e) -> processed.completeExceptionally(e));
    processed.get(5, TimeUnit.SECONDS);
}
 

@Test
public void testPartitionWithMeta() throws Exception {
    PartitionName partitionName = new PartitionName(
        new RelationName("doc", "test1"), ImmutableList.of("foo"));
    IndexMetaData.Builder indexMetaData = IndexMetaData
        .builder(partitionName.asIndexName())
        .settings(defaultSettings())
        .putMapping(Constants.DEFAULT_MAPPING_TYPE, "{\"_meta\":{\"partitioned_by\":[[\"col\", \"string\"]]}}")
        .numberOfShards(10)
        .numberOfReplicas(4);
    addIndexMetaDataToClusterState(indexMetaData);
    Iterable<PartitionInfo> partitioninfos = new PartitionInfos(clusterService);
    Iterator<PartitionInfo> iter = partitioninfos.iterator();
    PartitionInfo partitioninfo = iter.next();
    assertThat(partitioninfo.name().asIndexName(), is(partitionName.asIndexName()));
    assertThat(partitioninfo.numberOfShards(), is(10));
    assertThat(partitioninfo.numberOfReplicas(), is("4"));
    assertThat(partitioninfo.values(), hasEntry("col", "foo"));
    assertThat(iter.hasNext(), is(false));
}
 

@Test
public void testPartitionWithMetaMultiCol() throws Exception {
    PartitionName partitionName = new PartitionName(
        new RelationName("doc", "test1"), ImmutableList.of("foo", "1"));
    IndexMetaData.Builder indexMetaData = IndexMetaData
        .builder(partitionName.asIndexName())
        .settings(defaultSettings())
        .putMapping(Constants.DEFAULT_MAPPING_TYPE, "{\"_meta\":{\"partitioned_by\":[[\"col\", \"string\"], [\"col2\", \"integer\"]]}}")
        .numberOfShards(10)
        .numberOfReplicas(4);
    addIndexMetaDataToClusterState(indexMetaData);
    Iterable<PartitionInfo> partitioninfos = new PartitionInfos(clusterService);
    Iterator<PartitionInfo> iter = partitioninfos.iterator();
    PartitionInfo partitioninfo = iter.next();
    assertThat(partitioninfo.name().asIndexName(), is(partitionName.asIndexName()));
    assertThat(partitioninfo.numberOfShards(), is(10));
    assertThat(partitioninfo.numberOfReplicas(), is("4"));
    assertThat(partitioninfo.values(), hasEntry("col", "foo"));
    assertThat(partitioninfo.values(), hasEntry("col2", 1));
    assertThat(iter.hasNext(), is(false));
}
 

private static IndexMetaData.Builder getIndexMetaData(String indexName,
                                                      Settings settings,
                                                      @Nullable Map<String, Object> mapping,
                                                      Version smallestNodeVersion) throws IOException {
    Settings.Builder builder = Settings.builder()
        .put(settings)
        .put(SETTING_VERSION_CREATED, smallestNodeVersion)
        .put(SETTING_CREATION_DATE, Instant.now().toEpochMilli())
        .put(SETTING_INDEX_UUID, UUIDs.randomBase64UUID());

    Settings indexSettings = builder.build();
    IndexMetaData.Builder metaBuilder = IndexMetaData.builder(indexName)
        .settings(indexSettings);
    if (mapping != null) {
        metaBuilder.putMapping(new MappingMetaData(
            Constants.DEFAULT_MAPPING_TYPE,
            mapping));
    }

    return metaBuilder;
}
 

/**
 * Updates the current {@link MetaData} based on the changes of this RoutingChangesObserver. Specifically
 * we update {@link IndexMetaData#getInSyncAllocationIds()} and {@link IndexMetaData#primaryTerm(int)} based on
 * the changes made during this allocation.
 *
 * @param oldMetaData {@link MetaData} object from before the routing nodes was changed.
 * @param newRoutingTable {@link RoutingTable} object after routing changes were applied.
 * @return adapted {@link MetaData}, potentially the original one if no change was needed.
 */
public MetaData applyChanges(MetaData oldMetaData, RoutingTable newRoutingTable) {
    Map<Index, List<Map.Entry<ShardId, Updates>>> changesGroupedByIndex =
        shardChanges.entrySet().stream().collect(Collectors.groupingBy(e -> e.getKey().getIndex()));

    MetaData.Builder metaDataBuilder = null;
    for (Map.Entry<Index, List<Map.Entry<ShardId, Updates>>> indexChanges : changesGroupedByIndex.entrySet()) {
        Index index = indexChanges.getKey();
        final IndexMetaData oldIndexMetaData = oldMetaData.getIndexSafe(index);
        IndexMetaData.Builder indexMetaDataBuilder = null;
        for (Map.Entry<ShardId, Updates> shardEntry : indexChanges.getValue()) {
            ShardId shardId = shardEntry.getKey();
            Updates updates = shardEntry.getValue();
            indexMetaDataBuilder = updateInSyncAllocations(newRoutingTable, oldIndexMetaData, indexMetaDataBuilder, shardId, updates);
            indexMetaDataBuilder = updatePrimaryTerm(oldIndexMetaData, indexMetaDataBuilder, shardId, updates);
        }

        if (indexMetaDataBuilder != null) {
            if (metaDataBuilder == null) {
                metaDataBuilder = MetaData.builder(oldMetaData);
            }
            metaDataBuilder.put(indexMetaDataBuilder);
        }
    }

    if (metaDataBuilder != null) {
        return metaDataBuilder.build();
    } else {
        return oldMetaData;
    }
}
 

/**
 * Increases the primary term if {@link #increasePrimaryTerm} was called for this shard id.
 */
private IndexMetaData.Builder updatePrimaryTerm(IndexMetaData oldIndexMetaData, IndexMetaData.Builder indexMetaDataBuilder,
                                                ShardId shardId, Updates updates) {
    if (updates.increaseTerm) {
        if (indexMetaDataBuilder == null) {
            indexMetaDataBuilder = IndexMetaData.builder(oldIndexMetaData);
        }
        indexMetaDataBuilder.primaryTerm(shardId.id(), oldIndexMetaData.primaryTerm(shardId.id()) + 1);
    }
    return indexMetaDataBuilder;
}
 

private IndexMetaData getIndexMetaData(String indexName,
                                       XContentBuilder builder) throws IOException {
    Map<String, Object> mappingSource = XContentHelper.convertToMap(BytesReference.bytes(builder), true, XContentType.JSON).v2();
    mappingSource = sortProperties(mappingSource);

    Settings.Builder settingsBuilder = Settings.builder()
        .put("index.number_of_shards", 1)
        .put("index.number_of_replicas", 0)
        .put("index.version.created", org.elasticsearch.Version.CURRENT);

    IndexMetaData.Builder mdBuilder = IndexMetaData.builder(indexName)
        .settings(settingsBuilder)
        .putMapping(new MappingMetaData(Constants.DEFAULT_MAPPING_TYPE, mappingSource));
    return mdBuilder.build();
}
 

/**
 * Updates in-sync allocations with routing changes that were made to the routing table.
 */
private IndexMetaData.Builder updateInSyncAllocations(RoutingTable newRoutingTable, IndexMetaData oldIndexMetaData,
                                                      IndexMetaData.Builder indexMetaDataBuilder, ShardId shardId, Updates updates) {
    assert Sets.haveEmptyIntersection(updates.addedAllocationIds, updates.removedAllocationIds) :
        "allocation ids cannot be both added and removed in the same allocation round, added ids: " +
        updates.addedAllocationIds + ", removed ids: " + updates.removedAllocationIds;

    Set<String> oldInSyncAllocationIds = oldIndexMetaData.inSyncAllocationIds(shardId.id());

    // check if we have been force-initializing an empty primary or a stale primary
    if (updates.initializedPrimary != null && oldInSyncAllocationIds.isEmpty() == false &&
        oldInSyncAllocationIds.contains(updates.initializedPrimary.allocationId().getId()) == false) {
        // we're not reusing an existing in-sync allocation id to initialize a primary, which means that we're either force-allocating
        // an empty or a stale primary (see AllocateEmptyPrimaryAllocationCommand or AllocateStalePrimaryAllocationCommand).
        RecoverySource recoverySource = updates.initializedPrimary.recoverySource();
        RecoverySource.Type recoverySourceType = recoverySource.getType();
        boolean emptyPrimary = recoverySourceType == RecoverySource.Type.EMPTY_STORE;
        assert updates.addedAllocationIds.isEmpty() : (emptyPrimary ? "empty" : "stale") +
                                                      " primary is not force-initialized in same allocation round where shards are started";

        if (indexMetaDataBuilder == null) {
            indexMetaDataBuilder = IndexMetaData.builder(oldIndexMetaData);
        }
        if (emptyPrimary) {
            // forcing an empty primary resets the in-sync allocations to the empty set (ShardRouting.allocatedPostIndexCreate)
            indexMetaDataBuilder.putInSyncAllocationIds(shardId.id(), Collections.emptySet());
        } else {
            final String allocationId;
            if (recoverySource == RecoverySource.ExistingStoreRecoverySource.FORCE_STALE_PRIMARY_INSTANCE) {
                allocationId = RecoverySource.ExistingStoreRecoverySource.FORCED_ALLOCATION_ID;
            } else {
                assert recoverySource instanceof RecoverySource.SnapshotRecoverySource : recoverySource;
                allocationId = updates.initializedPrimary.allocationId().getId();
            }
            // forcing a stale primary resets the in-sync allocations to the singleton set with the stale id
            indexMetaDataBuilder.putInSyncAllocationIds(shardId.id(), Collections.singleton(allocationId));
        }
    } else {
        // standard path for updating in-sync ids
        Set<String> inSyncAllocationIds = new HashSet<>(oldInSyncAllocationIds);
        inSyncAllocationIds.addAll(updates.addedAllocationIds);
        inSyncAllocationIds.removeAll(updates.removedAllocationIds);

        assert oldInSyncAllocationIds.contains(RecoverySource.ExistingStoreRecoverySource.FORCED_ALLOCATION_ID) == false
               || inSyncAllocationIds.contains(RecoverySource.ExistingStoreRecoverySource.FORCED_ALLOCATION_ID) == false :
            "fake allocation id has to be removed, inSyncAllocationIds:" + inSyncAllocationIds;

        // Prevent set of inSyncAllocationIds to grow unboundedly. This can happen for example if we don't write to a primary
        // but repeatedly shut down nodes that have active replicas.
        // We use number_of_replicas + 1 (= possible active shard copies) to bound the inSyncAllocationIds set
        // Only trim the set of allocation ids when it grows, otherwise we might trim too eagerly when the number
        // of replicas was decreased while shards were unassigned.
        int maxActiveShards = oldIndexMetaData.getNumberOfReplicas() + 1; // +1 for the primary
        IndexShardRoutingTable newShardRoutingTable = newRoutingTable.shardRoutingTable(shardId);
        if (inSyncAllocationIds.size() > oldInSyncAllocationIds.size() && inSyncAllocationIds.size() > maxActiveShards) {
            // trim entries that have no corresponding shard routing in the cluster state (i.e. trim unavailable copies)
            List<ShardRouting> assignedShards = newShardRoutingTable.assignedShards();
            assert assignedShards.size() <= maxActiveShards :
                "cannot have more assigned shards " + assignedShards + " than maximum possible active shards " + maxActiveShards;
            Set<String> assignedAllocations = assignedShards.stream().map(s -> s.allocationId().getId()).collect(Collectors.toSet());
            inSyncAllocationIds = inSyncAllocationIds.stream()
                .sorted(Comparator.comparing(assignedAllocations::contains).reversed()) // values with routing entries first
                .limit(maxActiveShards)
                .collect(Collectors.toSet());
        }

        // only remove allocation id of failed active primary if there is at least one active shard remaining. Assume for example that
        // the primary fails but there is no new primary to fail over to. If we were to remove the allocation id of the primary from the
        // in-sync set, this could create an empty primary on the next allocation.
        if (newShardRoutingTable.activeShards().isEmpty() && updates.firstFailedPrimary != null) {
            // add back allocation id of failed primary
            inSyncAllocationIds.add(updates.firstFailedPrimary.allocationId().getId());
        }

        assert inSyncAllocationIds.isEmpty() == false || oldInSyncAllocationIds.isEmpty() :
            "in-sync allocations cannot become empty after they have been non-empty: " + oldInSyncAllocationIds;

        // be extra safe here and only update in-sync set if it is non-empty
        if (inSyncAllocationIds.isEmpty() == false) {
            if (indexMetaDataBuilder == null) {
                indexMetaDataBuilder = IndexMetaData.builder(oldIndexMetaData);
            }
            indexMetaDataBuilder.putInSyncAllocationIds(shardId.id(), inSyncAllocationIds);
        }
    }
    return indexMetaDataBuilder;
}