org.elasticsearch.index.mapper.Mapping Java Examples

private Engine.IndexingOperation updateMappingIfRequired(ShardUpsertRequest request,
                                                         ShardUpsertRequest.Item item,
                                                         long version,
                                                         IndexShard indexShard,
                                                         Engine.IndexingOperation operation) throws Throwable {
    Mapping update = operation.parsedDoc().dynamicMappingsUpdate();
    if (update != null) {
        mappingUpdatedAction.updateMappingOnMasterSynchronously(
                request.shardId().getIndex(), request.type(), update);

        operation = prepareIndexOnPrimary(indexShard, version, request, item);
        if (operation.parsedDoc().dynamicMappingsUpdate() != null) {
            throw new RetryOnPrimaryException(request.shardId(),
                    "Dynamics mappings are not available on the node that holds the primary yet");
        }
    }
    return operation;
}
 

public void updateMappingOnMaster(String index, String type, Mapping mappingUpdate, final TimeValue timeout, final MappingUpdateListener listener) {
    final PutMappingRequestBuilder request = updateMappingRequest(index, type, mappingUpdate, timeout);
    if (listener == null) {
        request.execute();
    } else {
        final ActionListener<PutMappingResponse> actionListener = new ActionListener<PutMappingResponse>() {
            @Override
            public void onResponse(PutMappingResponse response) {
                if (response.isAcknowledged()) {
                    listener.onMappingUpdate();
                } else {
                    listener.onFailure(new TimeoutException("Failed to acknowledge the mapping response within [" + timeout + "]"));
                }
            }

            @Override
            public void onFailure(Throwable e) {
                listener.onFailure(e);
            }
        };
        request.execute(actionListener);
    }
}
 

private void applyOperation(Engine engine, Engine.Operation operation) throws IOException {
    switch (operation.operationType()) {
        case INDEX:
            Engine.Index engineIndex = (Engine.Index) operation;
            Mapping update = engineIndex.parsedDoc().dynamicMappingsUpdate();
            if (engineIndex.parsedDoc().dynamicMappingsUpdate() != null) {
                recoveredTypes.compute(engineIndex.type(), (k, mapping) -> mapping == null ? update : mapping.merge(update, false));
            }
            engine.index(engineIndex);
            break;
        case DELETE:
            engine.delete((Engine.Delete) operation);
            break;
        case NO_OP:
            engine.noOp((Engine.NoOp) operation);
            break;
        default:
            throw new IllegalStateException("No operation defined for [" + operation + "]");
    }
}
 

protected static ParsedDocument testParsedDocument(
    String id, String routing, ParseContext.Document document, BytesReference source, Mapping mappingUpdate,
    boolean recoverySource) {
    Field uidField = new Field("_id", Uid.encodeId(id), IdFieldMapper.Defaults.FIELD_TYPE);
    Field versionField = new NumericDocValuesField("_version", 0);
    SeqNoFieldMapper.SequenceIDFields seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID();
    document.add(uidField);
    document.add(versionField);
    document.add(seqID.seqNo);
    document.add(seqID.seqNoDocValue);
    document.add(seqID.primaryTerm);
    BytesRef ref = source.toBytesRef();
    if (recoverySource) {
        document.add(new StoredField(SourceFieldMapper.RECOVERY_SOURCE_NAME, ref.bytes, ref.offset, ref.length));
        document.add(new NumericDocValuesField(SourceFieldMapper.RECOVERY_SOURCE_NAME, 1));
    } else {
        document.add(new StoredField(SourceFieldMapper.NAME, ref.bytes, ref.offset, ref.length));
    }
    return new ParsedDocument(versionField, seqID, id, routing, Arrays.asList(document), source, mappingUpdate);
}
 

private void validateMappingUpdate(final String indexName, final String type, Mapping update) {
    final CountDownLatch latch = new CountDownLatch(1);
    final AtomicReference<Throwable> error = new AtomicReference<>();
    mappingUpdatedAction.updateMappingOnMaster(indexName, type, update, waitForMappingUpdatePostRecovery, new MappingUpdatedAction.MappingUpdateListener() {
        @Override
        public void onMappingUpdate() {
            latch.countDown();
        }

        @Override
        public void onFailure(Throwable t) {
            latch.countDown();
            error.set(t);
        }
    });
    cancellableThreads.execute(new CancellableThreads.Interruptable() {
        @Override
        public void run() throws InterruptedException {
            try {
                if (latch.await(waitForMappingUpdatePostRecovery.millis(), TimeUnit.MILLISECONDS) == false) {
                    logger.debug("waited for mapping update on master for [{}], yet timed out", type);
                } else {
                    if (error.get() != null) {
                        throw new IndexShardRecoveryException(shardId, "Failed to propagate mappings on master post recovery", error.get());
                    }
                }
            } catch (InterruptedException e) {
                logger.debug("interrupted while waiting for mapping update");
                throw e;
            }
        }
    });
}
 

private void maybeAddMappingUpdate(String type, Mapping update, String docId, boolean allowMappingUpdates) {
    if (update == null) {
        return;
    }
    if (allowMappingUpdates == false) {
        throw new MapperException("mapping updates are not allowed (type: [" + type + "], id: [" + docId + "])");
    }
    Mapping currentUpdate = recoveredTypes.get(type);
    if (currentUpdate == null) {
        recoveredTypes.put(type, update);
    } else {
        currentUpdate = currentUpdate.merge(update, false);
    }
}
 

/**
 * After the store has been recovered, we need to start the engine in order to apply operations
 */
public Map<String, Mapping> performTranslogRecovery(boolean indexExists) {
    if (indexExists == false) {
        // note: these are set when recovering from the translog
        final RecoveryState.Translog translogStats = recoveryState().getTranslog();
        translogStats.totalOperations(0);
        translogStats.totalOperationsOnStart(0);
    }
    final Map<String, Mapping> recoveredTypes = internalPerformTranslogRecovery(false, indexExists);
    assert recoveryState.getStage() == RecoveryState.Stage.TRANSLOG : "TRANSLOG stage expected but was: " + recoveryState.getStage();
    return recoveredTypes;
}
 

private Map<String, Mapping> internalPerformTranslogRecovery(boolean skipTranslogRecovery, boolean indexExists) {
    if (state != IndexShardState.RECOVERING) {
        throw new IndexShardNotRecoveringException(shardId, state);
    }

    recoveryState.setStage(RecoveryState.Stage.VERIFY_INDEX);
    // also check here, before we apply the translog
    if (Booleans.parseBoolean(checkIndexOnStartup, false)) {
        try {
            checkIndex();
        } catch (IOException ex) {
            throw new RecoveryFailedException(recoveryState, "check index failed", ex);
        }
    }
    recoveryState.setStage(RecoveryState.Stage.TRANSLOG);
    // we disable deletes since we allow for operations to be executed against the shard while recovering
    // but we need to make sure we don't loose deletes until we are done recovering
    engineConfig.setEnableGcDeletes(false);
    engineConfig.setCreate(indexExists == false);
    if (recoveryState.getType() == RecoveryState.Type.SNAPSHOT) {
        engineConfig.setIgnoreTranslogStatus(true);
    } else {
        engineConfig.setIgnoreTranslogStatus(false);
    }
    if (skipTranslogRecovery == false) {
        // This will activate our shard so we get our fair share of the indexing buffer during recovery:
        markLastWrite();
    }
    createNewEngine(skipTranslogRecovery, engineConfig);
    return engineConfig.getTranslogRecoveryPerformer().getRecoveredTypes();
}
 

private PutMappingRequestBuilder updateMappingRequest(String index, String type, Mapping mappingUpdate, final TimeValue timeout) {
    if (type.equals(MapperService.DEFAULT_MAPPING)) {
        throw new IllegalArgumentException("_default_ mapping should not be updated");
    }
    return client.preparePutMapping(index).setType(type).setSource(mappingUpdate.toString())
        .setMasterNodeTimeout(timeout).setTimeout(timeout);
}
 

protected Result(Operation.TYPE operationType, Mapping requiredMappingUpdate) {
    this.operationType = operationType;
    this.version = Versions.NOT_FOUND;
    this.seqNo = SequenceNumbers.UNASSIGNED_SEQ_NO;
    this.term = 0L;
    this.failure = null;
    this.requiredMappingUpdate = requiredMappingUpdate;
    this.resultType = Type.MAPPING_UPDATE_REQUIRED;
}
 

public void blockingUpdateOnMaster(Index index, Mapping mappingUpdate) {
    TimeValue timeout = this.dynamicMappingUpdateTimeout;
    var response = schemaUpdateAction.execute(new SchemaUpdateRequest(index, mappingUpdate.toString())).actionGet();
    if (!response.isAcknowledged()) {
        throw new ElasticsearchTimeoutException("Failed to acknowledge mapping update within [" + timeout + "]");
    }
}
 

private PutMappingRequestBuilder updateMappingRequest(Index index, String type, Mapping mappingUpdate, final TimeValue timeout) {
    if (type.equals(MapperService.DEFAULT_MAPPING)) {
        throw new IllegalArgumentException("_default_ mapping should not be updated");
    }
    return client.preparePutMapping().setConcreteIndex(index).setType(type).setSource(mappingUpdate.toString(), XContentType.JSON)
            .setMasterNodeTimeout(timeout).setTimeout(timeout);
}
 

@Test
public void testObjectArrayMappingNewColumn() throws Exception {
    // @formatter: off
    String mapping = Strings.toString(XContentFactory.jsonBuilder()
        .startObject()
            .startObject(TYPE)
                .startObject("properties")
                    .startObject("array_field")
                        .field("type", ArrayMapper.CONTENT_TYPE)
                        .startObject(ArrayMapper.INNER_TYPE)
                            .field("type", "object")
                            .field("dynamic", true)
                            .startObject("properties")
                                .startObject("s")
                                    .field("type", "keyword")
                                .endObject()
                            .endObject()
                        .endObject()
                    .endObject()
                .endObject()
            .endObject()
        .endObject());
    DocumentMapper mapper = mapper(INDEX, mapping);
    // child object mapper
    assertThat(mapper.objectMappers().get("array_field"), is(instanceOf(ObjectArrayMapper.class)));
    BytesReference bytesReference = BytesReference.bytes(XContentFactory.jsonBuilder()
        .startObject()
        .startArray("array_field")
        .startObject()
        .field("s", "a")
        .field("new", true)
        .endObject()
        .endArray()
        .endObject());
    SourceToParse sourceToParse = new SourceToParse(INDEX, "abc", bytesReference, XContentType.JSON);
    ParsedDocument doc = mapper.parse(sourceToParse);

    Mapping mappingUpdate = doc.dynamicMappingsUpdate();
    assertThat(mappingUpdate, notNullValue());
    mapper = mapper.merge(mappingUpdate, true);
    assertThat(doc.docs().size(), is(1));
    String[] values = doc.docs().get(0).getValues("array_field.new");
    assertThat(values, arrayContainingInAnyOrder(is("T"), is("1")));
    String mappingSourceString = new CompressedXContent(mapper, XContentType.JSON, ToXContent.EMPTY_PARAMS).string();
    assertThat(
        mappingSourceString,
        is("{\"default\":{" +
           "\"properties\":{" +
           "\"array_field\":{" +
           "\"type\":\"array\"," +
           "\"inner\":{" +
           "\"dynamic\":\"true\"," +
           "\"properties\":{" +
           "\"new\":{\"type\":\"boolean\"}," +
           "\"s\":{" +
           "\"type\":\"keyword\"" +
           "}" +
           "}" +
           "}" +
           "}" +
           "}}}"));
}
 

/**
 * Returns the recovered types modifying the mapping during the recovery
 */
public Map<String, Mapping> getRecoveredTypes() {
    return recoveredTypes;
}
 

protected static ParsedDocument testParsedDocument(
    String id, String routing, ParseContext.Document document, BytesReference source, Mapping mappingUpdate) {
    return testParsedDocument(id, routing, document, source, mappingUpdate, false);
}
 

/**
 * Update mappings synchronously on the master node, waiting for at most
 * {@code timeout}. When this method returns successfully mappings have
 * been applied to the master node and propagated to data nodes.
 */
public void updateMappingOnMaster(Index index, String type, Mapping mappingUpdate, TimeValue timeout) {
    if (updateMappingRequest(index, type, mappingUpdate, timeout).get().isAcknowledged() == false) {
        throw new ElasticsearchTimeoutException("Failed to acknowledge mapping update within [" + timeout + "]");
    }
}
 

public DeleteResult(Mapping requiredMappingUpdate) {
    super(Operation.TYPE.DELETE, requiredMappingUpdate);
    this.found = false;
}
 

public IndexResult(Mapping requiredMappingUpdate) {
    super(Operation.TYPE.INDEX, requiredMappingUpdate);
    this.created = false;
}
 

/**
 * Update mappings synchronously on the master node, waiting for at most
 * {@code timeout}. When this method returns successfully mappings have
 * been applied to the master node and propagated to data nodes.
 */
public void updateMappingOnMasterSynchronously(String index, String type, Mapping mappingUpdate, TimeValue timeout) throws Throwable {
    if (updateMappingRequest(index, type, mappingUpdate, timeout).get().isAcknowledged() == false) {
        throw new TimeoutException("Failed to acknowledge mapping update within [" + timeout + "]");
    }
}
 

public void updateMappingOnMasterAsynchronously(String index, String type, Mapping mappingUpdate) throws Throwable {
    updateMappingOnMaster(index, type, mappingUpdate, dynamicMappingUpdateTimeout, null);
}
 

/**
 * Returns the recovered types modifying the mapping during the recovery
 */
public Map<String, Mapping> getRecoveredTypes() {
    return recoveredTypes;
}
 

/**
 * Same as {@link #updateMappingOnMaster(Index, String, Mapping, TimeValue)}
 * using the default timeout.
 */
public void updateMappingOnMaster(Index index, String type, Mapping mappingUpdate) {
    updateMappingOnMaster(index, type, mappingUpdate, dynamicMappingUpdateTimeout);
}
 

/**
 * Update the mappings on the master.
 */
void updateMappings(Mapping update, ShardId shardId, String type);
 

/**
 * If the operation was aborted due to missing mappings, this method will return the mappings
 * that are required to complete the operation.
 */
public Mapping getRequiredMappingUpdate() {
    return requiredMappingUpdate;
}
 

/**
 * Same as {@link #updateMappingOnMasterSynchronously(String, String, Mapping, TimeValue)}
 * using the default timeout.
 */
public void updateMappingOnMasterSynchronously(String index, String type, Mapping mappingUpdate) throws Throwable {
    updateMappingOnMasterSynchronously(index, type, mappingUpdate, dynamicMappingUpdateTimeout);
}