org.elasticsearch.index.mapper.SourceToParse Java Examples

void indexValue(String column, Object value) throws IOException {
    DocumentMapper mapper = indexEnv.mapperService().documentMapperSafe();
    InsertSourceGen sourceGen = InsertSourceGen.of(
        CoordinatorTxnCtx.systemTransactionContext(),
        sqlExecutor.functions(),
        table,
        table.concreteIndices()[0],
        GeneratedColumns.Validation.NONE,
        Collections.singletonList(table.getReference(ColumnIdent.fromPath(column)))
    );
    BytesReference source = sourceGen.generateSourceAndCheckConstraintsAsBytesReference(new Object[]{value});
    SourceToParse sourceToParse = new SourceToParse(
        table.concreteIndices()[0],
        UUIDs.randomBase64UUID(),
        source,
        XContentType.JSON
    );
    ParsedDocument parsedDocument = mapper.parse(sourceToParse);
    indexEnv.writer().addDocuments(parsedDocument.docs());
}
 

@Test
public void testParseDynamicNullArray() throws Exception {
    String mapping = Strings.toString(XContentFactory.jsonBuilder()
        .startObject().startObject(TYPE).startObject("properties")
        .endObject().endObject().endObject());
    DocumentMapper mapper = mapper(INDEX, mapping);

    // parse source with null array
    BytesReference bytesReference = BytesReference.bytes(XContentFactory.jsonBuilder()
        .startObject()
        .startArray("new_array_field").nullValue().endArray()
        .endObject());
    SourceToParse sourceToParse = new SourceToParse(INDEX, "abc", bytesReference, XContentType.JSON);
    ParsedDocument doc = mapper.parse(sourceToParse);
    assertThat(doc.docs().get(0).getField("new_array_field"), is(nullValue()));
    assertThat(mapper.mappers().getMapper("new_array_field"), is(nullValue()));
}
 

@Test
public void testParseDynamicEmptyArray() throws Exception {
    String mapping = Strings.toString(XContentFactory.jsonBuilder()
        .startObject().startObject(TYPE).startObject("properties")
        .endObject().endObject().endObject());
    DocumentMapper mapper = mapper(INDEX, mapping);

    // parse source with empty array
    BytesReference bytesReference = BytesReference.bytes(XContentFactory.jsonBuilder()
        .startObject()
        .array("new_array_field")
        .endObject());
    SourceToParse sourceToParse = new SourceToParse(INDEX, "abc", bytesReference, XContentType.JSON);
    ParsedDocument doc = mapper.parse(sourceToParse);
    assertThat(doc.docs().get(0).getField("new_array_field"), is(nullValue()));
    assertThat(mapper.mappers().getMapper("new_array_field"), is(nullValue()));
}
 

@Test
public void testInvalidArrayNonConvertableType() throws Exception {
    String mapping = Strings.toString(XContentFactory.jsonBuilder()
        .startObject().startObject(TYPE).startObject("properties")
        .startObject("array_field")
        .field("type", ArrayMapper.CONTENT_TYPE)
        .startObject(ArrayMapper.INNER_TYPE)
        .field("type", "double")
        .endObject()
        .endObject()
        .endObject().endObject().endObject());
    DocumentMapper mapper = mapper(INDEX, mapping);

    expectedException.expect(MapperParsingException.class);
    expectedException.expectMessage("failed to parse field [array_field] of type [double]");

    BytesReference bytesReference = BytesReference.bytes(XContentFactory.jsonBuilder()
        .startObject()
        .array("array_field", true, false, true)
        .endObject());
    SourceToParse sourceToParse = new SourceToParse(INDEX, "abc", bytesReference, XContentType.JSON);
    mapper.parse(sourceToParse);
}
 

@Test
public void testItemsWithoutSourceAreSkippedOnReplicaOperation() throws Exception {
    ShardId shardId = new ShardId(TABLE_IDENT.indexNameOrAlias(), charactersIndexUUID, 0);
    ShardUpsertRequest request = new ShardUpsertRequest.Builder(
        DUMMY_SESSION_INFO,
        TimeValue.timeValueSeconds(30),
        DuplicateKeyAction.UPDATE_OR_FAIL,
        false,
        null,
        new Reference[]{ID_REF},
        null,
        UUID.randomUUID(),
        false
    ).newRequest(shardId);
    request.add(1, new ShardUpsertRequest.Item("1", null, new Object[]{1}, null, null, null));

    reset(indexShard);

    // would fail with NPE if not skipped
    transportShardUpsertAction.processRequestItemsOnReplica(indexShard, request);
    verify(indexShard, times(0)).applyIndexOperationOnReplica(
        anyLong(), anyLong(), anyLong(), anyBoolean(), any(SourceToParse.class));
}
 

private void reindex(QueryFetchSearchResult hits, String index, String type) {
    logger.debug("Reindex: [index:{}, type:{}]", index, type);

    if (state == IndexShardState.STARTED) {
        for (InternalSearchHit hit : hits.fetchResult().hits().internalHits()) {
            // no difference between PRIMARY and REPLICA
            SourceToParse source = SourceToParse.source(SourceToParse.Origin.REPLICA, hit.sourceRef())
                    .index(index).type(type).id(hit.id());
            if (hit.field(ParentFieldMapper.NAME).getValue() != null) {
                source.parent((String) hit.field(ParentFieldMapper.NAME).getValue());
            }
            if (hit.field(TimestampFieldMapper.NAME).getValue() != null) {
                source.timestamp((long) hit.field(TimestampFieldMapper.NAME).getValue());
            }
            long version = 0;
            if (hit.field(VersionFieldMapper.NAME).getValue() != null) {
                version = (long) hit.field(VersionFieldMapper.NAME).getValue();
            }
            Engine.Index indexOp = prepareIndex(docMapper(source.type()), source, version, VersionType.EXTERNAL_GTE, Engine.Operation.Origin.RECOVERY, state != IndexShardState.STARTED);
            indexOp.setReindex(true);
            index(indexOp);
        }
    }
}
 

@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException, AlreadyExpiredException {
    if (enabledState.enabled && !context.sourceToParse().flyweight()) {
        long ttl = context.sourceToParse().ttl();
        if (ttl <= 0 && defaultTTL > 0) { // no ttl provided so we use the default value
            ttl = defaultTTL;
            context.sourceToParse().ttl(ttl);
        }
        if (ttl > 0) { // a ttl has been provided either externally or in the _source
            long timestamp = context.sourceToParse().timestamp();
            long expire = new Date(timestamp + ttl).getTime();
            long now = System.currentTimeMillis();
            // there is not point indexing already expired doc
            if (context.sourceToParse().origin() == SourceToParse.Origin.PRIMARY && now >= expire) {
                throw new AlreadyExpiredException(context.index(), context.type(), context.id(), timestamp, ttl, now);
            }
            // the expiration timestamp (timestamp + ttl) is set as field
            fields.add(new LongFieldMapper.CustomLongNumericField(expire, fieldType()));
        }
    }
}
 

public static Engine.Index prepareIndex(DocumentMapper docMapper,
                                        SourceToParse source,
                                        long seqNo,
                                        long primaryTerm,
                                        long version,
                                        VersionType versionType,
                                        Engine.Operation.Origin origin,
                                        long autoGeneratedIdTimestamp,
                                        boolean isRetry,
                                        long ifSeqNo,
                                        long ifPrimaryTerm) {
    long startTime = System.nanoTime();
    ParsedDocument doc = docMapper.parse(source);
    Term uid = new Term(IdFieldMapper.NAME, Uid.encodeId(doc.id()));
    return new Engine.Index(uid, doc, seqNo, primaryTerm, version, versionType, origin, startTime, autoGeneratedIdTimestamp, isRetry,
                            ifSeqNo, ifPrimaryTerm);
}
 

public void testNonStandardnumber() throws Exception {
    String mapping = copyToStringFromClasspath("mapping.json");
    DocumentMapper docMapper = createIndex("some_index")
            .mapperService().documentMapperParser()
            .parse("someType", new CompressedXContent(mapping));
    String sampleText = "Hello world";
    BytesReference json = BytesReference.bytes(XContentFactory.jsonBuilder()
            .startObject().field("someField", sampleText).endObject());
    SourceToParse sourceToParse = SourceToParse.source("some_index", "someType", "1", json, XContentType.JSON);
    ParseContext.Document doc = docMapper.parse(sourceToParse).rootDoc();
    assertEquals(0, doc.getFields("someField").length);
    // re-parse it
    String builtMapping = docMapper.mappingSource().string();
    logger.warn("testNonStandardnumber: built mapping =" + builtMapping);
    DocumentMapper docMapper2 = createIndex("some_index2")
            .mapperService().documentMapperParser()
            .parse("someType", new CompressedXContent(builtMapping));
    json = BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
            .field("someField", sampleText).endObject());
    sourceToParse = SourceToParse.source("some_index2", "someType", "1", json, XContentType.JSON);
    doc = docMapper2.parse(sourceToParse).rootDoc();
    assertEquals(0, doc.getFields("someField").length);
}
 

private Engine.IndexingOperation prepareIndexOnPrimary(IndexShard indexShard,
                                                       long version,
                                                       ShardUpsertRequest request,
                                                       ShardUpsertRequest.Item item) {
    SourceToParse sourceToParse = SourceToParse.source(SourceToParse.Origin.PRIMARY, item.source())
            .type(request.type())
            .id(item.id())
            .routing(request.routing());

    if (logger.isTraceEnabled()) {
        logger.trace("[{}] shard operation with opType={} id={} version={}  source={}",
                indexShard.shardId(), item.opType(), item.id(), version, item.source().toUtf8());
    }
    if (item.opType() == IndexRequest.OpType.INDEX) {
        return indexShard.prepareIndexOnPrimary(sourceToParse, version, item.versionType(), request.canHaveDuplicates());
    }
    return indexShard.prepareCreateOnPrimary(
            sourceToParse, version, item.versionType(), request.canHaveDuplicates(), false);
}
 

private void indexOperationOnReplica(IndexShard indexShard, IndexRequest indexRequest) {
    SourceToParse sourceToParse = SourceToParse.source(SourceToParse.Origin.REPLICA, indexRequest.source())
            .type(indexRequest.type())
            .id(indexRequest.id())
            .routing(indexRequest.routing())
            .parent(indexRequest.parent())
            .timestamp(indexRequest.timestamp())
            .ttl(indexRequest.ttl());
    if (indexRequest.opType() == IndexRequest.OpType.INDEX) {
        Engine.Index index = indexShard.prepareIndexOnReplica(sourceToParse,
                indexRequest.version(),
                indexRequest.versionType(),
                false);
        indexShard.index(index);
    } else {
        Engine.Create create = indexShard.prepareCreateOnReplica(sourceToParse,
                indexRequest.version(),
                indexRequest.versionType(),
                false,
                indexRequest.autoGeneratedId());
        indexShard.create(create);
    }
}
 

public Engine.IndexResult applyIndexOperationOnReplica(long seqNo,
                                                       long version,
                                                       long autoGeneratedTimeStamp,
                                                       boolean isRetry,
                                                       SourceToParse sourceToParse)
    throws IOException {
    return applyIndexOperation(getEngine(), seqNo, operationPrimaryTerm, version, null, UNASSIGNED_SEQ_NO, 0,
                               autoGeneratedTimeStamp, isRetry, Engine.Operation.Origin.REPLICA, sourceToParse);
}
 

@Test
public void testParseNullOnObjectArray() throws Exception {
    // @formatter: on
    String mapping = Strings.toString(XContentFactory.jsonBuilder()
        .startObject()
            .startObject(TYPE)
                .startObject("properties")
                    .startObject("array_field")
                        .field("type", ArrayMapper.CONTENT_TYPE)
                        .startObject(ArrayMapper.INNER_TYPE)
                            .field("type", "object")
                            .startObject("properties")
                            .endObject()
                        .endObject()
                    .endObject()
                .endObject()
            .endObject()
        .endObject());
    // @formatter: off
    DocumentMapper mapper = mapper(INDEX, mapping);
    BytesReference bytesReference = BytesReference.bytes(XContentFactory.jsonBuilder()
        .startObject()
        .nullField("array_field")
        .endObject());
    SourceToParse sourceToParse = new SourceToParse(INDEX, "abc", bytesReference, XContentType.JSON);
    ParsedDocument parsedDoc = mapper.parse(sourceToParse);
    assertThat(parsedDoc.docs().size(), is(1));
    assertThat(parsedDoc.docs().get(0).getField("array_field"), is(nullValue()));


}
 

@Test
public void testParseNull() throws Exception {
    // @formatter: off
    String mapping = Strings.toString(XContentFactory.jsonBuilder()
        .startObject()
            .startObject(TYPE)
                .startObject("properties")
                    .startObject("array_field")
                        .field("type", ArrayMapper.CONTENT_TYPE)
                        .startObject(ArrayMapper.INNER_TYPE)
                            .field("type", "double")
                        .endObject()
                    .endObject()
                .endObject()
            .endObject()
        .endObject());
    // @formatter: on
    DocumentMapper mapper = mapper(INDEX, mapping);
    BytesReference bytesReference = BytesReference.bytes(XContentFactory.jsonBuilder()
        .startObject()
        .nullField("array_field")
        .endObject());
    SourceToParse sourceToParse = new SourceToParse(INDEX, "abc", bytesReference, XContentType.JSON);
    ParsedDocument parsedDoc = mapper.parse(sourceToParse);
    assertThat(parsedDoc.docs().size(), is(1));
    assertThat(parsedDoc.docs().get(0).getField("array_field"), is(nullValue()));
}
 

public Engine.IndexResult applyIndexOperationOnPrimary(long version,
                                                       VersionType versionType,
                                                       SourceToParse sourceToParse,
                                                       long ifSeqNo,
                                                       long ifPrimaryTerm,
                                                       long autoGeneratedTimestamp,
                                                       boolean isRetry)
    throws IOException {
    return applyIndexOperation(getEngine(), UNASSIGNED_SEQ_NO, operationPrimaryTerm, version, versionType, ifSeqNo,
                               ifPrimaryTerm, autoGeneratedTimestamp, isRetry, Engine.Operation.Origin.PRIMARY, sourceToParse);
}
 

static Engine.Index prepareIndex(DocumentMapperForType docMapper, SourceToParse source, long version, VersionType versionType, Engine
        .Operation.Origin origin, boolean canHaveDuplicates) {
    long startTime = System.nanoTime();
    ParsedDocument doc = docMapper.getDocumentMapper().parse(source);
    if (docMapper.getMapping() != null) {
        doc.addDynamicMappingsUpdate(docMapper.getMapping());
    }
    return new Engine.Index(docMapper.getDocumentMapper().uidMapper().term(doc.uid().stringValue()), doc, version, versionType,
            origin, startTime, canHaveDuplicates);
}
 

private void shardIndexOperationOnReplica(ShardUpsertRequest request,
                                          ShardUpsertRequest.Item item,
                                          IndexShard indexShard) {
    SourceToParse sourceToParse = SourceToParse.source(SourceToParse.Origin.REPLICA, item.source())
            .type(request.type())
            .id(item.id())
            .routing(request.routing());

    try {
        if (item.opType() == IndexRequest.OpType.INDEX) {
            if (logger.isTraceEnabled()) {
                logger.trace("[{} (R)] Updating document with id {}, source: {}", indexShard.shardId(), item.id(), item.source().toUtf8());
            }
            Engine.Index index = indexShard.prepareIndexOnReplica(
                    sourceToParse, item.version(), item.versionType(), request.canHaveDuplicates());
            indexShard.index(index);
        } else {
            if (logger.isTraceEnabled()) {
                logger.trace("[{} (R)] Creating document with id {}, source: {}", indexShard.shardId(), item.id(), item.source().toUtf8());
            }
            Engine.Create create = indexShard.prepareCreateOnReplica(
                    sourceToParse, item.version(), item.versionType(), request.canHaveDuplicates(), false);
            indexShard.create(create);
        }
    } catch (Throwable t) {
        // if its not an ignore replica failure, we need to make sure to bubble up the failure
        // so we will fail the shard
        if (!ignoreReplicaException(t)) {
            throw t;
        }
    }
}
 

public Engine.Create prepareCreateOnPrimary(SourceToParse source, long version, VersionType versionType, boolean canHaveDuplicates,
                                            boolean autoGeneratedId) {
    try {
        if (shardRouting.primary() == false) {
            throw new IllegalIndexShardStateException(shardId, state, "shard is not a primary");
        }
        return prepareCreate(docMapper(source.type()), source, version, versionType, Engine.Operation.Origin.PRIMARY, state !=
                IndexShardState.STARTED || canHaveDuplicates, autoGeneratedId);
    } catch (Throwable t) {
        verifyNotClosed(t);
        throw t;
    }
}
 

public Engine.Create prepareCreateOnReplica(SourceToParse source, long version, VersionType versionType, boolean canHaveDuplicates,
                                            boolean autoGeneratedId) {
    try {
        return prepareCreate(docMapper(source.type()), source, version, versionType, Engine.Operation.Origin.REPLICA, state !=
                IndexShardState.STARTED || canHaveDuplicates, autoGeneratedId);
    } catch (Throwable t) {
        verifyNotClosed(t);
        throw t;
    }
}
 

static Engine.Create prepareCreate(DocumentMapperForType docMapper, SourceToParse source, long version, VersionType versionType,
                                   Engine.Operation.Origin origin, boolean canHaveDuplicates, boolean autoGeneratedId) {
    long startTime = System.nanoTime();
    ParsedDocument doc = docMapper.getDocumentMapper().parse(source);
    if (docMapper.getMapping() != null) {
        doc.addDynamicMappingsUpdate(docMapper.getMapping());
    }
    return new Engine.Create(docMapper.getDocumentMapper().uidMapper().term(doc.uid().stringValue()), doc, version, versionType,
            origin, startTime, canHaveDuplicates, autoGeneratedId);
}
 

public Engine.Index prepareIndexOnPrimary(SourceToParse source, long version, VersionType versionType, boolean canHaveDuplicates) {
    try {
        if (shardRouting.primary() == false) {
            throw new IllegalIndexShardStateException(shardId, state, "shard is not a primary");
        }
        return prepareIndex(docMapper(source.type()), source, version, versionType, Engine.Operation.Origin.PRIMARY, state !=
                IndexShardState.STARTED || canHaveDuplicates);
    } catch (Throwable t) {
        verifyNotClosed(t);
        throw t;
    }
}
 

public Engine.Index prepareIndexOnReplica(SourceToParse source, long version, VersionType versionType, boolean canHaveDuplicates) {
    try {
        return prepareIndex(docMapper(source.type()), source, version, versionType, Engine.Operation.Origin.REPLICA, state !=
                IndexShardState.STARTED || canHaveDuplicates);
    } catch (Throwable t) {
        verifyNotClosed(t);
        throw t;
    }
}
 

public void testRefFromID() throws Exception {
    IndexService indexService = createIndex("docs", Settings.EMPTY,
            "docs", getMapping("ref-mapping-from-id.json"));
    DocumentMapper docMapper = indexService.mapperService().documentMapper("docs");
    BytesReference json = BytesReference.bytes(jsonBuilder().startObject()
            .field("title", "A title")
            .field("authorID", "1")
            .endObject());
    SourceToParse sourceToParse = SourceToParse.source("docs", "docs", "1", json, XContentType.JSON);
    ParseContext.Document doc = docMapper.parse(sourceToParse).rootDoc();
    assertEquals(1, doc.getFields("ref").length, 1);
    assertEquals("John Doe", doc.getFields("ref")[0].stringValue());
}
 

public void testSimpleStandardNumber() throws Exception {
    String mapping = copyToStringFromClasspath("mapping.json");
    DocumentMapper docMapper = createIndex("some_index")
            .mapperService().documentMapperParser()
            .parse("someType", new CompressedXContent(mapping));
    String sampleText = "978-3-551-75213-0";
    BytesReference json = BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
            .field("someField", sampleText).endObject());
    SourceToParse sourceToParse = SourceToParse.source("some_index", "someType", "1", json, XContentType.JSON);
    ParseContext.Document doc = docMapper.parse(sourceToParse).rootDoc();
    assertEquals(2, doc.getFields("someField").length);
    assertEquals("978-3-551-75213-0", doc.getFields("someField")[0].stringValue());
    assertEquals("9783551752130", doc.getFields("someField")[1].stringValue());
}
 

public void testSimpleMapping() throws Exception {
    IndexService indexService = createIndex("some_index", Settings.EMPTY,
            "someType", getMapping("simple-mapping.json"));
    DocumentMapper docMapper = indexService.mapperService().documentMapper("someType");
    String sampleText = copyToStringFromClasspath("english.txt");
    BytesReference json = BytesReference.bytes(XContentFactory.jsonBuilder()
            .startObject().field("someField", sampleText).endObject());
    SourceToParse sourceToParse = SourceToParse.source("some_index", "someType", "1", json, XContentType.JSON);
    ParsedDocument doc = docMapper.parse(sourceToParse);
    assertEquals(1, doc.rootDoc().getFields("someField").length);
    assertEquals("en", doc.rootDoc().getFields("someField")[0].stringValue());
}
 

public void testToFields() throws Exception {
    IndexService indexService = createIndex("some_index", Settings.EMPTY,
            "someType", getMapping("mapping-to-fields.json"));
    DocumentMapper docMapper = indexService.mapperService().documentMapper("someType");
    String sampleText = copyToStringFromClasspath("english.txt");
    BytesReference json = BytesReference.bytes(XContentFactory.jsonBuilder()
            .startObject().field("someField", sampleText).endObject());
    SourceToParse sourceToParse = SourceToParse.source("some_index", "someType", "1", json, XContentType.JSON);
    ParsedDocument doc = docMapper.parse(sourceToParse);
    assertEquals(1, doc.rootDoc().getFields("someField").length);
    assertEquals("en", doc.rootDoc().getFields("someField")[0].stringValue());
    assertEquals(1, doc.rootDoc().getFields("english_field").length);
    assertEquals("This is a very small example of a text", doc.rootDoc().getFields("english_field")[0].stringValue());
}
 

protected Engine.IndexResult indexDoc(IndexShard shard,
                                      String id,
                                      String source,
                                      XContentType xContentType,
                                      String routing) throws IOException {
    SourceToParse sourceToParse = new SourceToParse(
        shard.shardId().getIndexName(), id, new BytesArray(source), xContentType, routing);
    Engine.IndexResult result;
    if (shard.routingEntry().primary()) {
        result = shard.applyIndexOperationOnPrimary(
            Versions.MATCH_ANY,
            VersionType.INTERNAL,
            sourceToParse,
            SequenceNumbers.UNASSIGNED_SEQ_NO,
            0,
            UNSET_AUTO_GENERATED_TIMESTAMP,
            false);
        if (result.getResultType() == Engine.Result.Type.MAPPING_UPDATE_REQUIRED) {
            updateMappings(shard, IndexMetaData.builder(shard.indexSettings().getIndexMetaData())
                .putMapping("default", result.getRequiredMappingUpdate().toString()).build());
            result = shard.applyIndexOperationOnPrimary(
                Versions.MATCH_ANY,
                VersionType.INTERNAL,
                sourceToParse,
                SequenceNumbers.UNASSIGNED_SEQ_NO,
                0,
                UNSET_AUTO_GENERATED_TIMESTAMP,
                false);
        }
        shard.sync(); // advance local checkpoint
        shard.updateLocalCheckpointForShard(shard.routingEntry().allocationId().getId(),
                                            shard.getLocalCheckpoint());
    } else {
        final long seqNo = shard.seqNoStats().getMaxSeqNo() + 1;
        shard.advanceMaxSeqNoOfUpdatesOrDeletes(seqNo); // manually replicate max_seq_no_of_updates
        result = shard.applyIndexOperationOnReplica(seqNo, 0, UNSET_AUTO_GENERATED_TIMESTAMP, false, sourceToParse);
        shard.sync(); // advance local checkpoint
        if (result.getResultType() == Engine.Result.Type.MAPPING_UPDATE_REQUIRED) {
            throw new TransportReplicationAction.RetryOnReplicaException(
                shard.shardId,
                "Mappings are not available on the replica yet, triggered update: " +
                result.getRequiredMappingUpdate());
        }
    }
    return result;
}
 

private Engine.Operation convertToEngineOp(Translog.Operation operation, Engine.Operation.Origin origin) {
    switch (operation.opType()) {
        case INDEX:
            final Translog.Index index = (Translog.Index) operation;
            final String indexName = mapperService.index().getName();
            return IndexShard.prepareIndex(
                docMapper(index.type()),
                new SourceToParse(
                    indexName,
                    index.id(),
                    index.source(),
                    XContentHelper.xContentType(index.source()),
                    index.routing()
                ),
                index.seqNo(),
                index.primaryTerm(),
                index.version(),
                null,
                origin,
                index.getAutoGeneratedIdTimestamp(),
                true,
                SequenceNumbers.UNASSIGNED_SEQ_NO,
                0
            );
        case DELETE:
            final Translog.Delete delete = (Translog.Delete) operation;
            return new Engine.Delete(delete.type(),
                                     delete.id(),
                                     delete.uid(),
                                     delete.seqNo(),
                                     delete.primaryTerm(),
                                     delete.version(),
                                     null,
                                     origin,
                                     System.nanoTime(),
                                     SequenceNumbers.UNASSIGNED_SEQ_NO,
                                     0);
        case NO_OP:
            final Translog.NoOp noOp = (Translog.NoOp) operation;
            return new Engine.NoOp(noOp.seqNo(), noOp.primaryTerm(), origin, System.nanoTime(), noOp.reason());
        default:
            throw new IllegalStateException("No operation defined for [" + operation + "]");
    }
}
 

@Test
public void testSimpleArrayMapping() throws Exception {
    // @formatter:off
    String mapping = Strings.toString(XContentFactory.jsonBuilder()
        .startObject()
            .startObject(TYPE)
                .startObject("properties")
                    .startObject("array_field")
                        .field("type", ArrayMapper.CONTENT_TYPE)
                        .startObject(ArrayMapper.INNER_TYPE)
                            .field("type", "keyword")
                        .endObject()
                    .endObject()
                .endObject()
            .endObject()
        .endObject());
    // @formatter:on
    DocumentMapper mapper = mapper(INDEX, mapping);

    assertThat(mapper.mappers().getMapper("array_field"), is(instanceOf(ArrayMapper.class)));

    BytesReference bytesReference = BytesReference.bytes(JsonXContent.contentBuilder()
        .startObject()
        .array("array_field", "a", "b", "c")
        .endObject());
    SourceToParse sourceToParse = new SourceToParse(INDEX, "abc", bytesReference, XContentType.JSON);
    ParsedDocument doc = mapper.parse(sourceToParse);
    assertThat(doc.dynamicMappingsUpdate() == null, is(true));
    assertThat(doc.docs().size(), is(1));

    ParseContext.Document fields = doc.docs().get(0);
    Set<String> values = uniqueValuesFromFields(fields, "array_field");
    assertThat(values, Matchers.containsInAnyOrder("a", "b", "c"));
    assertThat(
        mapper.mappingSource().string(),
        is("{\"default\":{" +
           "\"properties\":{" +
           "\"array_field\":{" +
           "\"type\":\"array\"," +
           "\"inner\":{" +
           "\"type\":\"keyword\"" +
           "}" +
           "}" +
           "}" +
           "}}"));
}
 

@Override
protected WriteReplicaResult<ShardUpsertRequest> processRequestItemsOnReplica(IndexShard indexShard, ShardUpsertRequest request) throws IOException {
    Translog.Location location = null;
    for (ShardUpsertRequest.Item item : request.items()) {
        if (item.source() == null) {
            if (logger.isTraceEnabled()) {
                logger.trace("[{} (R)] Document with id {}, has no source, primary operation must have failed",
                    indexShard.shardId(), item.id());
            }
            continue;
        }
        SourceToParse sourceToParse = new SourceToParse(
            indexShard.shardId().getIndexName(),
            item.id(),
            item.source(),
            XContentType.JSON
        );

        Engine.IndexResult indexResult = indexShard.applyIndexOperationOnReplica(
            item.seqNo(),
            item.version(),
            Translog.UNSET_AUTO_GENERATED_TIMESTAMP,
            false,
            sourceToParse
        );
        if (indexResult.getResultType() == Engine.Result.Type.MAPPING_UPDATE_REQUIRED) {
            // Even though the primary waits on all nodes to ack the mapping changes to the master
            // (see MappingUpdatedAction.updateMappingOnMaster) we still need to protect against missing mappings
            // and wait for them. The reason is concurrent requests. Request r1 which has new field f triggers a
            // mapping update. Assume that that update is first applied on the primary, and only later on the replica
            // (it’s happening concurrently). Request r2, which now arrives on the primary and which also has the new
            // field f might see the updated mapping (on the primary), and will therefore proceed to be replicated
            // to the replica. When it arrives on the replica, there’s no guarantee that the replica has already
            // applied the new mapping, so there is no other option than to wait.
            throw new TransportReplicationAction.RetryOnReplicaException(indexShard.shardId(),
                "Mappings are not available on the replica yet, triggered update: " + indexResult.getRequiredMappingUpdate());
        }
        location = indexResult.getTranslogLocation();
    }
    return new WriteReplicaResult<>(request, location, null, indexShard, logger);
}