org.elasticsearch.index.mapper.ParseContext Java Examples

private void updateDocs(final Term uid, final List<ParseContext.Document> docs, final IndexWriter indexWriter) throws IOException {
    if (softDeleteEnabled) {
        if (docs.size() > 1) {
            indexWriter.softUpdateDocuments(uid, docs, softDeletesField);
        } else {
            indexWriter.softUpdateDocument(uid, docs.get(0), softDeletesField);
        }
    } else {
        if (docs.size() > 1) {
            indexWriter.updateDocuments(uid, docs);
        } else {
            indexWriter.updateDocument(uid, docs.get(0));
        }
    }
    numDocUpdates.inc(docs.size());
}
 

@Override
protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
    final String value;
    if (context.externalValueSet()) {
        value = context.externalValue().toString();
    } else {
        XContentParser parser = context.parser();
        if (parser.currentToken() == XContentParser.Token.VALUE_NULL) {
            value = fieldType().nullValueAsString();
        } else {
            value = parser.textOrNull();
        }
    }
    if (value == null) {
        return;
    }
    RawCollationKey key = collator.getRawCollationKey(value, null);
    final BytesRef binaryValue = new BytesRef(key.bytes, 0, key.size);
    if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
        Field field = new Field(fieldType().name(), binaryValue, fieldType());
        fields.add(field);
    }
    if (fieldType().hasDocValues()) {
        fields.add(getDVField.apply(fieldType().name(), binaryValue));
    }
}
 

@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException, AlreadyExpiredException {
    if (enabledState.enabled && !context.sourceToParse().flyweight()) {
        long ttl = context.sourceToParse().ttl();
        if (ttl <= 0 && defaultTTL > 0) { // no ttl provided so we use the default value
            ttl = defaultTTL;
            context.sourceToParse().ttl(ttl);
        }
        if (ttl > 0) { // a ttl has been provided either externally or in the _source
            long timestamp = context.sourceToParse().timestamp();
            long expire = new Date(timestamp + ttl).getTime();
            long now = System.currentTimeMillis();
            // there is not point indexing already expired doc
            if (context.sourceToParse().origin() == SourceToParse.Origin.PRIMARY && now >= expire) {
                throw new AlreadyExpiredException(context.index(), context.type(), context.id(), timestamp, ttl, now);
            }
            // the expiration timestamp (timestamp + ttl) is set as field
            fields.add(new LongFieldMapper.CustomLongNumericField(expire, fieldType()));
        }
    }
}
 

@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
    XContentParser parser = context.parser();
    if (parser.currentName() != null && parser.currentName().equals(Defaults.NAME) && parser.currentToken().isValue()) {
        // we are in the parse Phase
        String id = parser.text();
        if (context.id() != null && !context.id().equals(id)) {
            throw new MapperParsingException("Provided id [" + context.id() + "] does not match the content one [" + id + "]");
        }
        context.id(id);
    } // else we are in the pre/post parse phase

    if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
        fields.add(new Field(fieldType().names().indexName(), context.id(), fieldType()));
    }
    if (fieldType().hasDocValues()) {
        fields.add(new BinaryDocValuesField(fieldType().names().indexName(), new BytesRef(context.id())));
    }
}
 

MemoryIndex indexDoc(ParseContext.Document d, Analyzer analyzer, MemoryIndex memoryIndex) {
    for (IndexableField field : d.getFields()) {
        if (field.fieldType().indexOptions() == IndexOptions.NONE && field.name().equals(UidFieldMapper.NAME)) {
            continue;
        }
        try {
            // TODO: instead of passing null here, we can have a CTL<Map<String,TokenStream>> and pass previous,
            // like the indexer does
            try (TokenStream tokenStream = field.tokenStream(analyzer, null)) {
                if (tokenStream != null) {
                    memoryIndex.addField(field.name(), tokenStream, field.boost());
                }
             }
        } catch (IOException e) {
            throw new ElasticsearchException("Failed to create token stream", e);
        }
    }
    return memoryIndex;
}
 

private void updateDocs(final Term uid, final List<ParseContext.Document> docs, final IndexWriter indexWriter) throws IOException {
    if (softDeleteEnabled) {
        if (docs.size() > 1) {
            indexWriter.softUpdateDocuments(uid, docs, softDeletesField);
        } else {
            indexWriter.softUpdateDocument(uid, docs.get(0), softDeletesField);
        }
    } else {
        if (docs.size() > 1) {
            indexWriter.updateDocuments(uid, docs);
        } else {
            indexWriter.updateDocument(uid, docs.get(0));
        }
    }
    numDocUpdates.inc(docs.size());
}
 

private Engine.Index getIndex(final String id) {
    final String type = "test";
    final ParseContext.Document document = new ParseContext.Document();
    document.add(new TextField("test", "test", Field.Store.YES));
    final Field idField = new Field("_id", Uid.encodeId(id), IdFieldMapper.Defaults.FIELD_TYPE);
    final Field versionField = new NumericDocValuesField("_version", Versions.MATCH_ANY);
    final SeqNoFieldMapper.SequenceIDFields seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID();
    document.add(idField);
    document.add(versionField);
    document.add(seqID.seqNo);
    document.add(seqID.seqNoDocValue);
    document.add(seqID.primaryTerm);
    final BytesReference source = new BytesArray(new byte[] { 1 });
    final ParsedDocument doc =
        new ParsedDocument(versionField, seqID, id, type, List.of(document), source, null);
    return new Engine.Index(
        new Term("_id", Uid.encodeId(doc.id())), doc, UNASSIGNED_SEQ_NO, 0,
        Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false, UNASSIGNED_SEQ_NO, 0);

}
 

protected static ParsedDocument testParsedDocument(
    String id, String routing, ParseContext.Document document, BytesReference source, Mapping mappingUpdate,
    boolean recoverySource) {
    Field uidField = new Field("_id", Uid.encodeId(id), IdFieldMapper.Defaults.FIELD_TYPE);
    Field versionField = new NumericDocValuesField("_version", 0);
    SeqNoFieldMapper.SequenceIDFields seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID();
    document.add(uidField);
    document.add(versionField);
    document.add(seqID.seqNo);
    document.add(seqID.seqNoDocValue);
    document.add(seqID.primaryTerm);
    BytesRef ref = source.toBytesRef();
    if (recoverySource) {
        document.add(new StoredField(SourceFieldMapper.RECOVERY_SOURCE_NAME, ref.bytes, ref.offset, ref.length));
        document.add(new NumericDocValuesField(SourceFieldMapper.RECOVERY_SOURCE_NAME, 1));
    } else {
        document.add(new StoredField(SourceFieldMapper.NAME, ref.bytes, ref.offset, ref.length));
    }
    return new ParsedDocument(versionField, seqID, id, routing, Arrays.asList(document), source, mappingUpdate);
}
 

@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
    ValueAndBoost valueAndBoost = parseCreateFieldForString(context, fieldType().nullValueAsString(), fieldType().boost());
    if (valueAndBoost.value() == null) {
        return;
    }
    if (ignoreAbove > 0 && valueAndBoost.value().length() > ignoreAbove) {
        return;
    }
    if (context.includeInAll(includeInAll, this)) {
        context.allEntries().addText(fieldType().names().fullName(), valueAndBoost.value(), valueAndBoost.boost());
    }

    if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
        Field field = new Field(fieldType().names().indexName(), valueAndBoost.value(), fieldType());
        field.setBoost(valueAndBoost.boost());
        fields.add(field);
    }
    if (fieldType().hasDocValues()) {
        fields.add(new SortedSetDocValuesField(fieldType().names().indexName(), new BytesRef(valueAndBoost.value())));
    }
}
 

@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
    if (fieldType().indexOptions() == IndexOptions.NONE && !fieldType().stored() && !fieldType().hasDocValues()) {
        return;
    }

    Boolean value = context.parseExternalValue(Boolean.class);
    if (value == null) {
        XContentParser.Token token = context.parser().currentToken();
        if (token == XContentParser.Token.VALUE_NULL) {
            if (fieldType().nullValue() != null) {
                value = fieldType().nullValue();
            }
        } else {
            value = context.parser().booleanValue();
        }
    }

    if (value == null) {
        return;
    }
    fields.add(new Field(fieldType().names().indexName(), value ? "T" : "F", fieldType()));
    if (fieldType().hasDocValues()) {
        fields.add(new SortedNumericDocValuesField(fieldType().names().indexName(), value ? 1 : 0));
    }
}
 

@Override
protected void parse(ParseContext context, GeoPoint point, String geoHash) throws IOException {
    if (ignoreMalformed.value() == false) {
        if (point.lat() > 90.0 || point.lat() < -90.0) {
            throw new IllegalArgumentException("illegal latitude value [" + point.lat() + "] for " + name());
        }
        if (point.lon() > 180.0 || point.lon() < -180) {
            throw new IllegalArgumentException("illegal longitude value [" + point.lon() + "] for " + name());
        }
    } else {
        // LUCENE WATCH: This will be folded back into Lucene's GeoPointField
        GeoUtils.normalizePoint(point);
    }
    if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
        context.doc().add(new GeoPointField(fieldType().names().indexName(), point.lon(), point.lat(), fieldType() ));
    }
    super.parse(context, point, geoHash);
}
 

@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
    ValueAndBoost valueAndBoost = StringFieldMapper.parseCreateFieldForString(context, null /* Out null value is an int so we convert*/, fieldType().boost());
    if (valueAndBoost.value() == null && fieldType().nullValue() == null) {
        return;
    }

    if (fieldType().indexOptions() != NONE || fieldType().stored() || fieldType().hasDocValues()) {
        int count;
        if (valueAndBoost.value() == null) {
            count = fieldType().nullValue();
        } else {
            count = countPositions(analyzer, simpleName(), valueAndBoost.value());
        }
        addIntegerFields(context, fields, count, valueAndBoost.boost());
    }
}
 

protected void createFieldNamesField(ParseContext context, List<IndexableField> fields) {
    FieldNamesFieldMapper.FieldNamesFieldType fieldNamesFieldType = context.docMapper()
            .metadataMapper(FieldNamesFieldMapper.class).fieldType();
    if (fieldNamesFieldType != null && fieldNamesFieldType.isEnabled()) {
        for (String fieldName : extractFieldNames(fieldType().name())) {
            fields.add(new Field(FieldNamesFieldMapper.NAME, fieldName, fieldNamesFieldType));
        }
    }
}
 

@Override
public void preParse(ParseContext context) throws IOException {
    // if we have the id provided, fill it, and parse now
    if (context.sourceToParse().id() != null) {
        context.id(context.sourceToParse().id());
        super.parse(context);
    }
}
 

@Override
protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
    if (context.externalValueSet()) {
        return;
    }
    XContentParser parser = context.parser();
    if (parser.currentToken() == XContentParser.Token.VALUE_NULL) {
        return;
    }
    String value = fieldType().nullValueAsString();
    if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
        XContentParser.Token token;
        String currentFieldName = null;
        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
            if (token == XContentParser.Token.FIELD_NAME) {
                currentFieldName = parser.currentName();
            } else {
                if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) {
                    value = parser.textOrNull();
                }
            }
        }
    } else {
        value = parser.textOrNull();
    }
    try {
        Collection<CharSequence> stdnums = service.lookup(settings, value);
        for (CharSequence stdnum : stdnums) {
            Field field = new Field(fieldType().name(), stdnum.toString(), fieldType());
            fields.add(field);
        }
    } catch (NumberFormatException e) {
        logger.trace(e.getMessage(), e);
        context.createExternalValueContext("unknown");
    }
}
 

/** Creates instances of the fields that the current field should be copied to */
private static void parseCopyBitsFields(ParseContext context,
        final List<String> copyToFields) throws IOException {
    if (!context.isWithinCopyTo() && copyToFields.isEmpty() == false) {
        context = context.createCopyToContext();
        for (final String field : copyToFields) {
            // In case of a hierarchy of nested documents, we need to figure out
            // which document the field should go to
            ParseContext.Document targetDoc = null;
            for (ParseContext.Document doc = context
                    .doc(); doc != null; doc = doc.getParent()) {
                if (field.startsWith(doc.getPrefix())) {
                    targetDoc = doc;
                    break;
                }
            }
            assert targetDoc != null;
            final ParseContext copyToContext;
            if (targetDoc == context.doc()) {
                copyToContext = context;
            } else {
                copyToContext = context.switchDoc(targetDoc);
            }
            parseCopy(field, copyToContext);
        }
    }
}
 

private void addDocs(final List<ParseContext.Document> docs, final IndexWriter indexWriter) throws IOException {
    if (docs.size() > 1) {
        indexWriter.addDocuments(docs);
    } else {
        indexWriter.addDocument(docs.get(0));
    }
    numDocAppends.inc(docs.size());
}
 

@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
    if (!fieldType().stored() && !fieldType().hasDocValues()) {
        return;
    }
    byte[] value = context.parseExternalValue(byte[].class);
    if (value == null) {
        if (context.parser().currentToken() == XContentParser.Token.VALUE_NULL) {
            return;
        } else {
            value = context.parser().binaryValue();
        }
    }
    if (value == null) {
        return;
    }
    if (fieldType().stored()) {
        fields.add(new Field(fieldType().names().indexName(), value, fieldType()));
    }

    if (fieldType().hasDocValues()) {
        CustomBinaryDocValuesField field = (CustomBinaryDocValuesField) context.doc().getByKey(fieldType().names().indexName());
        if (field == null) {
            field = new CustomBinaryDocValuesField(fieldType().names().indexName(), value);
            context.doc().addWithKey(fieldType().names().indexName(), field);
        } else {
            field.add(value);
        }
    }

}
 

@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
    Field uid = new Field(NAME, Uid.createUid(context.stringBuilder(), context.type(), context.id()), Defaults.FIELD_TYPE);
    context.uid(uid);
    fields.add(uid);
    if (fieldType().hasDocValues()) {
        fields.add(new BinaryDocValuesField(NAME, new BytesRef(uid.stringValue())));
    }
}
 

private static void parseCopyFields(ParseContext originalContext, List<String> copyToFields) throws IOException {
    if (!originalContext.isWithinCopyTo() && !copyToFields.isEmpty()) {
        ParseContext context = originalContext.createCopyToContext();
        for (String field : copyToFields) {
            // In case of a hierarchy of nested documents, we need to figure out
            // which document the field should go to
            ParseContext.Document targetDoc = null;
            for (ParseContext.Document doc = context.doc(); doc != null; doc = doc.getParent()) {
                if (field.startsWith(doc.getPrefix())) {
                    targetDoc = doc;
                    break;
                }
            }
            if (targetDoc == null) {
                throw new IllegalArgumentException("target doc is null");
            }
            final ParseContext copyToContext;
            if (targetDoc == context.doc()) {
                copyToContext = context;
            } else {
                copyToContext = context.switchDoc(targetDoc);
            }
            // simplified - no dynamic field creation
            FieldMapper fieldMapper = copyToContext.docMapper().mappers().getMapper(field);
            if (fieldMapper != null) {
                fieldMapper.parse(copyToContext);
            } else {
                throw new MapperParsingException("attempt to copy value to non-existing field [" + field + "]");
            }
        }
    }
}
 

@Override
public Mapper parse(ParseContext context) throws IOException {
    // no need ot parse here, we either get the routing in the sourceToParse
    // or we don't have routing, if we get it in sourceToParse, we process it in preParse
    // which will always be called
    return null;
}
 

@SuppressWarnings("unchecked")
private static void parseLanguageToFields(ParseContext originalContext, Object languageToFields) throws IOException {
    List<Object> fieldList = languageToFields instanceof List ?
            (List<Object>)languageToFields : Collections.singletonList(languageToFields);
    ParseContext context = originalContext.createCopyToContext();
    for (Object field : fieldList) {
        ParseContext.Document targetDoc = null;
        for (ParseContext.Document doc = context.doc(); doc != null; doc = doc.getParent()) {
            if (field.toString().startsWith(doc.getPrefix())) {
                targetDoc = doc;
                break;
            }
        }
        if (targetDoc == null) {
            throw new IllegalArgumentException("target doc is null");
        }
        final ParseContext copyToContext;
        if (targetDoc == context.doc()) {
            copyToContext = context;
        } else {
            copyToContext = context.switchDoc(targetDoc);
        }
        FieldMapper fieldMapper = copyToContext.docMapper().mappers().getMapper(field.toString());
        if (fieldMapper != null) {
            fieldMapper.parse(copyToContext);
        } else {
            throw new MapperParsingException("attempt to copy value to non-existing field [" + field + "]");
        }
    }
}
 

protected void addIntegerFields(ParseContext context, List<Field> fields, int value, float boost) {
    if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
        CustomIntegerNumericField field = new CustomIntegerNumericField(value, fieldType());
        field.setBoost(boost);
        fields.add(field);
    }
    if (fieldType().hasDocValues()) {
        addDocValue(context, fields, value);
    }
}
 

private Analyzer findAnalyzer(ParseContext context) {
    Analyzer analyzer = fieldType().indexAnalyzer();
    if (analyzer == null) {
        analyzer = context.docMapper().mappers().indexAnalyzer();
        if (analyzer == null) {
            // This should not happen, should we log warn it?
            analyzer = Lucene.STANDARD_ANALYZER;
        }
    }
    return analyzer;
}
 

@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
    if (!enabledState.enabled) {
        return;
    }
    // reset the entries
    context.allEntries().reset();
    Analyzer analyzer = findAnalyzer(context);
    fields.add(new AllField(fieldType().names().indexName(), context.allEntries(), analyzer, fieldType()));
}
 

@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
    if (fieldType().indexOptions() == IndexOptions.NONE && !fieldType().stored()) {
        return;
    }
    fields.add(new Field(fieldType().names().indexName(), context.type(), fieldType()));
    if (fieldType().hasDocValues()) {
        fields.add(new SortedSetDocValuesField(fieldType().names().indexName(), new BytesRef(context.type())));
    }
}
 

public void testSimpleStandardNumber() throws Exception {
    String mapping = copyToStringFromClasspath("mapping.json");
    DocumentMapper docMapper = createIndex("some_index")
            .mapperService().documentMapperParser()
            .parse("someType", new CompressedXContent(mapping));
    String sampleText = "978-3-551-75213-0";
    BytesReference json = BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
            .field("someField", sampleText).endObject());
    SourceToParse sourceToParse = SourceToParse.source("some_index", "someType", "1", json, XContentType.JSON);
    ParseContext.Document doc = docMapper.parse(sourceToParse).rootDoc();
    assertEquals(2, doc.getFields("someField").length);
    assertEquals("978-3-551-75213-0", doc.getFields("someField")[0].stringValue());
    assertEquals("9783551752130", doc.getFields("someField")[1].stringValue());
}
 

@Override
public void postParse(ParseContext context) throws IOException {
    if (context.id() == null && !context.sourceToParse().flyweight()) {
        throw new MapperParsingException("No id found while parsing the content source");
    }
    // it either get built in the preParse phase, or get parsed...
}
 

@Override
public void preParse(ParseContext context) throws IOException {
    if (context.sourceToParse().id() != null) {
        context.id(context.sourceToParse().id());
        super.parse(context);
    }
}
 

private void addStaleDocs(final List<ParseContext.Document> docs, final IndexWriter indexWriter) throws IOException {
    assert softDeleteEnabled : "Add history documents but soft-deletes is disabled";
    for (ParseContext.Document doc : docs) {
        doc.add(softDeletesField); // soft-deleted every document before adding to Lucene
    }
    if (docs.size() > 1) {
        indexWriter.addDocuments(docs);
    } else {
        indexWriter.addDocument(docs.get(0));
    }
}