org.elasticsearch.index.mapper.internal.TTLFieldMapper Java Examples

private void purgeShards(List<IndexShard> shardsToPurge) {
    for (IndexShard shardToPurge : shardsToPurge) {
        Query query = shardToPurge.indexService().mapperService().smartNameFieldType(TTLFieldMapper.NAME).rangeQuery(null, System.currentTimeMillis(), false, true);
        Engine.Searcher searcher = shardToPurge.acquireSearcher("indices_ttl");
        try {
            logger.debug("[{}][{}] purging shard", shardToPurge.routingEntry().index(), shardToPurge.routingEntry().id());
            ExpiredDocsCollector expiredDocsCollector = new ExpiredDocsCollector();
            searcher.searcher().search(query, expiredDocsCollector);
            List<DocToPurge> docsToPurge = expiredDocsCollector.getDocsToPurge();

            BulkRequest bulkRequest = new BulkRequest();
            for (DocToPurge docToPurge : docsToPurge) {

                bulkRequest.add(new DeleteRequest().index(shardToPurge.routingEntry().index()).type(docToPurge.type).id(docToPurge.id).version(docToPurge.version).routing(docToPurge.routing));
                bulkRequest = processBulkIfNeeded(bulkRequest, false);
            }
            processBulkIfNeeded(bulkRequest, true);
        } catch (Exception e) {
            logger.warn("failed to purge", e);
        } finally {
            searcher.close();
        }
    }
}
 

/**
 * Returns the shards to purge, i.e. the local started primary shards that have ttl enabled and disable_purge to false
 */
private List<IndexShard> getShardsToPurge() {
    List<IndexShard> shardsToPurge = new ArrayList<>();
    MetaData metaData = clusterService.state().metaData();
    for (IndexService indexService : indicesService) {
        // check the value of disable_purge for this index
        IndexMetaData indexMetaData = metaData.index(indexService.index().name());
        if (indexMetaData == null) {
            continue;
        }
        boolean disablePurge = indexMetaData.getSettings().getAsBoolean(INDEX_TTL_DISABLE_PURGE, false);
        if (disablePurge) {
            continue;
        }

        // check if ttl is enabled for at least one type of this index
        boolean hasTTLEnabled = false;
        for (String type : indexService.mapperService().types()) {
            DocumentMapper documentType = indexService.mapperService().documentMapper(type);
            if (documentType.TTLFieldMapper().enabled()) {
                hasTTLEnabled = true;
                break;
            }
        }
        if (hasTTLEnabled) {
            for (IndexShard indexShard : indexService) {
                if (indexShard.state() == IndexShardState.STARTED && indexShard.routingEntry().primary() && indexShard.routingEntry().started()) {
                    shardsToPurge.add(indexShard);
                }
            }
        }
    }
    return shardsToPurge;
}
 

/**
 * Prepares an update request by converting it into an index or delete request or an update response (no action).
 */
@SuppressWarnings("unchecked")
public Result prepare(UpdateRequest request, IndexShard indexShard) {
    final GetResult getResult = indexShard.getService().get(request.type(), request.id(),
            new String[]{RoutingFieldMapper.NAME, ParentFieldMapper.NAME, TTLFieldMapper.NAME, TimestampFieldMapper.NAME},
            true, request.version(), request.versionType(), FetchSourceContext.FETCH_SOURCE, false);
    return prepare(request, getResult);
}
 

private static Map<String, MetadataFieldMapper.TypeParser> registerBuiltInMetadataMappers() {
    Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers = new LinkedHashMap<>();
    metadataMapperParsers.put(UidFieldMapper.NAME, new UidFieldMapper.TypeParser());
    metadataMapperParsers.put(IdFieldMapper.NAME, new IdFieldMapper.TypeParser());
    metadataMapperParsers.put(RoutingFieldMapper.NAME, new RoutingFieldMapper.TypeParser());
    metadataMapperParsers.put(IndexFieldMapper.NAME, new IndexFieldMapper.TypeParser());
    metadataMapperParsers.put(SourceFieldMapper.NAME, new SourceFieldMapper.TypeParser());
    metadataMapperParsers.put(TypeFieldMapper.NAME, new TypeFieldMapper.TypeParser());
    metadataMapperParsers.put(AllFieldMapper.NAME, new AllFieldMapper.TypeParser());
    metadataMapperParsers.put(TimestampFieldMapper.NAME, new TimestampFieldMapper.TypeParser());
    metadataMapperParsers.put(TTLFieldMapper.NAME, new TTLFieldMapper.TypeParser());
    metadataMapperParsers.put(VersionFieldMapper.NAME, new VersionFieldMapper.TypeParser());
    metadataMapperParsers.put(ParentFieldMapper.NAME, new ParentFieldMapper.TypeParser());
    return metadataMapperParsers;
}
 

/**
 * Prepares an update request by converting it into an index request.
 * <p/>
 * TODO: detect a NOOP and return an update response if true
 */
@SuppressWarnings("unchecked")
private SourceAndVersion prepareUpdate(DocTableInfo tableInfo,
                                       ShardUpsertRequest request,
                                       ShardUpsertRequest.Item item,
                                       IndexShard indexShard) throws ElasticsearchException {
    final GetResult getResult = indexShard.getService().get(request.type(), item.id(),
            new String[]{RoutingFieldMapper.NAME, ParentFieldMapper.NAME, TTLFieldMapper.NAME},
            true, Versions.MATCH_ANY, VersionType.INTERNAL, FetchSourceContext.FETCH_SOURCE, false);

    if (!getResult.isExists()) {
        throw new DocumentMissingException(new ShardId(request.index(), request.shardId().id()), request.type(), item.id());
    }

    if (getResult.internalSourceRef() == null) {
        // no source, we can't do nothing, through a failure...
        throw new DocumentSourceMissingException(new ShardId(request.index(), request.shardId().id()), request.type(), item.id());
    }

    if (item.version() != Versions.MATCH_ANY && item.version() != getResult.getVersion()) {
        throw new VersionConflictEngineException(
                indexShard.shardId(), Constants.DEFAULT_MAPPING_TYPE, item.id(), getResult.getVersion(), item.version());
    }

    Tuple<XContentType, Map<String, Object>> sourceAndContent = XContentHelper.convertToMap(getResult.internalSourceRef(), true);
    final Map<String, Object> updatedSourceAsMap;
    final XContentType updateSourceContentType = sourceAndContent.v1();

    updatedSourceAsMap = sourceAndContent.v2();

    SymbolToFieldExtractorContext ctx = new SymbolToFieldExtractorContext(functions, item.insertValues());

    Map<String, Object> pathsToUpdate = new LinkedHashMap<>();
    Map<String, Object> updatedGeneratedColumns = new LinkedHashMap<>();
    for (int i = 0; i < request.updateColumns().length; i++) {
        /**
         * NOTE: mapping isn't applied. So if an Insert was done using the ES Rest Endpoint
         * the data might be returned in the wrong format (date as string instead of long)
         */
        String columnPath = request.updateColumns()[i];
        Object value = SYMBOL_TO_FIELD_EXTRACTOR.convert(item.updateAssignments()[i], ctx).apply(getResult);
        ReferenceInfo referenceInfo = tableInfo.getReferenceInfo(ColumnIdent.fromPath(columnPath));
        if (referenceInfo instanceof GeneratedReferenceInfo) {
            updatedGeneratedColumns.put(columnPath, value);

        } else {
            pathsToUpdate.put(columnPath, value);
        }
    }

    processGeneratedColumns(tableInfo, pathsToUpdate, updatedGeneratedColumns, request.validateGeneratedColumns(), getResult);

    updateSourceByPaths(updatedSourceAsMap, pathsToUpdate);

    try {
        XContentBuilder builder = XContentFactory.contentBuilder(updateSourceContentType);
        builder.map(updatedSourceAsMap);
        return new SourceAndVersion(builder.bytes(), getResult.getVersion());
    } catch (IOException e) {
        throw new ElasticsearchGenerationException("Failed to generate [" + updatedSourceAsMap + "]", e);
    }
}
 

public TTLFieldMapper TTLFieldMapper() {
    return metadataMapper(TTLFieldMapper.class);
}