org.elasticsearch.index.fielddata.IndexFieldData Java Examples

/**
 * Instantiates a new reusable {@link BytesRefTermStream} based on the field type.
 */
public static BytesRefTermStream get(IndexReader reader, IndexFieldData indexFieldData) {
  if (indexFieldData instanceof IndexNumericFieldData) {
    IndexNumericFieldData numFieldData = (IndexNumericFieldData) indexFieldData;
    switch (numFieldData.getNumericType()) {

      case INT:
        return new IntegerBytesRefTermStream(reader, numFieldData);

      case LONG:
        return new LongBytesRefTermStream(reader, numFieldData);

      default:
        throw new UnsupportedOperationException("Streaming numeric type '" + numFieldData.getNumericType().name() + "' is unsupported");

    }
  }
  else {
    return new BytesBytesRefTermStream(reader, indexFieldData);
  }
}
 

@Override
public IndexFieldData<?> build(Index index, Settings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache,
                               CircuitBreakerService breakerService, MapperService mapperService) {
    // Ignore Circuit Breaker
    final Names fieldNames = fieldType.names();
    final Settings fdSettings = fieldType.fieldDataType().getSettings();
    final Map<String, Settings> filter = fdSettings.getGroups("filter");
    if (filter != null && !filter.isEmpty()) {
        throw new IllegalArgumentException("Doc values field data doesn't support filters [" + fieldNames.fullName() + "]");
    }

    if (BINARY_INDEX_FIELD_NAMES.contains(fieldNames.indexName())) {
        assert numericType == null;
        return new BinaryDVIndexFieldData(index, fieldNames, fieldType.fieldDataType());
    } else if (numericType != null) {
        if (TimestampFieldMapper.NAME.equals(fieldNames.indexName())
                || Version.indexCreated(indexSettings).onOrAfter(Version.V_1_4_0_Beta1)) {
            return new SortedNumericDVIndexFieldData(index, fieldNames, numericType, fieldType.fieldDataType());
        } else {
            // prior to ES 1.4: multi-valued numerics were boxed inside a byte[] as BINARY
            return new BinaryDVNumericIndexFieldData(index, fieldNames, numericType, fieldType.fieldDataType());
        }
    } else {
        return new SortedSetDVOrdinalsIndexFieldData(index, cache, indexSettings, fieldNames, breakerService, fieldType.fieldDataType());
    }
}
 

private TermsCollector getTermsCollector(TermsByQueryRequest.TermsEncoding termsEncoding,
                                         IndexFieldData indexFieldData, SearchContext context) {
  switch (termsEncoding) {
    case LONG:
      return new LongTermsCollector(indexFieldData, context, breakerService.getBreaker(CircuitBreaker.REQUEST));
    case INTEGER:
      return new IntegerTermsCollector(indexFieldData, context, breakerService.getBreaker(CircuitBreaker.REQUEST));
    case BLOOM:
      return new BloomFilterTermsCollector(indexFieldData, context, breakerService.getBreaker(CircuitBreaker.REQUEST));
    case BYTES:
      return new BytesRefTermsCollector(indexFieldData, context, breakerService.getBreaker(CircuitBreaker.REQUEST));
    default:
      throw new IllegalArgumentException("[termsByQuery] Invalid terms encoding: " + termsEncoding.name());
  }
}
 

private ValuesSource bytesField(ValuesSourceConfig<?> config) throws IOException {
    final IndexFieldData<?> indexFieldData = config.fieldContext.indexFieldData();
    ValuesSource dataSource;
    if (indexFieldData instanceof ParentChildIndexFieldData) {
        dataSource = new ValuesSource.Bytes.WithOrdinals.ParentChild((ParentChildIndexFieldData) indexFieldData);
    } else if (indexFieldData instanceof IndexOrdinalsFieldData) {
        dataSource = new ValuesSource.Bytes.WithOrdinals.FieldData((IndexOrdinalsFieldData) indexFieldData);
    } else {
        dataSource = new ValuesSource.Bytes.FieldData(indexFieldData);
    }
    if (config.script != null) {
        dataSource = new ValuesSource.WithScript(dataSource, config.script);
    }
    return dataSource;
}
 

private final Query toFieldDataTermsQuery(MappedFieldType fieldType, IndexFieldData fieldData,
                                          byte[] encodedTerms, long cacheKey) {
  Query query = null;

  if (fieldType instanceof NumberFieldMapper.NumberFieldType) {
    query = FieldDataTermsQuery.newLongs(encodedTerms, (IndexNumericFieldData) fieldData, cacheKey);
  } else if (fieldType instanceof StringFieldMapper.StringFieldType) {
    query = FieldDataTermsQuery.newBytes(encodedTerms, fieldData, cacheKey);
  } else {
    throw new ElasticsearchParseException("[fielddata_terms] query does not support field data type " + fieldType.fieldDataType().getType());
  }

  return query;
}
 

/**
 * generate a SortField from a Reference symbol.
 *
 * the implementation is similar to what {@link org.elasticsearch.search.sort.SortParseElement}
 * does.
 */
@Override
public SortField visitReference(final Reference symbol, final SortSymbolContext context) {
    // can't use the SortField(fieldName, type) constructor
    // because values are saved using docValues and therefore they're indexed in lucene as binary and not
    // with the reference valueType.
    // this is why we use a custom comparator source with the same logic as ES

    ColumnIdent columnIdent = symbol.info().ident().columnIdent();

    if (columnIdent.isColumn()) {
        if (SortParseElement.SCORE_FIELD_NAME.equals(columnIdent.name())) {
            return !context.reverseFlag ? SORT_SCORE_REVERSE : SortParseElement.SORT_SCORE;
        } else if (DocSysColumns.RAW.equals(columnIdent) || DocSysColumns.ID.equals(columnIdent)) {
            return customSortField(DocSysColumns.nameForLucene(columnIdent), symbol, context,
                    LUCENE_TYPE_MAP.get(symbol.valueType()), false);
        }
    }

    MultiValueMode sortMode = context.reverseFlag ? MultiValueMode.MAX : MultiValueMode.MIN;

    String indexName;
    IndexFieldData.XFieldComparatorSource fieldComparatorSource;
    MappedFieldType fieldType = context.context.mapperService().smartNameFieldType(columnIdent.fqn());
    if (fieldType == null){
        indexName = columnIdent.fqn();
        fieldComparatorSource = new NullFieldComparatorSource(LUCENE_TYPE_MAP.get(symbol.valueType()), context.reverseFlag, context.nullFirst);
    } else {
        indexName = fieldType.names().indexName();
        fieldComparatorSource = context.context.fieldData()
                .getForField(fieldType)
                .comparatorSource(SortOrder.missing(context.reverseFlag, context.nullFirst), sortMode, null);
    }
    return new SortField(
            indexName,
            fieldComparatorSource,
            context.reverseFlag
    );
}
 

/**
 * Instantiates a new reusable {@link NumericTermStream} based on the field type.
 */
public static NumericTermStream get(IndexReader reader, IndexFieldData indexFieldData) {
  if (indexFieldData instanceof IndexNumericFieldData) {
    IndexNumericFieldData numFieldData = (IndexNumericFieldData) indexFieldData;
    if (!numFieldData.getNumericType().isFloatingPoint()) {
      return new LongTermStream(reader, numFieldData);
    }
    else {
      throw new UnsupportedOperationException("Streaming floating points is unsupported");
    }
  }
  else {
    return new HashTermStream(reader, indexFieldData);
  }
}
 

@Override
public IndexFieldData<?> build(Index index, Settings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache,
                               CircuitBreakerService breakerService, MapperService mapperService) {
    // Ignore breaker
    return new GeoPointDVIndexFieldData(index, fieldType.names(), fieldType.fieldDataType(),
            Version.indexCreated(indexSettings).before(Version.V_2_2_0));
}
 

@Override
public IndexFieldData<?> build(Index index, Settings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache,
                               CircuitBreakerService breakerService, MapperService mapperService) {
    // Ignore breaker
    final Names fieldNames = fieldType.names();
    return new BytesBinaryDVIndexFieldData(index, fieldNames, fieldType.fieldDataType());
}
 

/**
 * Creates a RandomScoreFunction.
 *
 * @param seed A seed for randomness
 * @param salt A value to salt the seed with, ideally unique to the running node/index
 * @param uidFieldData The field data for _uid to use for generating consistent random values for the same id
 */
public RandomScoreFunction(int seed, int salt, IndexFieldData<?> uidFieldData) {
    super(CombineFunction.MULT);
    this.originalSeed = seed;
    this.saltedSeed = seed ^ salt;
    this.uidFieldData = uidFieldData;
    if (uidFieldData == null) throw new NullPointerException("uid missing");
}
 

protected LongBytesRefTermStream(IndexReader reader, IndexFieldData indexFieldData) {
  super(reader, indexFieldData);
}
 

public IntegerTermsCollector(final IndexFieldData indexFieldData, final SearchContext context,
                             final CircuitBreaker breaker) {
  super(indexFieldData, context, breaker);
}
 

public TermsCollector(final IndexFieldData indexFieldData, final SearchContext context,
                      final CircuitBreaker breaker) {
  this.indexFieldData = indexFieldData;
  this.context = context;
  this.breaker = breaker;
}
 

protected BytesRefTermStream(IndexReader reader, IndexFieldData indexFieldData) {
  super(reader);
  this.fieldData = indexFieldData;
}
 

protected BytesBytesRefTermStream(IndexReader reader, IndexFieldData indexFieldData) {
  super(reader, indexFieldData);
}
 

protected NumericBytesRefTermStream(IndexReader reader, IndexFieldData indexFieldData) {
  super(reader, indexFieldData);
}
 

protected IntegerBytesRefTermStream(IndexReader reader, IndexFieldData indexFieldData) {
  super(reader, indexFieldData);
}
 

@Override
public Query parse(QueryParseContext parseContext) throws IOException, QueryParsingException {
  XContentParser parser = parseContext.parser();

  XContentParser.Token token = parser.nextToken();
  if (token != XContentParser.Token.FIELD_NAME) {
      throw new QueryParsingException(parseContext, "[fielddata_terms] a field name is required");
  }
  String fieldName = parser.currentName();

  String queryName = null;
  byte[] value = null;
  Long cacheKey = null;

  token = parser.nextToken();
  if (token == XContentParser.Token.START_OBJECT) {
    String currentFieldName = null;
    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
      if (token == XContentParser.Token.FIELD_NAME) {
        currentFieldName = parser.currentName();
      } else {
        if ("value".equals(currentFieldName)) {
          value = parser.binaryValue();
        } else if ("_name".equals(currentFieldName)) {
          queryName = parser.text();
        } else if ("_cache_key".equals(currentFieldName) || "_cacheKey".equals(currentFieldName)) {
          cacheKey = parser.longValue();
        } else {
          throw new QueryParsingException(parseContext, "[fielddata_terms] filter does not support [" + currentFieldName + "]");
        }
      }
    }
    parser.nextToken();
  } else {
    value = parser.binaryValue();
    // move to the next token
    parser.nextToken();
  }

  if (value == null) {
    throw new QueryParsingException(parseContext, "[fielddata_terms] a binary value is required");
  }
  if (cacheKey == null) { // cache key is mandatory - see #170
    throw new QueryParsingException(parseContext, "[fielddata_terms] a cache key is required");
  }

  if (fieldName == null) {
    throw new QueryParsingException(parseContext, "[fielddata_terms] a field name is required");
  }

  MappedFieldType fieldType = parseContext.fieldMapper(fieldName);
  if (fieldType == null) {
    return new MatchNoDocsQuery();
  }

  IndexFieldData fieldData = parseContext.getForField(fieldType);
  Query query = this.toFieldDataTermsQuery(fieldType, fieldData, value, cacheKey);

  if (queryName != null) {
    parseContext.addNamedQuery(queryName, query);
  }

  return query;
}
 

protected HashTermStream(IndexReader reader, IndexFieldData fieldData) {
  super(reader);
  this.fieldData = fieldData;
}
 

public NumericTermsCollector(final IndexFieldData indexFieldData, final SearchContext context,
                             final CircuitBreaker breaker) {
  super(indexFieldData, context, breaker);
}
 

public BytesRefTermsCollector(final IndexFieldData indexFieldData, final SearchContext context,
                              final CircuitBreaker breaker) {
  super(indexFieldData, context, breaker);
}
 

public LongTermsCollector(final IndexFieldData indexFieldData, final SearchContext context,
                          final CircuitBreaker breaker) {
  super(indexFieldData, context, breaker);
}
 

@Override
public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) {
    failIfNoDocValues();
    return new BytesBinaryDVIndexFieldData.Builder();
}
 

@Override
public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) {
    failIfNoDocValues();
    return new DocValuesIndexFieldData.Builder();
}
 

public BloomFilterTermsCollector(final IndexFieldData indexFieldData, final SearchContext context,
                                 final CircuitBreaker breaker) {
  super(indexFieldData, context, breaker);
}
 

/**
 * The operation that executes the query and generates a {@link TermsByQueryShardResponse} for each shard.
 */
@Override
protected TermsByQueryShardResponse shardOperation(TermsByQueryShardRequest shardRequest) throws ElasticsearchException {
  IndexService indexService = indicesService.indexServiceSafe(shardRequest.shardId().getIndex());
  IndexShard indexShard = indexService.shardSafe(shardRequest.shardId().id());
  TermsByQueryRequest request = shardRequest.request();
  OrderByShardOperation orderByOperation = OrderByShardOperation.get(request.getOrderBy(), request.maxTermsPerShard());

  SearchShardTarget shardTarget = new SearchShardTarget(clusterService.localNode().id(),
                                                        shardRequest.shardId().getIndex(),
                                                        shardRequest.shardId().id());

  ShardSearchRequest shardSearchRequest = new ShardSearchLocalRequest(request.types(), request.nowInMillis(),
                                                                      shardRequest.filteringAliases());

  SearchContext context = new DefaultSearchContext(0, shardSearchRequest, shardTarget,
    indexShard.acquireSearcher("termsByQuery"), indexService, indexShard, scriptService,
    pageCacheRecycler, bigArrays, threadPool.estimatedTimeInMillisCounter(), parseFieldMatcher,
    SearchService.NO_TIMEOUT);
  SearchContext.setCurrent(context);

  try {
    MappedFieldType fieldType = context.smartNameFieldType(request.field());
    if (fieldType == null) {
      throw new SearchContextException(context, "[termsByQuery] field '" + request.field() +
              "' not found for types " + Arrays.toString(request.types()));
    }

    IndexFieldData indexFieldData = context.fieldData().getForField(fieldType);

    BytesReference querySource = request.querySource();
    if (querySource != null && querySource.length() > 0) {
      XContentParser queryParser = null;
      try {
        queryParser = XContentFactory.xContent(querySource).createParser(querySource);
        QueryParseContext.setTypes(request.types());
        ParsedQuery parsedQuery = orderByOperation.getParsedQuery(queryParser, indexService);
        if (parsedQuery != null) {
          context.parsedQuery(parsedQuery);
        }
      }
      finally {
        QueryParseContext.removeTypes();
        if (queryParser != null) {
          queryParser.close();
        }
      }
    }

    context.preProcess();

    // execute the search only gathering the hit count and bitset for each segment
    logger.debug("{}: Executes search for collecting terms {}", Thread.currentThread().getName(),
      shardRequest.shardId());

    TermsCollector termsCollector = this.getTermsCollector(request.termsEncoding(), indexFieldData, context);
    if (request.expectedTerms() != null) termsCollector.setExpectedTerms(request.expectedTerms());
    if (request.maxTermsPerShard() != null) termsCollector.setMaxTerms(request.maxTermsPerShard());
    HitStream hitStream = orderByOperation.getHitStream(context);
    TermsSet terms = termsCollector.collect(hitStream);

    logger.debug("{}: Returns terms response with {} terms for shard {}", Thread.currentThread().getName(),
      terms.size(), shardRequest.shardId());

    return new TermsByQueryShardResponse(shardRequest.shardId(), terms);
  }
  catch (Throwable e) {
    logger.error("[termsByQuery] Error executing shard operation", e);
    throw new QueryPhaseExecutionException(context, "[termsByQuery] Failed to execute query", e);
  }
  finally {
    // this will also release the index searcher
    context.close();
    SearchContext.removeCurrent();
  }
}
 

/**
 * Creates a new {@link FieldDataTermsQuery} from the given field data.
 */
public FieldDataTermsQuery(final byte[] encodedTerms, final IndexFieldData fieldData, final long cacheKey) {
  this.encodedTerms = encodedTerms;
  this.fieldData = fieldData;
  this.cacheKey = cacheKey;
}
 

@Override
public IndexFieldData<?> build(Index index, Settings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache,
                               CircuitBreakerService breakerService, MapperService mapperService) {
    return new GeoPointArrayIndexFieldData(index, indexSettings, fieldType.names(), fieldType.fieldDataType(), cache,
            breakerService);
}
 

@Override
public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) {
    return new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
}
 

@Override
public IndexFieldData<?> build(Index index, Settings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache,
        CircuitBreakerService breakerService, MapperService mapperService) {
    return new IndexIndexFieldData(index, fieldType.names());
}