org.apache.lucene.util.CharsRef Java Examples

@Override
public CharsRef add(CharsRef prefix, CharsRef output) {
  assert prefix != null;
  assert output != null;
  if (prefix == NO_OUTPUT) {
    return output;
  } else if (output == NO_OUTPUT) {
    return prefix;
  } else {
    assert prefix.length > 0;
    assert output.length > 0;
    CharsRef result = new CharsRef(prefix.length + output.length);
    System.arraycopy(prefix.chars, prefix.offset, result.chars, 0, prefix.length);
    System.arraycopy(output.chars, output.offset, result.chars, prefix.length, output.length);
    result.length = prefix.length + output.length;
    return result;
  }
}
 

private FST<CharsRef> parseConversions(LineNumberReader reader, int num) throws IOException, ParseException {
  Map<String,String> mappings = new TreeMap<>();
  
  for (int i = 0; i < num; i++) {
    String line = reader.readLine();
    String parts[] = line.split("\\s+");
    if (parts.length != 3) {
      throw new ParseException("invalid syntax: " + line, reader.getLineNumber());
    }
    if (mappings.put(parts[1], parts[2]) != null) {
      throw new IllegalStateException("duplicate mapping specified for: " + parts[1]);
    }
  }
  
  Outputs<CharsRef> outputs = CharSequenceOutputs.getSingleton();
  FSTCompiler<CharsRef> fstCompiler = new FSTCompiler<>(FST.INPUT_TYPE.BYTE2, outputs);
  IntsRefBuilder scratchInts = new IntsRefBuilder();
  for (Map.Entry<String,String> entry : mappings.entrySet()) {
    Util.toUTF16(entry.getKey(), scratchInts);
    fstCompiler.add(scratchInts.get(), new CharsRef(entry.getValue()));
  }
  
  return fstCompiler.compile();
}
 

/**
 * Find the unique stem(s) of the provided word
 * 
 * @param word Word to find the stems for
 * @return List of stems for the word
 */
public List<CharsRef> uniqueStems(char word[], int length) {
  List<CharsRef> stems = stem(word, length);
  if (stems.size() < 2) {
    return stems;
  }
  CharArraySet terms = new CharArraySet(8, dictionary.ignoreCase);
  List<CharsRef> deduped = new ArrayList<>();
  for (CharsRef s : stems) {
    if (!terms.contains(s)) {
      deduped.add(s);
      terms.add(s);
    }
  }
  return deduped;
}
 

/** Builds the NormalizeCharMap; call this once you
 *  are done calling {@link #add}. */
public NormalizeCharMap build() {

  final FST<CharsRef> map;
  try {
    final Outputs<CharsRef> outputs = CharSequenceOutputs.getSingleton();
    final FSTCompiler<CharsRef> fstCompiler = new FSTCompiler<>(FST.INPUT_TYPE.BYTE2, outputs);
    final IntsRefBuilder scratch = new IntsRefBuilder();
    for(Map.Entry<String,String> ent : pendingPairs.entrySet()) {
      fstCompiler.add(Util.toUTF16(ent.getKey(), scratch),
                  new CharsRef(ent.getValue()));
    }
    map = fstCompiler.compile();
    pendingPairs.clear();
  } catch (IOException ioe) {
    // Bogus FST IOExceptions!!  (will never happen)
    throw new RuntimeException(ioe);
  }

  return new NormalizeCharMap(map);
}
 

private NormalizeCharMap(FST<CharsRef> map) {
  this.map = map;
  if (map != null) {
    try {
      // Pre-cache root arcs:
      final FST.Arc<CharsRef> scratchArc = new FST.Arc<>();
      final FST.BytesReader fstReader = map.getBytesReader();
      map.getFirstArc(scratchArc);
      if (FST.targetHasArcs(scratchArc)) {
        map.readFirstRealTargetArc(scratchArc.target(), scratchArc, fstReader);
        while(true) {
          assert scratchArc.label() != FST.END_LABEL;
          cachedRootArcs.put(Character.valueOf((char) scratchArc.label()), new FST.Arc<CharsRef>().copyFrom(scratchArc));
          if (scratchArc.isLast()) {
            break;
          }
          map.readNextRealArc(scratchArc, fstReader);
        }
      }
      //System.out.println("cached " + cachedRootArcs.size() + " root arcs");
    } catch (IOException ioe) {
      // Bogus FST IOExceptions!!  (will never happen)
      throw new RuntimeException(ioe);
    }
  }
}
 

private void addInternal(CharsRef synset[], int size) {
  if (size <= 1) {
    return; // nothing to do
  }
  
  if (expand) {
    for (int i = 0; i < size; i++) {
      for (int j = 0; j < size; j++) {
        if (i != j) {
          add(synset[i], synset[j], true);
        }
      }
    }
  } else {
    for (int i = 0; i < size; i++) {
      add(synset[i], synset[0], false);
    }
  }
}
 

private static void recompileAndWalk(String fstFilePath) throws IOException {
  try (InputStreamDataInput in = new InputStreamDataInput(newInputStream(Paths.get(fstFilePath)))) {

    System.out.println("Reading FST");
    long startTimeMs = System.currentTimeMillis();
    FST<CharsRef> originalFst = new FST<>(in, in, CharSequenceOutputs.getSingleton());
    long endTimeMs = System.currentTimeMillis();
    System.out.println("time = " + (endTimeMs - startTimeMs) + " ms");

    for (float oversizingFactor : List.of(0f, 0f, 0f, 1f, 1f, 1f)) {
      System.out.println("\nFST construction (oversizingFactor=" + oversizingFactor + ")");
      startTimeMs = System.currentTimeMillis();
      FST<CharsRef> fst = recompile(originalFst, oversizingFactor);
      endTimeMs = System.currentTimeMillis();
      System.out.println("time = " + (endTimeMs - startTimeMs) + " ms");
      System.out.println("FST RAM = " + fst.ramBytesUsed() + " B");

      System.out.println("FST enum");
      startTimeMs = System.currentTimeMillis();
      walk(fst);
      endTimeMs = System.currentTimeMillis();
      System.out.println("time = " + (endTimeMs - startTimeMs) + " ms");
    }
  }
}
 

@Override
public CharsRef subtract(CharsRef output, CharsRef inc) {
  assert output != null;
  assert inc != null;
  if (inc == NO_OUTPUT) {
    // no prefix removed
    return output;
  } else if (inc.length == output.length) {
    // entire output removed
    return NO_OUTPUT;
  } else {
    assert inc.length < output.length: "inc.length=" + inc.length + " vs output.length=" + output.length;
    assert inc.length > 0;
    return new CharsRef(output.chars, output.offset + inc.length, output.length-inc.length);
  }
}
 

/** only used for asserting! */
private boolean hasHoles(CharsRef chars) {
  final int end = chars.offset + chars.length;
  for(int idx=chars.offset+1;idx<end;idx++) {
    if (chars.chars[idx] == SynonymMap.WORD_SEPARATOR && chars.chars[idx-1] == SynonymMap.WORD_SEPARATOR) {
      return true;
    }
  }
  if (chars.chars[chars.offset] == '\u0000') {
    return true;
  }
  if (chars.chars[chars.offset + chars.length - 1] == '\u0000') {
    return true;
  }

  return false;
}
 

/** Sugar: just joins the provided terms with {@link
 *  SynonymMap#WORD_SEPARATOR}.  reuse and its chars
 *  must not be null. */
public static CharsRef join(String[] words, CharsRefBuilder reuse) {
  int upto = 0;
  char[] buffer = reuse.chars();
  for (String word : words) {
    final int wordLen = word.length();
    final int needed = (0 == upto ? wordLen : 1 + upto + wordLen); // Add 1 for WORD_SEPARATOR
    if (needed > buffer.length) {
      reuse.grow(needed);
      buffer = reuse.chars();
    }
    if (upto > 0) {
      buffer[upto++] = SynonymMap.WORD_SEPARATOR;
    }

    word.getChars(0, wordLen, buffer, upto);
    upto += wordLen;
  }
  reuse.setLength(upto);
  return reuse.get();
}
 

@Test
public void testSeparatorWithSynonyms() throws IOException {
  SynonymMap.Builder builder = new SynonymMap.Builder(true);
  builder.add(new CharsRef("mykeyword"), new CharsRef("mysynonym"), true);
  builder.add(new CharsRef("mykeyword"), new CharsRef("three words synonym"), true);
  Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, true);
  String input = " mykeyword another keyword   ";
  tokenizer.setReader(new StringReader(input));
  SynonymGraphFilter filter = new SynonymGraphFilter(tokenizer, builder.build(), true);
  ConcatenateGraphFilter stream = new ConcatenateGraphFilter(filter, '-', false, 100);
  assertTokenStreamContents(stream, new String[] {
      "mykeyword-another-keyword",
      "mysynonym-another-keyword",
      "three words synonym-another-keyword"
  }, null, null, new int[] { 1, 0 ,0});
}
 

/**
 * Build a minimal, deterministic automaton from a sorted list of {@link BytesRef} representing
 * strings in UTF-8. These strings must be binary-sorted.
 */
public static Automaton build(Collection<BytesRef> input) {
  final DaciukMihovAutomatonBuilder builder = new DaciukMihovAutomatonBuilder();
  
  char[] chars = new char[0];
  CharsRef ref = new CharsRef();
  for (BytesRef b : input) {
    chars = ArrayUtil.grow(chars, b.length);
    final int len = UnicodeUtil.UTF8toUTF16(b, chars);
    ref.chars = chars;
    ref.length = len;
    builder.add(ref);
  }
  
  Automaton.Builder a = new Automaton.Builder();
  convert(a,
      builder.complete(), 
      new IdentityHashMap<State,Integer>());

  return a.finish();
}
 

public void testMaxPosition3WithSynomyms() throws IOException {
  for (final boolean consumeAll : new boolean[]{true, false}) {
    MockTokenizer tokenizer = whitespaceMockTokenizer("one two three four five");
    // if we are consuming all tokens, we can use the checks, otherwise we can't
    tokenizer.setEnableChecks(consumeAll);

    SynonymMap.Builder builder = new SynonymMap.Builder(true);
    builder.add(new CharsRef("one"), new CharsRef("first"), true);
    builder.add(new CharsRef("one"), new CharsRef("alpha"), true);
    builder.add(new CharsRef("one"), new CharsRef("beguine"), true);
    CharsRefBuilder multiWordCharsRef = new CharsRefBuilder();
    SynonymMap.Builder.join(new String[]{"and", "indubitably", "single", "only"}, multiWordCharsRef);
    builder.add(new CharsRef("one"), multiWordCharsRef.get(), true);
    SynonymMap.Builder.join(new String[]{"dopple", "ganger"}, multiWordCharsRef);
    builder.add(new CharsRef("two"), multiWordCharsRef.get(), true);
    SynonymMap synonymMap = builder.build();
    @SuppressWarnings("deprecation")
    TokenStream stream = new SynonymFilter(tokenizer, synonymMap, true);
    stream = new LimitTokenPositionFilter(stream, 3, consumeAll);

    // "only", the 4th word of multi-word synonym "and indubitably single only" is not emitted, since its position is greater than 3.
    assertTokenStreamContents(stream,
        new String[]{"one", "first", "alpha", "beguine", "and", "two", "indubitably", "dopple", "three", "single", "ganger"},
        new int[]{1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0});
  }
}
 

@Override
public CharsRef analyze(String text, CharsRefBuilder reuse) throws IOException {
    try {
        return super.analyze(text, reuse);
    } catch (IllegalArgumentException ex) {
        if (lenient) {
            LOGGER.info("Synonym rule for [" + text + "] was ignored");
            return new CharsRef("");
        } else {
            throw ex;
        }
    }
}
 

@Override
public CharsRef indexedToReadable(BytesRef input, CharsRefBuilder output) {
  final Integer intValue = LegacyNumericUtils.prefixCodedToInt(input);
  final String stringValue = enumMapping.intValueToStringValue(intValue);
  output.grow(stringValue.length());
  output.setLength(stringValue.length());
  stringValue.getChars(0, output.length(), output.chars(), 0);
  return output.get();
}
 

private void addTerms( NamedList<NamedList<Number>> terms, SynonymMap.Builder fieldBuilder, SynonymMap.Builder termBuilder, ArrayList<String> searchFields ) throws IOException {
  TermsResponse termsResponse = new TermsResponse( terms );
  for (String fieldName : searchFields ) {
    CharsRef fieldChars = new CharsRef( fieldName );
    List<TermsResponse.Term> termList = termsResponse.getTerms( fieldName );
    if (termList != null) {
      for (TermsResponse.Term tc : termList) {
        String term = tc.getTerm();
        Log.debug( "Add distributed term: " + fieldName + " = " + term );
        addTerm( fieldChars, term, fieldBuilder, termBuilder );
      }
    }
  }
}
 

private void buildFieldMap( ResponseBuilder rb ) throws IOException {
  Log.debug( "buildFieldMap" );
  SolrIndexSearcher searcher = rb.req.getSearcher();
  // build a synonym map from the SortedDocValues -
  // for each field value: lower case, stemmed, lookup synonyms from synonyms.txt - map to fieldValue
  SynonymMap.Builder fieldBuilder = new SynonymMap.Builder( true );
  SynonymMap.Builder termBuilder = new SynonymMap.Builder( true );
    
  ArrayList<String> searchFields = getStringFields( searcher );

  for (String searchField : searchFields ) {
    Log.debug( "adding searchField " + searchField );
    CharsRef fieldChars = new CharsRef( searchField );
    SortedSetDocValues sdv = FieldCache.DEFAULT.getDocTermOrds( searcher.getAtomicReader( ), searchField );
    if (sdv == null) continue;
    Log.debug( "got SortedSetDocValues for " + searchField );
    TermsEnum te = sdv.termsEnum();
    while (te.next() != null) {
      BytesRef term = te.term();
      String fieldValue = term.utf8ToString( );
      addTerm ( fieldChars, fieldValue, fieldBuilder, termBuilder );
    }
  }
    
  addDistributedTerms( rb, fieldBuilder, termBuilder, searchFields );
    
  fieldMap = fieldBuilder.build( );
  termMap = termBuilder.build( );
}
 

@Override
public void add(CharsRef input, CharsRef output, boolean includeOrig) {
    // This condition follows up on the overridden analyze method. In case lenient was set to true and there was an
    // exception during super.analyze we return a zero-length CharsRef for that word which caused an exception. When
    // the synonym mappings for the words are added using the add method we skip the ones that were left empty by
    // analyze i.e., in the case when lenient is set we only add those combinations which are non-zero-length. The
    // else would happen only in the case when the input or output is empty and lenient is set, in which case we
    // quietly ignore it. For more details on the control-flow see SolrSynonymParser::addInternal.
    if (lenient == false || (input.length > 0 && output.length > 0)) {
        super.add(input, output, includeOrig);
    }
}
 

private void addTerms( NamedList<NamedList<Number>> terms, SynonymMap.Builder fieldBuilder, SynonymMap.Builder termBuilder, ArrayList<String> searchFields ) throws IOException {
  TermsResponse termsResponse = new TermsResponse( terms );
  for (String fieldName : searchFields ) {
    CharsRef fieldChars = new CharsRef( fieldName );
    List<TermsResponse.Term> termList = termsResponse.getTerms( fieldName );
    if (termList != null) {
      for (TermsResponse.Term tc : termList) {
        String term = tc.getTerm();
        Log.debug( "Add distributed term: " + fieldName + " = " + term );
        addTerm( fieldChars, term, fieldBuilder, termBuilder );
      }
    }
  }
}
 

public CharsRef pullNext() {
  assert upto < count;
  lastEndOffset = endOffsets[upto];
  lastPosLength = posLengths[upto];
  final CharsRefBuilder result = outputs[upto++];
  posIncr = 0;
  if (upto == count) {
    reset();
  }
  return result.get();
}
 

/**
 * Add another character sequence to this automaton. The sequence must be
 * lexicographically larger or equal compared to any previous sequences added
 * to this automaton (the input must be sorted).
 */
public void add(CharsRef current) {
  if (current.length > MAX_TERM_LENGTH) {
    throw new IllegalArgumentException("This builder doesn't allow terms that are larger than 1,000 characters, got " + current);
  }
  assert stateRegistry != null : "Automaton already built.";
  assert previous == null
      || comparator.compare(previous, current) <= 0 : "Input must be in sorted UTF-8 order: "
      + previous + " >= " + current;
 

/**
 * Add the managed synonyms and their mappings into the SynonymMap builder.
 */
@Override
public void parse(Reader in) throws IOException, ParseException {
  boolean ignoreCase = synonymManager.getIgnoreCase();
  for (CasePreservedSynonymMappings cpsm : synonymManager.synonymMappings.values()) {
    for (Map.Entry<String, Set<String>> entry : cpsm.mappings.entrySet()) {
      for (String mapping : entry.getValue()) {
        // apply the case setting to match the behavior of the SynonymMap builder
        CharsRef casedTerm = analyze(synonymManager.applyCaseSetting(ignoreCase, entry.getKey()), new CharsRefBuilder());
        CharsRef casedMapping = analyze(synonymManager.applyCaseSetting(ignoreCase, mapping), new CharsRefBuilder());
        add(casedTerm, casedMapping, false);
      }
    }
  }
}
 

/** Convert NamedList (suggester response) to {@link SuggesterResult} */
private SuggesterResult toSuggesterResult(Map<String, SimpleOrderedMap<NamedList<Object>>> suggestionsMap) {
  SuggesterResult result = new SuggesterResult();
  if (suggestionsMap == null) {
    return result;
  }
  // for each token
  for(Map.Entry<String, SimpleOrderedMap<NamedList<Object>>> entry : suggestionsMap.entrySet()) {
    String suggesterName = entry.getKey();
    for (Iterator<Map.Entry<String, NamedList<Object>>> suggestionsIter = entry.getValue().iterator(); suggestionsIter.hasNext();) {
      Map.Entry<String, NamedList<Object>> suggestions = suggestionsIter.next(); 
      String tokenString = suggestions.getKey();
      List<LookupResult> lookupResults = new ArrayList<>();
      NamedList<Object> suggestion = suggestions.getValue();
      // for each suggestion
      for (int j = 0; j < suggestion.size(); j++) {
        String property = suggestion.getName(j);
        if (property.equals(SuggesterResultLabels.SUGGESTIONS)) {
          @SuppressWarnings("unchecked")
          List<NamedList<Object>> suggestionEntries = (List<NamedList<Object>>) suggestion.getVal(j);
          for(NamedList<Object> suggestionEntry : suggestionEntries) {
            String term = (String) suggestionEntry.get(SuggesterResultLabels.SUGGESTION_TERM);
            Long weight = (Long) suggestionEntry.get(SuggesterResultLabels.SUGGESTION_WEIGHT);
            String payload = (String) suggestionEntry.get(SuggesterResultLabels.SUGGESTION_PAYLOAD);
            LookupResult res = new LookupResult(new CharsRef(term), weight, new BytesRef(payload));
            lookupResults.add(res);
          }
        }
        result.add(suggesterName, tokenString, lookupResults);
      }
    }
  }
  return result;
}
 

private static int walk(FST<CharsRef> read) throws IOException {
  IntsRefFSTEnum<CharsRef> fstEnum = new IntsRefFSTEnum<>(read);
  IntsRefFSTEnum.InputOutput<CharsRef> inputOutput;
  int terms = 0;
  while ((inputOutput = fstEnum.next()) != null) {
    terms += inputOutput.input.length;
    terms += inputOutput.output.length;
  }
  return terms;
}
 

private static FST<CharsRef> recompile(FST<CharsRef> fst, float oversizingFactor) throws IOException {
  FSTCompiler<CharsRef> fstCompiler = new FSTCompiler.Builder<>(FST.INPUT_TYPE.BYTE4, CharSequenceOutputs.getSingleton())
      .directAddressingMaxOversizingFactor(oversizingFactor)
      .build();
  IntsRefFSTEnum<CharsRef> fstEnum = new IntsRefFSTEnum<>(fst);
  IntsRefFSTEnum.InputOutput<CharsRef> inputOutput;
  while ((inputOutput = fstEnum.next()) != null) {
    fstCompiler.add(inputOutput.input, CharsRef.deepCopyOf(inputOutput.output));
  }
  return fstCompiler.compile();
}
 

@Override
public CharsRef read(DataInput in) throws IOException {
  final int len = in.readVInt();
  if (len == 0) {
    return NO_OUTPUT;
  } else {
    final CharsRef output = new CharsRef(len);
    for(int idx=0;idx<len;idx++) {
      output.chars[idx] = (char) in.readVInt();
    }
    output.length = len;
    return output;
  }
}
 

@Override
public void write(CharsRef prefix, DataOutput out) throws IOException {
  assert prefix != null;
  out.writeVInt(prefix.length);
  // TODO: maybe UTF8?
  for(int idx=0;idx<prefix.length;idx++) {
    out.writeVInt(prefix.chars[prefix.offset+idx]);
  }
}
 

@Override
public CharsRef common(CharsRef output1, CharsRef output2) {
  assert output1 != null;
  assert output2 != null;

  int pos1 = output1.offset;
  int pos2 = output2.offset;
  int stopAt1 = pos1 + Math.min(output1.length, output2.length);
  while(pos1 < stopAt1) {
    if (output1.chars[pos1] != output2.chars[pos2]) {
      break;
    }
    pos1++;
    pos2++;
  }

  if (pos1 == output1.offset) {
    // no common prefix
    return NO_OUTPUT;
  } else if (pos1 == output1.offset + output1.length) {
    // output1 is a prefix of output2
    return output1;
  } else if (pos2 == output2.offset + output2.length) {
    // output2 is a prefix of output1
    return output2;
  } else {
    return new CharsRef(output1.chars, output1.offset, pos1-output1.offset);
  }
}
 

/**
 * Copy <code>current</code> into an internal buffer.
 */
private boolean setPrevious(CharsRef current) {
  // don't need to copy, once we fix https://issues.apache.org/jira/browse/LUCENE-3277
  // still, called only from assert
  previous = CharsRef.deepCopyOf(current);
  return true;
}
 

@Override
public CharsRef analyze(String text, CharsRefBuilder reuse) throws IOException {
    try {
        return super.analyze(text, reuse);
    } catch (IllegalArgumentException ex) {
        if (lenient) {
            LOGGER.info("Synonym rule for [" + text + "] was ignored");
            return new CharsRef("");
        } else {
            throw ex;
        }
    }
}