Java Code Examples for org.apache.lucene.util.TestUtil#randomHtmlishString()

The following examples show how to use org.apache.lucene.util.TestUtil#randomHtmlishString() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestDuelingAnalyzers.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up 
public void testLetterHtmlish() throws Exception {
  Random random = random();
  Analyzer left = new MockAnalyzer(random, jvmLetter, false);
  Analyzer right = new Analyzer() {
    @Override
    protected TokenStreamComponents createComponents(String fieldName) {
      Tokenizer tokenizer = new LetterTokenizer(newAttributeFactory());
      return new TokenStreamComponents(tokenizer, tokenizer);
    }
  };
  for (int i = 0; i < 200; i++) {
    String s = TestUtil.randomHtmlishString(random, 20);
    assertEquals(s, left.tokenStream("foo", newStringReader(s)), 
                 right.tokenStream("foo", newStringReader(s)));
  }
  IOUtils.close(left, right);
}
Example 2
Source File: TestDuelingAnalyzers.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up 
public void testLetterHtmlishHuge() throws Exception {
  Random random = random();
  int maxLength = 1024; // this is number of elements, not chars!
  MockAnalyzer left = new MockAnalyzer(random, jvmLetter, false);
  left.setMaxTokenLength(255); // match CharTokenizer's max token length
  Analyzer right = new Analyzer() {
    @Override
    protected TokenStreamComponents createComponents(String fieldName) {
      Tokenizer tokenizer = new LetterTokenizer(newAttributeFactory());
      return new TokenStreamComponents(tokenizer, tokenizer);
    }
  };
  int numIterations = atLeast(10);
  for (int i = 0; i < numIterations; i++) {
    String s = TestUtil.randomHtmlishString(random, maxLength);
    assertEquals(s, left.tokenStream("foo", newStringReader(s)), 
                 right.tokenStream("foo", newStringReader(s)));
  }
  IOUtils.close(left, right);
}
Example 3
Source File: TestMockAnalyzer.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up 
public void testForwardOffsets() throws Exception {
  int num = atLeast(1000);
  for (int i = 0; i < num; i++) {
    String s = TestUtil.randomHtmlishString(random(), 20);
    StringReader reader = new StringReader(s);
    MockCharFilter charfilter = new MockCharFilter(reader, 2);
    MockAnalyzer analyzer = new MockAnalyzer(random());
    try (TokenStream ts = analyzer.tokenStream("bogus", charfilter)) {
      ts.reset();
      while (ts.incrementToken()) {
        ;
      }
      ts.end();
    }
  }
}
Example 4
Source File: HTMLStripCharFilterTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up 
public void testRandomBrokenHTML() throws Exception {
  int maxNumElements = 10000;
  String text = TestUtil.randomHtmlishString(random(), maxNumElements);
  Analyzer a = newTestAnalyzer();
  checkAnalysisConsistency(random(), a, random().nextBoolean(), text);
  a.close();
}