Java Code Examples for org.apache.lucene.util.LuceneTestCase#VERBOSE

@Override
public void delete(Path path) throws IOException {

  // Fake but deterministic and hopefully portable like-randomness:
  long hash = state.incrementAndGet() * path.getFileName().hashCode();
  
  if (enabled // test infra disables when it's "really" time to delete after test is done, so it can reclaim temp dirs
      && Files.exists(path) // important that we NOT delay a NoSuchFileException until later
      && path.getFileName().toString().equals(IndexWriter.WRITE_LOCK_NAME) == false // life is particularly difficult if the virus checker hits our lock file
      && (hash % 5) == 1) {
    if (LuceneTestCase.VERBOSE) {
      System.out.println("NOTE: VirusCheckingFS now refusing to delete " + path);
    }
    throw new AccessDeniedException("VirusCheckingFS is randomly refusing to delete file \"" + path + "\"");
  }
  super.delete(path);
}
 

private RandomIndexWriter(Random r, Directory dir, IndexWriterConfig c, boolean closeAnalyzer, boolean useSoftDeletes) throws IOException {
  // TODO: this should be solved in a different way; Random should not be shared (!).
  this.r = new Random(r.nextLong());
  if (useSoftDeletes) {
    c.setSoftDeletesField("___soft_deletes");
    softDeletesRatio = 1.d / (double)1 + r.nextInt(10);
  } else {
    softDeletesRatio = 0d;
  }
  w = mockIndexWriter(dir, c, r);
  config = w.getConfig();
  flushAt = TestUtil.nextInt(r, 10, 1000);
  if (closeAnalyzer) {
    analyzer = w.getAnalyzer();
  } else {
    analyzer = null;
  }
  if (LuceneTestCase.VERBOSE) {
    System.out.println("RIW dir=" + dir);
  }

  // Make sure we sometimes test indices that don't get
  // any forced merges:
  doRandomForceMerge = !(c.getMergePolicy() instanceof NoMergePolicy) && r.nextBoolean();
}
 

private void maybeFlushOrCommit() throws IOException {
  LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
  if (docCount++ == flushAt) {
    if (r.nextBoolean()) {
      flushAllBuffersSequentially();
    } else if (r.nextBoolean()) {
      if (LuceneTestCase.VERBOSE) {
        System.out.println("RIW.add/updateDocument: now doing a flush at docCount=" + docCount);
      }
      w.flush();
    } else {
      if (LuceneTestCase.VERBOSE) {
        System.out.println("RIW.add/updateDocument: now doing a commit at docCount=" + docCount);
      }
      w.commit();
    }
    flushAt += TestUtil.nextInt(r, (int) (flushAtFactor * 10), (int) (flushAtFactor * 1000));
    if (flushAtFactor < 2e6) {
      // gradually but exponentially increase time b/w flushes
      flushAtFactor *= 1.05;
    }
  }
}
 

/**
 * Iterate through the failures list, giving each object a
 * chance to throw an IOE
 */
synchronized void maybeThrowDeterministicException() throws IOException {
  if (failures != null) {
    for(int i = 0; i < failures.size(); i++) {
      try {
        failures.get(i).eval(this);
      } catch (Throwable t) {
        if (LuceneTestCase.VERBOSE) {
          System.out.println("MockDirectoryWrapper: throw exc");
          t.printStackTrace(System.out);
        }
        throw IOUtils.rethrowAlways(t);
      }
    }
  }
}
 

void start() throws Exception {
  if (LuceneTestCase.VERBOSE) System.out.println("Run QueryScorer");
  run();
  if (LuceneTestCase.VERBOSE) System.out.println("Run QueryTermScorer");
  mode = QUERY_TERM;
  run();
}
 

/** Returns an indexwriter that enables the specified test point */
public static IndexWriter mockIndexWriter(Random r, Directory dir, IndexWriterConfig conf, TestPoint testPoint) throws IOException {
  conf.setInfoStream(new TestPointInfoStream(conf.getInfoStream(), testPoint));
  DirectoryReader reader = null;
  if (r.nextBoolean() && DirectoryReader.indexExists(dir) && conf.getOpenMode() != IndexWriterConfig.OpenMode.CREATE) {
    if (LuceneTestCase.VERBOSE) {
      System.out.println("RIW: open writer from reader");
    }
    reader = DirectoryReader.open(dir);
    conf.setIndexCommit(reader.getIndexCommit());
  }

  IndexWriter iw;
  boolean success = false;
  try {
    iw = new IndexWriter(dir, conf) {
      @Override
      protected boolean isEnableTestPoints() {
        return true;
      }
    };
    success = true;
  } finally {
    if (reader != null) {
      if (success) {
        IOUtils.close(reader);
      } else {
        IOUtils.closeWhileHandlingException(reader);
      }
    }
  }
  return iw;
}
 

private void flushAllBuffersSequentially() throws IOException {
  if (LuceneTestCase.VERBOSE) {
    System.out.println("RIW.add/updateDocument: now flushing the largest writer at docCount=" + docCount);
  }
  int threadPoolSize = w.docWriter.perThreadPool.size();
  int numFlushes = Math.min(1, r.nextInt(threadPoolSize+1));
  for (int i = 0; i < numFlushes; i++) {
    if (w.flushNextBuffer() == false) {
      break; // stop once we didn't flush anything
    }
  }
}
 

public DirectoryReader getReader(boolean applyDeletions, boolean writeAllDeletes) throws IOException {
  LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
  getReaderCalled = true;
  if (r.nextInt(20) == 2) {
    doRandomForceMerge();
  }
  if (!applyDeletions || r.nextBoolean()) {
    // if we have soft deletes we can't open from a directory
    if (LuceneTestCase.VERBOSE) {
      System.out.println("RIW.getReader: use NRT reader");
    }
    if (r.nextInt(5) == 1) {
      w.commit();
    }
    return w.getReader(applyDeletions, writeAllDeletes);
  } else {
    if (LuceneTestCase.VERBOSE) {
      System.out.println("RIW.getReader: open new reader");
    }
    w.commit();
    if (r.nextBoolean()) {
      DirectoryReader reader = DirectoryReader.open(w.getDirectory());
      if (config.getSoftDeletesField() != null) {
        return new SoftDeletesDirectoryReaderWrapper(reader, config.getSoftDeletesField());
      } else {
        return reader;
      }
    } else {
      return w.getReader(applyDeletions, writeAllDeletes);
    }
  }
}
 

@Override
public CodecReader wrapForMerge(CodecReader reader) throws IOException {

  // wrap it (e.g. prevent bulk merge etc)
  // TODO: cut this over to FilterCodecReader api, we can explicitly
  // enable/disable bulk merge for portions of the index we want.
  int thingToDo = r.nextInt(7);
  if (thingToDo == 0) {
    // simple no-op FilterReader
    if (LuceneTestCase.VERBOSE) {
      System.out.println("NOTE: MockRandomMergePolicy now swaps in a SlowCodecReaderWrapper for merging reader=" + reader);
    }
    return SlowCodecReaderWrapper.wrap(new FilterLeafReader(new MergeReaderWrapper(reader)) {

      @Override
      public CacheHelper getCoreCacheHelper() {
        return in.getCoreCacheHelper();
      }

      @Override
      public CacheHelper getReaderCacheHelper() {
        return in.getReaderCacheHelper();
      }
    });
  } else if (thingToDo == 1) {
    // renumber fields
    // NOTE: currently this only "blocks" bulk merges just by
    // being a FilterReader. But it might find bugs elsewhere, 
    // and maybe the situation can be improved in the future.
    if (LuceneTestCase.VERBOSE) {
      System.out.println("NOTE: MockRandomMergePolicy now swaps in a MismatchedLeafReader for merging reader=" + reader);
    }
    return SlowCodecReaderWrapper.wrap(new MismatchedLeafReader(new MergeReaderWrapper(reader), r));
  } else {
    // otherwise, reader is unchanged
    return reader;
  }
}
 

private synchronized TokenFilter maybePayload(TokenFilter stream, String fieldName) {
  Integer val = previousMappings.get(fieldName);
  if (val == null) {
    val = -1; // no payloads
    if (LuceneTestCase.rarely(random)) {
      switch(random.nextInt(3)) {
        case 0: val = -1; // no payloads
                break;
        case 1: val = Integer.MAX_VALUE; // variable length payload
                break;
        case 2: val = random.nextInt(12); // fixed length payload
                break;
      }
    }
    if (LuceneTestCase.VERBOSE) {
      if (val == Integer.MAX_VALUE) {
        System.out.println("MockAnalyzer: field=" + fieldName + " gets variable length payloads");
      } else if (val != -1) {
        System.out.println("MockAnalyzer: field=" + fieldName + " gets fixed length=" + val + " payloads");
      }
    }
    previousMappings.put(fieldName, val); // save it so we are consistent for this field
  }
  
  if (val == -1)
    return stream;
  else if (val == Integer.MAX_VALUE)
    return new MockVariableLengthPayloadFilter(random, stream);
  else
    return new MockFixedLengthPayloadFilter(random, stream, val);
}
 

void maybeThrowIOException(String message) throws IOException {
  if (randomState.nextDouble() < randomIOExceptionRate) {
    IOException ioe = new IOException("a random IOException" + (message == null ? "" : " (" + message + ")"));
    if (LuceneTestCase.VERBOSE) {
      System.out.println(Thread.currentThread().getName() + ": MockDirectoryWrapper: now throw random exception" + (message == null ? "" : " (" + message + ")"));
      ioe.printStackTrace(System.out);
    }
    throw ioe;
  }
}
 

void maybeThrowIOExceptionOnOpen(String name) throws IOException {
  if (randomState.nextDouble() < randomIOExceptionRateOnOpen) {
    if (LuceneTestCase.VERBOSE) {
      System.out.println(Thread.currentThread().getName() + ": MockDirectoryWrapper: now throw random exception during open file=" + name);
      new Throwable().printStackTrace(System.out);
    }
    if (allowRandomFileNotFoundException == false || randomState.nextBoolean()) {
      throw new IOException("a random IOException (" + name + ")");
    } else {
      throw randomState.nextBoolean() ? new FileNotFoundException("a random IOException (" + name + ")") : new NoSuchFileException("a random IOException (" + name + ")");
    }
  }
}
 

private IndexOutput maybeThrottle(String name, IndexOutput output) {
  // throttling REALLY slows down tests, so don't do it very often for SOMETIMES.
  if (throttling == Throttling.ALWAYS ||
      (throttling == Throttling.SOMETIMES && randomState.nextInt(200) == 0)) {
    if (LuceneTestCase.VERBOSE) {
      System.out.println("MockDirectoryWrapper: throttling indexOutput (" + name + ")");
    }
    return throttledOutput.newFromDelegate(output);
  } else {
    return output;
  }
}
 

@Override
public synchronized IndexInput openInput(String name, IOContext context) throws IOException {
  maybeThrowDeterministicException();
  maybeThrowIOExceptionOnOpen(name);
  maybeYield();
  if (failOnOpenInput) {
    maybeThrowDeterministicException();
  }
  if (!LuceneTestCase.slowFileExists(in, name)) {
    throw randomState.nextBoolean() ? new FileNotFoundException(name + " in dir=" + in) : new NoSuchFileException(name + " in dir=" + in);
  }

  // cannot open a file for input if it's still open for output.
  if (!allowReadingFilesStillOpenForWrite && openFilesForWrite.contains(name)) {
    throw fillOpenTrace(new AccessDeniedException("MockDirectoryWrapper: file \"" + name + "\" is still open for writing"), name, false);
  }

  IndexInput delegateInput = in.openInput(name, LuceneTestCase.newIOContext(randomState, context));

  final IndexInput ii;
  int randomInt = randomState.nextInt(500);
  if (useSlowOpenClosers && randomInt == 0) {
    if (LuceneTestCase.VERBOSE) {
      System.out.println("MockDirectoryWrapper: using SlowClosingMockIndexInputWrapper for file " + name);
    }
    ii = new SlowClosingMockIndexInputWrapper(this, name, delegateInput);
  } else if (useSlowOpenClosers && randomInt  == 1) { 
    if (LuceneTestCase.VERBOSE) {
      System.out.println("MockDirectoryWrapper: using SlowOpeningMockIndexInputWrapper for file " + name);
    }
    ii = new SlowOpeningMockIndexInputWrapper(this, name, delegateInput);
  } else {
    ii = new MockIndexInputWrapper(this, name, delegateInput, null);
  }
  addFileHandle(ii, name, Handle.Input);
  return ii;
}
 

private void checkDiskFull(byte[] b, int offset, DataInput in, long len) throws IOException {
  long freeSpace = dir.maxSize == 0 ? 0 : dir.maxSize - dir.sizeInBytes();
  long realUsage = 0;

  // Enforce disk full:
  if (dir.maxSize != 0 && freeSpace <= len) {
    // Compute the real disk free.  This will greatly slow
    // down our test but makes it more accurate:
    realUsage = dir.sizeInBytes();
    freeSpace = dir.maxSize - realUsage;
  }

  if (dir.maxSize != 0 && freeSpace <= len) {
    if (freeSpace > 0) {
      realUsage += freeSpace;
      if (b != null) {
        delegate.writeBytes(b, offset, (int) freeSpace);
      } else {
        delegate.copyBytes(in, (int) freeSpace);
      }
    }
    if (realUsage > dir.maxUsedSize) {
      dir.maxUsedSize = realUsage;
    }
    String message = "fake disk full at " + dir.sizeInBytes() + " bytes when writing " + name + " (file length=" + delegate.getFilePointer();
    if (freeSpace > 0) {
      message += "; wrote " + freeSpace + " of " + len + " bytes";
    }
    message += ")";
    if (LuceneTestCase.VERBOSE) {
      System.out.println(Thread.currentThread().getName() + ": MDW: now throw fake disk full");
      new Throwable().printStackTrace(System.out);
    }
    throw new IOException(message);
  }
}