Java Code Examples for org.apache.lucene.store.Directory#deleteFile()

public void testExceptionDuringRefresh() throws Exception {

    Directory indexDir = newDirectory();
    Directory taxoDir = newDirectory();

    IndexWriter w = new IndexWriter(indexDir, newIndexWriterConfig(new MockAnalyzer(random())));
    DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir);
    w.commit();
    tw.commit();

    SearcherTaxonomyManager mgr = new SearcherTaxonomyManager(indexDir, taxoDir, null);

    tw.addCategory(new FacetLabel("a", "b"));
    w.addDocument(new Document());

    tw.commit();
    w.commit();

    // intentionally corrupt the taxo index:
    SegmentInfos infos = SegmentInfos.readLatestCommit(taxoDir);
    taxoDir.deleteFile(infos.getSegmentsFileName());
    expectThrows(IndexNotFoundException.class, mgr::maybeRefreshBlocking);
    IOUtils.close(w, tw, mgr, indexDir, taxoDir);
  }
 

/**
 * Deletes all given file names.  Some of the
 * file names may be null; they are
 * ignored.  After everything is deleted, the method either
 * throws the first exception it hit while deleting, or
 * completes normally if there were no exceptions.
 * 
 * @param dir Directory to delete files from
 * @param names file names to delete
 */
public static void deleteFiles(Directory dir, Collection<String> names) throws IOException {
  Throwable th = null;
  for (String name : names) {
    if (name != null) {
      try {
        dir.deleteFile(name);
      } catch (Throwable t) {
        th = useOrSuppress(th, t);
      }
    }
  }

  if (th != null) {
    throw rethrowAlways(th);
  }
}
 

/**
 * This method removes all lucene files from the given directory. It will first try to delete all commit points / segments
 * files to ensure broken commits or corrupted indices will not be opened in the future. If any of the segment files can't be deleted
 * this operation fails.
 */
public static void cleanLuceneIndex(Directory directory) throws IOException {
    try (Lock writeLock = directory.obtainLock(IndexWriter.WRITE_LOCK_NAME)) {
        for (final String file : directory.listAll()) {
            if (file.startsWith(IndexFileNames.SEGMENTS) || file.equals(IndexFileNames.OLD_SEGMENTS_GEN)) {
                directory.deleteFile(file); // remove all segment_N files
            }
        }
    }
    try (IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)
            .setSoftDeletesField(Lucene.SOFT_DELETES_FIELD)
            .setMergePolicy(NoMergePolicy.INSTANCE) // no merges
            .setCommitOnClose(false) // no commits
            .setOpenMode(IndexWriterConfig.OpenMode.CREATE))) { // force creation - don't append...
        // do nothing and close this will kick of IndexFileDeleter which will remove all pending files
    }
}
 

/**
 * Deletes all corruption markers from this store.
 */
public void removeCorruptionMarker() throws IOException {
    ensureOpen();
    final Directory directory = directory();
    IOException firstException = null;
    final String[] files = directory.listAll();
    for (String file : files) {
        if (file.startsWith(CORRUPTED)) {
            try {
                directory.deleteFile(file);
            } catch (IOException ex) {
                if (firstException == null) {
                    firstException = ex;
                } else {
                    firstException.addSuppressed(ex);
                }
            }
        }
    }
    if (firstException != null) {
        throw firstException;
    }
}
 

/**
 * Delete the files in the Directory
 */
public static boolean empty(Directory dir) {
  boolean isSuccess = true;
  String contents[];
  try {
    contents = dir.listAll();
    if (contents != null) {
      for (String file : contents) {
        dir.deleteFile(file);
      }
    }
  } catch (IOException e) {
    SolrException.log(log, "Error deleting files from Directory", e);
    isSuccess = false;
  }
  return isSuccess;
}
 

/**
 * Deletes all given files, suppressing all thrown IOExceptions.
 * <p>
 * Note that the files should not be null.
 */
public static void deleteFilesIgnoringExceptions(Directory dir, Collection<String> files) {
  for(String name : files) {
    try {
      dir.deleteFile(name);
    } catch (Throwable ignored) {
      // ignore
    }
  }
}
 

private static void deleteFileIfExists(Path stateLocation, Directory directory, String fileName) throws IOException {
    try {
        directory.deleteFile(fileName);
    } catch (FileNotFoundException | NoSuchFileException ignored) {

    }
    LOGGER.trace("cleaned up {}", stateLocation.resolve(fileName));
}
 

public void testSave() throws IOException {
  final int valueCount = TestUtil.nextInt(random(), 1, 2048);
  for (int bpv = 1; bpv <= 64; ++bpv) {
    final int maxValue = (int) Math.min(PackedInts.maxValue(31), PackedInts.maxValue(bpv));
    final Directory directory = new ByteBuffersDirectory();
    List<PackedInts.Mutable> packedInts = createPackedInts(valueCount, bpv);
    for (PackedInts.Mutable mutable : packedInts) {
      for (int i = 0; i < mutable.size(); ++i) {
        mutable.set(i, random().nextInt(maxValue));
      }

      IndexOutput out = directory.createOutput("packed-ints.bin", IOContext.DEFAULT);
      mutable.save(out);
      out.close();

      IndexInput in = directory.openInput("packed-ints.bin", IOContext.DEFAULT);
      PackedInts.Reader reader = PackedInts.getReader(in);
      assertEquals(valueCount, reader.size());
      if (mutable instanceof Packed64SingleBlock) {
        // make sure that we used the right format so that the reader has
        // the same performance characteristics as the mutable that has been
        // serialized
        assertTrue(reader instanceof Packed64SingleBlock);
      } else {
        assertFalse(reader instanceof Packed64SingleBlock);
      }
      for (int i = 0; i < valueCount; ++i) {
        assertEquals(mutable.get(i), reader.get(i));
      }
      in.close();
      directory.deleteFile("packed-ints.bin");
    }
    directory.close();
  }
}
 

private final void deleteFiles(Vector files, Directory directory)
     throws IOException {
  for (int i = 0; i < files.size(); i++)
    directory.deleteFile((String)files.elementAt(i));
}
 

private final void deleteFiles(Vector files, Directory directory)
     throws IOException {
  for (int i = 0; i < files.size(); i++)
    directory.deleteFile((String)files.elementAt(i));
}
 

private final void deleteFiles(Vector files, Directory directory)
     throws IOException {
  for (int i = 0; i < files.size(); i++)
    directory.deleteFile((String)files.elementAt(i));
}
 

private final void deleteFiles(Vector files, Directory directory)
     throws IOException {
  for (int i = 0; i < files.size(); i++)
    directory.deleteFile((String)files.elementAt(i));
}
 

private final void deleteFiles(Vector files, Directory directory)
     throws IOException {
  for (int i = 0; i < files.size(); i++)
    directory.deleteFile((String)files.elementAt(i));
}
 

private final void deleteFiles(Vector files, Directory directory)
     throws IOException {
  for (int i = 0; i < files.size(); i++)
    directory.deleteFile((String)files.elementAt(i));
}
 

public void testKeepLastNDeletionPolicy() throws IOException {
  final int N = 5;

  for(int pass=0;pass<2;pass++) {

    boolean useCompoundFile = (pass % 2) != 0;

    Directory dir = newDirectory();

    KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(N);
    for(int j=0;j<N+1;j++) {
      IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()))
          .setOpenMode(OpenMode.CREATE)
          .setIndexDeletionPolicy(policy)
          .setMaxBufferedDocs(10);
      MergePolicy mp = conf.getMergePolicy();
      mp.setNoCFSRatio(useCompoundFile ? 1.0 : 0.0);
      IndexWriter writer = new IndexWriter(dir, conf);
      policy = (KeepLastNDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
      for(int i=0;i<17;i++) {
        addDoc(writer);
      }
      writer.forceMerge(1);
      writer.close();
    }

    assertTrue(policy.numDelete > 0);
    assertEquals(N+1, policy.numOnInit);
    assertEquals(N+1, policy.numOnCommit);

    // Simplistic check: just verify only the past N segments_N's still
    // exist, and, I can open a reader on each:
    long gen = SegmentInfos.getLastCommitGeneration(dir);
    for(int i=0;i<N+1;i++) {
      try {
        IndexReader reader = DirectoryReader.open(dir);
        reader.close();
        if (i == N) {
          fail("should have failed on commits prior to last " + N);
        }
      } catch (IOException e) {
        if (i != N) {
          throw e;
        }
      }
      if (i < N) {
        dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
      }
      gen--;
    }

    dir.close();
  }
}
 

@Nightly
public void testExpirationTimeDeletionPolicy() throws IOException, InterruptedException {

  final double SECONDS = 2.0;

  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()))
      .setIndexDeletionPolicy(new ExpirationTimeDeletionPolicy(dir, SECONDS));
  MergePolicy mp = conf.getMergePolicy();
  mp.setNoCFSRatio(1.0);
  IndexWriter writer = new IndexWriter(dir, conf);
  ExpirationTimeDeletionPolicy policy = (ExpirationTimeDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
  Map<String,String> commitData = new HashMap<>();
  commitData.put("commitTime", String.valueOf(System.currentTimeMillis()));
  writer.setLiveCommitData(commitData.entrySet());
  writer.commit();
  writer.close();

  long lastDeleteTime = 0;
  final int targetNumDelete = TestUtil.nextInt(random(), 1, 5);
  while (policy.numDelete < targetNumDelete) {
    // Record last time when writer performed deletes of
    // past commits
    lastDeleteTime = System.currentTimeMillis();
    conf = newIndexWriterConfig(new MockAnalyzer(random()))
             .setOpenMode(OpenMode.APPEND)
             .setIndexDeletionPolicy(policy);
    mp = conf.getMergePolicy();
    mp.setNoCFSRatio(1.0);
    writer = new IndexWriter(dir, conf);
    policy = (ExpirationTimeDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
    for(int j=0;j<17;j++) {
      addDoc(writer);
    }
    commitData = new HashMap<>();
    commitData.put("commitTime", String.valueOf(System.currentTimeMillis()));
    writer.setLiveCommitData(commitData.entrySet());
    writer.commit();
    writer.close();

    Thread.sleep((int) (1000.0*(SECONDS/5.0)));
  }

  // Then simplistic check: just verify that the
  // segments_N's that still exist are in fact within SECONDS
  // seconds of the last one's mod time, and, that I can
  // open a reader on each:
  long gen = SegmentInfos.getLastCommitGeneration(dir);
  
  String fileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
                                                          "",
                                                          gen);
  boolean oneSecondResolution = true;

  while(gen > 0) {
    try {
      IndexReader reader = DirectoryReader.open(dir);
      reader.close();
      fileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
                                                       "",
                                                       gen);

      // if we are on a filesystem that seems to have only
      // 1 second resolution, allow +1 second in commit
      // age tolerance:
      SegmentInfos sis = SegmentInfos.readCommit(dir, fileName);
      assertEquals(Version.LATEST, sis.getCommitLuceneVersion());
      assertEquals(Version.LATEST, sis.getMinSegmentLuceneVersion());
      long modTime = Long.parseLong(sis.getUserData().get("commitTime"));
      oneSecondResolution &= (modTime % 1000) == 0;
      final long leeway = (long) ((SECONDS + (oneSecondResolution ? 1.0:0.0))*1000);

      assertTrue("commit point was older than " + SECONDS + " seconds (" + (lastDeleteTime - modTime) + " msec) but did not get deleted ", lastDeleteTime - modTime <= leeway);
    } catch (IOException e) {
      // OK
      break;
    }
    
    dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
    gen--;
  }

  dir.close();
}
 

/**
 * Override for more efficient moves.
 * 
 * Intended for use with replication - use
 * carefully - some Directory wrappers will
 * cache files for example.
 * 
 * @throws IOException If there is a low-level I/O error.
 */
public void move(Directory fromDir, Directory toDir, String fileName, IOContext ioContext) throws IOException {
  toDir.copyFrom(fromDir, fileName, fileName, ioContext);
  fromDir.deleteFile(fileName);
}