org.apache.lucene.store.NIOFSDirectory Java Examples

public static void main(String[] args) throws Exception {
    // index
    try (Directory index = new NIOFSDirectory(Paths.get("/tmp/index"))) {
        // add
        try (IndexWriter writer = new IndexWriter(index, new IndexWriterConfig(new StandardAnalyzer()))) {
            Document doc = new Document();
            doc.add(new TextField("blog", "yuzhouwan.com", Field.Store.YES));
            doc.add(new StringField("github", "asdf2014", Field.Store.YES));
            writer.addDocument(doc);
            writer.commit();
        }
        // search
        try (DirectoryReader reader = DirectoryReader.open(index)) {
            IndexSearcher searcher = new IndexSearcher(reader);
            QueryParser parser = new QueryParser("blog", new StandardAnalyzer());
            Query query = parser.parse("yuzhouwan.com");
            ScoreDoc[] hits = searcher.search(query, 1000).scoreDocs;
            for (ScoreDoc hit : hits) {
                Document hitDoc = searcher.doc(hit.doc);
                System.out.println(hitDoc.get("blog"));
            }
        }
    }
}
 

@Override
protected Directory createDirectory(Properties parameters) throws IOException {
	if (parameters.containsKey(LuceneSail.LUCENE_DIR_KEY)) {
		return new NIOFSDirectory(Paths.get(parameters.getProperty(LuceneSail.LUCENE_DIR_KEY)));
	} else {
		return super.createDirectory(parameters);
	}
}
 

public LuceneSearcher() {
    File indexDir = new File(PathKit.getRootClassPath(), INDEX_PATH);
    if (!indexDir.exists()) {
        indexDir.mkdirs();
    }
    try {
        directory = NIOFSDirectory.open(indexDir.toPath());
    } catch (Exception e) {
        LOG.error(e.toString(), e);
    }
}
 

public LuceneSearcher() {
    File indexDir = new File(PathKit.getRootClassPath(), INDEX_PATH);
    if (!indexDir.exists()) {
        indexDir.mkdirs();
    }
    try {
        directory = NIOFSDirectory.open(indexDir.toPath());
    } catch (Exception e) {
        LOG.error(e.toString(), e);
    }
}
 

public void testPendingDeletionsRollbackWithReader() throws IOException {
  // irony: currently we don't emulate windows well enough to work on windows!
  assumeFalse("windows is not supported", Constants.WINDOWS);

  Path path = createTempDir();

  // Use WindowsFS to prevent open files from being deleted:
  FileSystem fs = new WindowsFS(path.getFileSystem()).getFileSystem(URI.create("file:///"));
  Path root = new FilterPath(path, fs);
  try (FSDirectory _dir = new NIOFSDirectory(root)) {
    Directory dir = new FilterDirectory(_dir) {};

    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
    IndexWriter w = new IndexWriter(dir, iwc);
    Document d = new Document();
    d.add(new StringField("id", "1", Field.Store.YES));
    d.add(new NumericDocValuesField("numval", 1));
    w.addDocument(d);
    w.commit();
    w.addDocument(d);
    w.flush();
    DirectoryReader reader = DirectoryReader.open(w);
    w.rollback();

    // try-delete superfluous files (some will fail due to windows-fs)
    IndexWriterConfig iwc2 = new IndexWriterConfig(new MockAnalyzer(random()));
    new IndexWriter(dir, iwc2).close();

    // test that we can index on top of pending deletions
    IndexWriterConfig iwc3 = new IndexWriterConfig(new MockAnalyzer(random()));
    w = new IndexWriter(dir, iwc3);
    w.addDocument(d);
    w.commit();

    reader.close();
    w.close();
  }
}
 

public void testPendingDeletesAlreadyWrittenFiles() throws IOException {
  Path path = createTempDir();
  // irony: currently we don't emulate windows well enough to work on windows!
  assumeFalse("windows is not supported", Constants.WINDOWS);

  // Use WindowsFS to prevent open files from being deleted:
  FileSystem fs = new WindowsFS(path.getFileSystem()).getFileSystem(URI.create("file:///"));
  Path root = new FilterPath(path, fs);
  DirectoryReader reader;
  // MMapDirectory doesn't work because it closes its file handles after mapping!
  try (FSDirectory dir = new NIOFSDirectory(root)) {
    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
    IndexWriter w = new IndexWriter(dir, iwc);
    w.commit();
    IndexInput in = dir.openInput("segments_1", IOContext.DEFAULT);
    w.addDocument(new Document());
    w.close();

    assertTrue(dir.getPendingDeletions().size() > 0);

    // make sure we get NoSuchFileException if we try to delete and already-pending-delete file:
    expectThrows(NoSuchFileException.class, () -> {
      dir.deleteFile("segments_1");
    });
    new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random()))).close();
    in.close();
  }
}
 

protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException {
    final String storeType = indexSettings.getSettings()
        .get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.FS.getSettingsKey());
    IndexModule.Type type;
    if (IndexModule.Type.FS.match(storeType)) {
        type = IndexModule.defaultStoreType(
            IndexModule.NODE_STORE_ALLOW_MMAP.getWithFallback(indexSettings.getNodeSettings()));
    } else {
        type = IndexModule.Type.fromSettingsKey(storeType);
    }
    switch (type) {
        case HYBRIDFS:
            // Use Lucene defaults
            final FSDirectory primaryDirectory = FSDirectory.open(location, lockFactory);
            if (primaryDirectory instanceof MMapDirectory) {
                MMapDirectory mMapDirectory = (MMapDirectory) primaryDirectory;
                return new HybridDirectory(lockFactory, mMapDirectory);
            } else {
                return primaryDirectory;
            }
        case MMAPFS:
            return new MMapDirectory(location, lockFactory);
        case SIMPLEFS:
            return new SimpleFSDirectory(location, lockFactory);
        case NIOFS:
            return new NIOFSDirectory(location, lockFactory);
        default:
            throw new AssertionError("unexpected built-in store type [" + type + "]");
    }
}
 

/**
 * Creates a corpus adapter which uses the Lucene index with the given path
 * and searches on the field with the given field name.
 * 
 * @param indexPath
 * @param fieldName
 * @return
 * @throws CorruptIndexException
 * @throws IOException
 */
public static LuceneCorpusAdapter create(String indexPath, String fieldName)
        throws CorruptIndexException, IOException {
    DirectoryReader dirReader = DirectoryReader.open(new NIOFSDirectory(new File(indexPath)));
    List<AtomicReaderContext> leaves = dirReader.leaves();
    AtomicReader reader[] = new AtomicReader[leaves.size()];
    AtomicReaderContext contexts[] = new AtomicReaderContext[leaves.size()];
    for (int i = 0; i < reader.length; i++) {
        contexts[i] = leaves.get(i);
        reader[i] = contexts[i].reader();
    }
    return new LuceneCorpusAdapter(dirReader, reader, contexts, fieldName);
}
 

@Activate
public void activate() throws Exception {
    directory = new NIOFSDirectory(Paths.get(System.getProperty("karaf.data"), INDEX_DIRECTORY));
    IndexWriterConfig indexWriterConfig = new IndexWriterConfig(new StandardAnalyzer());
    indexWriter = new IndexWriter(directory, indexWriterConfig);
    points = loadPoints();
}
 

public void open() throws Exception
{
    if ( !directory.exists() && !directory.mkdirs() )
    {
        throw new IOException("Could not make: " + directory);
    }

    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_35, new KeywordAnalyzer()).setOpenMode(IndexWriterConfig.OpenMode.CREATE);

    niofsDirectory = new NIOFSDirectory(directory, new SingleInstanceLockFactory());
    writer = new IndexWriter(niofsDirectory, conf);
}
 

private void verify(Path backup, int nDocs) throws IOException {
  log.info("Verifying ndocs={} in {}", nDocs, backup);
  try (Directory dir = new NIOFSDirectory(backup);
       IndexReader reader = DirectoryReader.open(dir)) {
    IndexSearcher searcher = new IndexSearcher(reader);
    TopDocs hits = searcher.search(new MatchAllDocsQuery(), 1);
    assertEquals(nDocs, hits.totalHits.value);
  }
}
 

private List<IndexCommit> listCommits(String directory) throws Exception {
  Directory dir = new NIOFSDirectory(Paths.get(directory));
  try {
    return DirectoryReader.listCommits(dir);
  } catch (IndexNotFoundException ex) {
    // This can happen when the delete snapshot functionality cleans up the index files (when the directory
    // storing these files is not the *current* index directory).
    return Collections.emptyList();
  }
}
 

public LogSearch(File file) throws Exception
{
    this.file = file;
    directory = new NIOFSDirectory(file, new NativeFSLockFactory());
    reader = IndexReader.open(directory);
    searcher = new IndexSearcher(reader);
}
 

private void doTestStoreDirectory(Index index,
                                  Path tempDir,
                                  String typeSettingValue,
                                  IndexModule.Type type) throws IOException {
    Settings.Builder settingsBuilder = Settings.builder()
        .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
    if (typeSettingValue != null) {
        settingsBuilder.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), typeSettingValue);
    }
    Settings settings = settingsBuilder.build();
    IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings);
    FsDirectoryService service = new FsDirectoryService(
        indexSettings, new ShardPath(false, tempDir, tempDir, new ShardId(index, 0)));
    try (Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) {
        switch (type) {
            case HYBRIDFS:
                assertHybridDirectory(directory);
                break;
            case NIOFS:
                assertTrue(type + " " + directory.toString(), directory instanceof NIOFSDirectory);
                break;
            case MMAPFS:
                assertTrue(type + " " + directory.toString(), directory instanceof MMapDirectory);
                break;
            case SIMPLEFS:
                assertTrue(type + " " + directory.toString(), directory instanceof SimpleFSDirectory);
                break;
            case FS:
                if (Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
                    assertHybridDirectory(directory);
                } else if (Constants.WINDOWS) {
                    assertTrue(directory.toString(), directory instanceof SimpleFSDirectory);
                } else {
                    assertTrue(directory.toString(), directory instanceof NIOFSDirectory);
                }
                break;
            default:
                fail();
        }
    }
}
 

static Directory getIndexDataDirectory() throws IOException {
	return new NIOFSDirectory(getIndexDataFolder().toPath());
}
 

protected Directory getDirectory() throws IOException {
  return NIOFSDirectory.open(getPath());
}
 

@Override
public void copyFileTo(URI sourceDir, String fileName, Directory dest) throws IOException {
  try (FSDirectory dir = new NIOFSDirectory(Paths.get(sourceDir), NoLockFactory.INSTANCE)) {
    dest.copyFrom(dir, fileName, fileName, DirectoryFactory.IOCONTEXT_NO_CACHE);
  }
}
 

@Override
public void copyFileFrom(Directory sourceDir, String fileName, URI dest) throws IOException {
  try (FSDirectory dir = new NIOFSDirectory(Paths.get(dest), NoLockFactory.INSTANCE)) {
    dir.copyFrom(sourceDir, fileName, fileName, DirectoryFactory.IOCONTEXT_NO_CACHE);
  }
}
 

@Override
public String[] listAll(URI dirPath) throws IOException {
  try (FSDirectory dir = new NIOFSDirectory(Paths.get(dirPath), NoLockFactory.INSTANCE)) {
    return dir.listAll();
  }
}
 

@Override
public IndexInput openInput(URI dirPath, String fileName, IOContext ctx) throws IOException {
  try (FSDirectory dir = new NIOFSDirectory(Paths.get(dirPath), NoLockFactory.INSTANCE)) {
    return dir.openInput(fileName, ctx);
  }
}
 

@Override
protected Directory create(String path, LockFactory lockFactory, DirContext dirContext) throws IOException {
  // we pass NoLockFactory, because the real lock factory is set later by injectLockFactory:
  return new NIOFSDirectory(new File(path).toPath(), lockFactory);
}
 

public void testCommandLineArgs() throws Exception {

    PrintStream savedSystemOut = System.out;
    System.setOut(new PrintStream(new ByteArrayOutputStream(), false, "UTF-8"));
    try {
      for (Map.Entry<String,Directory> entry : oldIndexDirs.entrySet()) {
        String name = entry.getKey();
        int indexCreatedVersion = SegmentInfos.readLatestCommit(entry.getValue()).getIndexCreatedVersionMajor();
        Path dir = createTempDir(name);
        TestUtil.unzip(getDataInputStream("index." + name + ".zip"), dir);
        
        String path = dir.toAbsolutePath().toString();
        
        List<String> args = new ArrayList<>();
        if (random().nextBoolean()) {
          args.add("-verbose");
        }
        if (random().nextBoolean()) {
          args.add("-delete-prior-commits");
        }
        if (random().nextBoolean()) {
          // TODO: need to better randomize this, but ...
          //  - LuceneTestCase.FS_DIRECTORIES is private
          //  - newFSDirectory returns BaseDirectoryWrapper
          //  - BaseDirectoryWrapper doesn't expose delegate
          Class<? extends FSDirectory> dirImpl = NIOFSDirectory.class;
          
          args.add("-dir-impl");
          args.add(dirImpl.getName());
        }
        args.add(path);
        
        IndexUpgrader upgrader = null;
        try {
          upgrader = IndexUpgrader.parseArgs(args.toArray(new String[0]));
        } catch (Exception e) {
          throw new AssertionError("unable to parse args: " + args, e);
        }
        upgrader.upgrade();
        
        Directory upgradedDir = newFSDirectory(dir);
        try {
          checkAllSegmentsUpgraded(upgradedDir, indexCreatedVersion);
        } finally {
          upgradedDir.close();
        }
      }
    } finally {
      System.setOut(savedSystemOut);
    }
  }
 

public void testWithPendingDeletions() throws Exception {
  // irony: currently we don't emulate windows well enough to work on windows!
  assumeFalse("windows is not supported", Constants.WINDOWS);

  Path path = createTempDir();

  // Use WindowsFS to prevent open files from being deleted:
  FileSystem fs = new WindowsFS(path.getFileSystem()).getFileSystem(URI.create("file:///"));
  Path root = new FilterPath(path, fs);
  IndexCommit indexCommit;
  DirectoryReader reader;
  // MMapDirectory doesn't work because it closes its file handles after mapping!
  try (FSDirectory dir = new NIOFSDirectory(root)) {
    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())).setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE);
    IndexWriter w = new IndexWriter(dir, iwc);
    w.commit();
    reader = w.getReader();
    // we pull this commit to open it again later to check that we fail if a future file delete is pending
    indexCommit = reader.getIndexCommit();
    w.close();
    w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())).setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE));
    w.addDocument(new Document());
    w.close();
    IndexInput in = dir.openInput("segments_2", IOContext.DEFAULT);
    dir.deleteFile("segments_2");
    assertTrue(dir.getPendingDeletions().size() > 0);

    // make sure we get NoSuchFileException if we try to delete and already-pending-delete file:
    expectThrows(NoSuchFileException.class, () -> {
      dir.deleteFile("segments_2");
    });

    try (IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())).setIndexCommit(indexCommit))) {
      writer.addDocument(new Document());
      writer.commit();
      assertEquals(1, writer.getDocStats().maxDoc);
      // now check that we moved to 3
      dir.openInput("segments_3", IOContext.READ).close();;
    }
    reader.close();
    in.close();
  }
}