Java Code Examples for org.apache.lucene.store.FSDirectory#open()

@Test
public void testRandomAccessWrites() throws IOException {
  long t1 = System.nanoTime();

  int i = 0;
  try {
    for (; i < 10; i++) {
      Directory fsDir = FSDirectory.open(new File(file, "normal").toPath());
      String name = getName();
      createFile(name, fsDir, directory);
      assertInputsEquals(name, fsDir, directory);
    }
  } catch (Exception e) {
    e.printStackTrace();
    fail("Test failed on pass [" + i + "]");
  }
  long t2 = System.nanoTime();
  System.out.println("Total time is " + ((t2 - t1)/1000000) + "ms");
}
 

private Directory createDirectory(boolean eraseIndex, String dirName,
    String dirParam) throws IOException {
  String dirImpl = config.get(dirParam, DEFAULT_DIRECTORY);
  if ("FSDirectory".equals(dirImpl)) {
    Path workDir = Paths.get(config.get("work.dir", "work"));
    Path indexDir = workDir.resolve(dirName);
    if (eraseIndex && Files.exists(indexDir)) {
      IOUtils.rm(indexDir);
    }
    Files.createDirectories(indexDir);
    return FSDirectory.open(indexDir);
  }

  if ("RAMDirectory".equals(dirImpl)) {
    throw new IOException("RAMDirectory has been removed, use ByteBuffersDirectory.");
  }

  if ("ByteBuffersDirectory".equals(dirImpl)) {
    return new ByteBuffersDirectory();
  }

  throw new IOException("Directory type not supported: " + dirImpl);
}
 

public LuceneEngine(IndexWriterConfig config, Path path) {
    try {
        this.config = config;
        Directory transienceDirectory = new ByteBuffersDirectory();
        this.transienceManager = new TransienceManager((IndexWriterConfig) BeanUtils.cloneBean(config), transienceDirectory);
        Directory persistenceDirectory = FSDirectory.open(path);
        this.persistenceManager = new PersistenceManager((IndexWriterConfig) BeanUtils.cloneBean(config), persistenceDirectory);
        this.searcher = new LuceneSearcher(this.transienceManager, this.persistenceManager);

        this.semaphore = new AtomicInteger();
        ReadWriteLock lock = new ReentrantReadWriteLock();
        this.readLock = lock.readLock();
        this.writeLock = lock.writeLock();
    } catch (Exception exception) {
        throw new StorageException(exception);
    }
}
 

/**
 * Index all text files under a directory.
 */
public void main(final File INDEX_DIR, final String[] args) throws IOException {
  IndexWriterConfig IWConfig = new IndexWriterConfig();
  IWConfig.setOpenMode (IndexWriterConfig.OpenMode.CREATE);
  IWConfig.setMergePolicy (new LogByteSizeMergePolicy());
  IndexWriter writer = new IndexWriter(FSDirectory.open(Paths.get(INDEX_DIR.getCanonicalPath())), IWConfig);
  for (int arg = 0; arg < args.length; arg++) {
    final File docDir = new File(args[arg]);
    if (!docDir.exists() || !docDir.canRead()) {
      System.out.println("Document directory '" + docDir.getAbsolutePath() + "' does not exist or is not readable, please check the path");
      throw new IOException("Cannot read from document directory");
    }

    indexDocs(writer, docDir);
    System.out.println("Optimizing...");
    writer.forceMerge(1);
  }
  writer.close();
}
 

/**
 *
 * @param indexDirectory 用于存放索引文件的目录
 * @param pictureDirectory 要进行图像索引的图片对象所在的目录
 */
public void generateIndex(String indexDirectory, String pictureDirectory){
    if(indexDirectory == null || indexDirectory.length() == 0 || pictureDirectory == null || pictureDirectory.length() == 0){
        return;
    }
    else {
        try {
            IndexWriter writer = new IndexWriter(FSDirectory.open(Paths.get(indexDirectory)), this.config);
            iterateFile(writer, new File(pictureDirectory));
            writer.close();
        } catch (IOException e) {
            e.printStackTrace();
            return;
        }
    }
}
 

public VectorStoreReaderLucene(String vectorFileName, FlagConfig flagConfig) throws IOException {
  this.flagConfig = flagConfig;
  this.vectorFileName = vectorFileName;
  this.vectorFile = new File(vectorFileName);
  try {
    String parentPath = this.vectorFile.getParent();
    if (parentPath == null) parentPath = "";
    this.directory = FSDirectory.open(FileSystems.getDefault().getPath(parentPath));  // Old from FSDirectory impl.
    // Read number of dimension from header information.
    this.threadLocalIndexInput = new ThreadLocal<IndexInput>() {
      @Override
      protected IndexInput initialValue() {
        try {
          return directory.openInput(vectorFile.getName(), IOContext.READ);
        } catch (IOException e) {
          throw new RuntimeException(e.getMessage(), e);
        }
      }
    };
    readHeadersFromIndexInput(flagConfig);
  } catch (IOException e) {
    logger.warning("Cannot open file: " + this.vectorFileName + "\n" + e.getMessage());
    throw e;
  }
}
 

public void split(Path destDir, String[] segs) throws IOException {
  Files.createDirectories(destDir);
  FSDirectory destFSDir = FSDirectory.open(destDir);
  SegmentInfos destInfos = new SegmentInfos(infos.getIndexCreatedVersionMajor());
  destInfos.counter = infos.counter;
  for (String n : segs) {
    SegmentCommitInfo infoPerCommit = getInfo(n);
    SegmentInfo info = infoPerCommit.info;
    // Same info just changing the dir:
    SegmentInfo newInfo = new SegmentInfo(destFSDir, info.getVersion(), info.getMinVersion(), info.name, info.maxDoc(),
                                          info.getUseCompoundFile(), info.getCodec(), info.getDiagnostics(), info.getId(), Collections.emptyMap(), null);
    destInfos.add(new SegmentCommitInfo(newInfo, infoPerCommit.getDelCount(), infoPerCommit.getSoftDelCount(),
        infoPerCommit.getDelGen(), infoPerCommit.getFieldInfosGen(),
        infoPerCommit.getDocValuesGen(), infoPerCommit.getId()));
    // now copy files over
    Collection<String> files = infoPerCommit.files();
    for (final String srcName : files) {
      Path srcFile = dir.resolve(srcName);
      Path destFile = destDir.resolve(srcName);
      Files.copy(srcFile, destFile);
    }
  }
  destInfos.changed();
  destInfos.commit(destFSDir);
  // System.out.println("destDir:"+destDir.getAbsolutePath());
}
 

protected static boolean clearIndex(File path) {
	try {
		if (path == null || !path.exists())
			return true; // if path doesnt exist, then there is nothing to
							// clear

		FSDirectory indexDir = new FSDirectoryProvider().getDirectory();
		IndexWriter writer = new IndexWriter(indexDir.open(path),
				new IndexWriterConfig(Version.LUCENE_CURRENT,
						new WhitespaceAnalyzer(Version.LUCENE_CURRENT)));
		writer.deleteAll();
		writer.close();
		return true;
	} catch (Exception ex) {
		org.webdsl.logging.Logger.error(
				"Error while clearing index on location: " + path, ex);
		return false;
	}

}
 

/**
 * Constructor
 * @param fs
 * @param shard
 * @param tempDir
 * @param conf
 * @throws IOException
 */
public ShardWriter(FileSystem fs, Shard shard, String tempDir, Configuration conf)
    throws IOException {
  logger.info("Construct a shard writer");

  this.conf = conf;
  this.fs = fs;
  localFs = FileSystem.getLocal(conf);
  perm = new Path(shard.getDirectory());
  taxoPerm = new Path(shard.getDirectory() + ".taxonomy");
  String indexDir = tempDir + "/" + "index";
  String taxoDir = tempDir + "/" + "taxo";
  temp = new Path(indexDir);
  taxoTemp = new Path(taxoDir);

  if (localFs.exists(temp)) {
    File tempFile = new File(temp.getName());
    if (tempFile.exists()) {
      LindenReducer.deleteDir(tempFile);
    }
  }

  if (!fs.exists(perm)) {
    fs.mkdirs(perm);
  } else {
    moveToTrash(conf, perm);
    fs.mkdirs(perm);
  }

  if (!fs.exists(taxoPerm)) {
    fs.mkdirs(taxoPerm);
  } else {
    moveToTrash(conf, taxoPerm);
    fs.mkdirs(taxoPerm);
  }
  IndexWriterConfig config = new IndexWriterConfig(Version.LATEST, null);
  config.setIndexDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
  writer = new IndexWriter(FSDirectory.open(new File(indexDir)), config);
  taxoWriter = new DirectoryTaxonomyWriter(FSDirectory.open(new File(taxoDir)));
}
 

public static void main(String[] args) {

        String indexPath = "index";
        String docsPath = "/Users/admin/github/elasticsearch-full/JAVA-QUERY-API/";
        boolean create = true;

        final Path docDir = Paths.get(docsPath);
        if (!Files.isReadable(docDir)) {
            System.out.println("Document directory '" + docDir.toAbsolutePath() + "' does not exist or is not readable, please check the path");
            System.exit(1);
        }

        Date start = new Date();
        try {
            System.out.println("Indexing to directory '" + indexPath + "'...");
            Directory dir = FSDirectory.open(Paths.get(indexPath));
            Analyzer analyzer = new StandardAnalyzer();
            IndexWriterConfig iwc = new IndexWriterConfig(analyzer);
            if (create) {
                iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
            } else {
                iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
            }

            IndexWriter writer = new IndexWriter(dir, iwc);
            indexDocs(writer, docDir);
            writer.close();
            Date end = new Date();
            System.out.println(end.getTime() - start.getTime() + " total milliseconds");
        } catch (Exception e) {
            System.out.println(" caught a " + e.getClass() +
                    "\n with message: " + e.getMessage());
        }

    }
 

@Override
public EventIndexSearcher borrowIndexSearcher(final File indexDir) throws IOException {
    final File absoluteFile = indexDir.getAbsoluteFile();

    final IndexWriterCount writerCount;
    synchronized (writerCounts) {
        writerCount = writerCounts.remove(absoluteFile);

        if (writerCount != null) {
            // Increment writer count and create an Index Searcher based on the writer
            writerCounts.put(absoluteFile, new IndexWriterCount(writerCount.getWriter(), writerCount.getAnalyzer(),
                writerCount.getDirectory(), writerCount.getCount() + 1, writerCount.isCloseableWhenUnused()));
        }
    }

    final DirectoryReader directoryReader;
    if (writerCount == null) {
        logger.trace("Creating index searcher for {}", indexDir);
        final Directory directory = FSDirectory.open(indexDir);
        directoryReader = DirectoryReader.open(directory);
    } else {
        final EventIndexWriter eventIndexWriter = writerCount.getWriter();
        directoryReader = DirectoryReader.open(eventIndexWriter.getIndexWriter(), false);
    }

    final IndexSearcher searcher = new IndexSearcher(directoryReader, this.searchExecutor);

    logger.trace("Created index searcher {} for {}", searcher, indexDir);
    return new LuceneEventIndexSearcher(searcher, indexDir, null, directoryReader);
}
 

public static void doSearch(String indexDir , String queryStr) throws IOException, ParseException, InvalidTokenOffsetsException {
    Directory directory = FSDirectory.open(Paths.get(indexDir));
    DirectoryReader reader = DirectoryReader.open(directory);
    IndexSearcher searcher = new IndexSearcher(reader);
    Analyzer analyzer = new SmartChineseAnalyzer();
    QueryParser parser = new QueryParser("tcontent",analyzer);
    Query query = parser.parse(queryStr);

    long startTime = System.currentTimeMillis();
    TopDocs docs = searcher.search(query,10);

    System.out.println("查找"+queryStr+"所用时间："+(System.currentTimeMillis()-startTime));
    System.out.println("查询到"+docs.totalHits+"条记录");

    //加入高亮显示的
    SimpleHTMLFormatter simpleHTMLFormatter = new SimpleHTMLFormatter("<b><font color=red>","</font></b>");
    QueryScorer scorer = new QueryScorer(query);//计算查询结果最高的得分
    Fragmenter fragmenter = new SimpleSpanFragmenter(scorer);//根据得分算出一个片段
    Highlighter highlighter = new Highlighter(simpleHTMLFormatter,scorer);
    highlighter.setTextFragmenter(fragmenter);//设置显示高亮的片段

    //遍历查询结果
    for(ScoreDoc scoreDoc : docs.scoreDocs){
        Document doc = searcher.doc(scoreDoc.doc);
        System.out.println(doc.get("title"));
        System.out.println(doc.get("tcontent"));
        String tcontent = doc.get("tcontent");
        if(tcontent != null){
            TokenStream tokenStream =  analyzer.tokenStream("tcontent", new StringReader(tcontent));
            String summary = highlighter.getBestFragment(tokenStream, tcontent);
            System.out.println(summary);
        }
    }
    reader.close();
}
 

private static void runTest(AccessControlFactory accessControlFactory) throws IOException {
  File file = new File("./src/test/resouces/loadtestindex-" + accessControlFactory.getClass().getName());
  FSDirectory directory = FSDirectory.open(file);
  if (!file.exists() || !DirectoryReader.indexExists(directory)) {
    long s = System.nanoTime();
    createIndex(directory, accessControlFactory);
    long e = System.nanoTime();
    System.out.println("Index Creation Time [" + (e - s) / 1000000.0 + "]");
  }
  DirectoryReader reader = DirectoryReader.open(directory);

  IndexSearcher searcher = new IndexSearcher(reader);

  SecureIndexSearcher secureIndexSearcher1 = new SecureIndexSearcher(reader, accessControlFactory,
      Arrays.asList("nothing"), Arrays.asList("nothing"), new HashSet<String>(), null);

  SecureIndexSearcher secureIndexSearcher2 = new SecureIndexSearcher(reader, accessControlFactory,
      Arrays.asList("r1"), Arrays.asList("nothing"), new HashSet<String>(), null);

  MatchAllDocsQuery query = new MatchAllDocsQuery();
  for (int p = 0; p < 10; p++) {
    hitEnterToContinue();
    runSearch(searcher, query);
    hitEnterToContinue();
    runSearch(secureIndexSearcher1, query);
    hitEnterToContinue();
    runSearch(secureIndexSearcher2, query);
  }
}
 

/**
 * Build the gazetteer index line by line
 *
 * @param gazetteerPath
 *            path of the gazetteer file
 * @param indexerPath
 *            path to the created Lucene index directory.
 * @param reverseGeocodingEnabled 
 * @throws IOException
 * @throws RuntimeException
 */
public void buildIndex(String gazetteerPath, String indexerPath, boolean reverseGeocodingEnabled)
		throws IOException {
	File indexfile = new File(indexerPath);
	indexDir = FSDirectory.open(indexfile.toPath());
	if (!DirectoryReader.indexExists(indexDir)) {
		IndexWriterConfig config = new IndexWriterConfig(analyzer);
		indexWriter = new IndexWriter(indexDir, config);
		Logger logger = Logger.getLogger(this.getClass().getName());
		logger.log(Level.WARNING, "Start Building Index for Gazatteer");
		BufferedReader filereader = new BufferedReader(
				new InputStreamReader(new FileInputStream(gazetteerPath),
						"UTF-8"));
		String line;
		int count = 0;
		while ((line = filereader.readLine()) != null) {
			try {
				count += 1;
				if (count % 100000 == 0) {
					logger.log(Level.INFO, "Indexed Row Count: " + count);
				}
				addDoc(indexWriter, line, reverseGeocodingEnabled);

			} catch (RuntimeException re) {
				logger.log(Level.WARNING, "Skipping... Error on line: {0}",
						line);
				re.printStackTrace();
			}
		}
		logger.log(Level.WARNING, "Building Finished");
		filereader.close();
		indexWriter.close();
	}
}
 

@PostConstruct
public void init() {
    try {
        directory = FSDirectory.open(Paths.get(indexPath));
    } catch (IOException e) {
        e.printStackTrace();
    }
}
 

/**
 * 生成索引
 *
 * @param indexDir
 * @throws Exception
 */
public void index(String indexDir) throws Exception {
  dir = FSDirectory.open(Paths.get(indexDir));
  IndexWriter writer = getWriter();
  for (int i = 0; i < usernames.length; i++) {
    Document doc = new Document();
    doc.add(new StringField("username", usernames[i] + "", Field.Store.YES));
    doc.add(new StringField("city", cities[i], Field.Store.YES));
    doc.add(new TextField("description", descriptions[i], Field.Store.YES));
    writer.addDocument(doc); // 添加文档
  }
  writer.close();
}
 

public void test1D() throws Exception {
  Directory dir = FSDirectory.open(createTempDir("2BPoints1D"));

  IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()))
    .setCodec(getCodec())
    .setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH)
    .setRAMBufferSizeMB(256.0)
    .setMergeScheduler(new ConcurrentMergeScheduler())
    .setMergePolicy(newLogMergePolicy(false, 10))
    .setOpenMode(IndexWriterConfig.OpenMode.CREATE);

  ((ConcurrentMergeScheduler) iwc.getMergeScheduler()).setMaxMergesAndThreads(6, 3);
  
  IndexWriter w = new IndexWriter(dir, iwc);

  MergePolicy mp = w.getConfig().getMergePolicy();
  if (mp instanceof LogByteSizeMergePolicy) {
   // 1 petabyte:
   ((LogByteSizeMergePolicy) mp).setMaxMergeMB(1024*1024*1024);
  }

  final int numDocs = (Integer.MAX_VALUE / 26) + 1;
  int counter = 0;
  for (int i = 0; i < numDocs; i++) {
    Document doc = new Document();
    for (int j=0;j<26;j++) {
      long x = (((long) random().nextInt() << 32)) | (long) counter;
      doc.add(new LongPoint("long", x));
      counter++;
    }
    w.addDocument(doc);
    if (VERBOSE && i % 100000 == 0) {
      System.out.println(i + " of " + numDocs + "...");
    }
  }
  w.forceMerge(1);
  DirectoryReader r = DirectoryReader.open(w);
  IndexSearcher s = new IndexSearcher(r);
  assertEquals(numDocs, s.count(LongPoint.newRangeQuery("long", Long.MIN_VALUE, Long.MAX_VALUE)));
  assertTrue(r.leaves().get(0).reader().getPointValues("long").size() > Integer.MAX_VALUE);
  r.close();
  w.close();
  System.out.println("TEST: now CheckIndex");
  TestUtil.checkIndex(dir);
  dir.close();
}
 

/**
 * @param args
 */
public static void main(String[] args) {
	// TODO Auto-generated method stub

	try {
	FlagConfig flagConfig 			= FlagConfig.getFlagConfig(args);
	VectorStoreRAM objectVectors 	= new VectorStoreRAM(flagConfig);
	String[] argsRemaining			= flagConfig.remainingArgs;
	String incomingVecs				= argsRemaining[0];
	int newDimension				= Integer.parseInt(argsRemaining[1]);
	objectVectors.initFromFile(incomingVecs);
	
	if (newDimension > flagConfig.dimension())
		{
		
				System.out.println("Incoming file has dimensionality of " +flagConfig.dimension());
				System.out.println("New dimensionality must be less than incoming vector length, quitting");
				System.exit(0);	
		}
	
		String vectorFileName = incomingVecs.replaceAll("\\.bin", "")+"_"+newDimension+".bin";
	  	File vectorFile = new File(vectorFileName);
	    String parentPath = vectorFile.getParent();
	    if (parentPath == null) parentPath = "";
	    FSDirectory fsDirectory = FSDirectory.open(FileSystems.getDefault().getPath(parentPath));
	    IndexOutput outputStream = fsDirectory.createOutput(vectorFile.getName(), IOContext.DEFAULT);
	 	flagConfig.setDimension(newDimension);
		outputStream.writeString(VectorStoreWriter.generateHeaderString(flagConfig));
	    Enumeration<ObjectVector> vecEnum = objectVectors.getAllVectors();

	    // Write each vector.
	    while (vecEnum.hasMoreElements()) {
	      ObjectVector objectVector = vecEnum.nextElement();
	      outputStream.writeString(objectVector.getObject().toString());
	      objectVector.getVector().writeToLuceneStream(outputStream,flagConfig.dimension());
	    }
	    
	    
	    outputStream.close();
	    fsDirectory.close();
		
	    VerbatimLogger.info("wrote "+objectVectors.getNumVectors()+" vectors to file "+ vectorFileName);
	    VerbatimLogger.info("finished writing vectors.\n");
	 		
	} catch (IOException e) {
		// TODO Auto-generated catch block
		e.printStackTrace();
		System.out.println("Usage: VectorStoreTruncater incomingFile.bin newDimensinoality");
	}
	
	
	
}
 

/** Index all text files under a directory. */
   public static void main(String[] args) {
String usage = "java pitt.search.lucene.LuceneIndexFromTriples [triples text file] ";
if (args.length == 0) {
    System.err.println("Usage: " + usage);
    System.exit(1);
}
      
if (Files.exists(INDEX_DIR)) {
	throw new IllegalArgumentException(
					   "Cannot save index to '" + INDEX_DIR + "' directory, please delete it first");
    }


try {
    // Create IndexWriter using WhiteSpaceAnalyzer without any stopword list.
    //IndexWriterConfig writerConfig = new IndexWriterConfig(
    //							   LUCENE_VERSION, new WhitespaceAnalyzer(LUCENE_VERSION));

    IndexWriterConfig writerConfig = new IndexWriterConfig(new WhitespaceAnalyzer());

    IndexWriter writer = new IndexWriter(FSDirectory.open(INDEX_DIR), writerConfig);




    final File triplesTextFile = new File(args[0]);
    if (!triplesTextFile.exists() || !triplesTextFile.canRead()) {
	writer.close();
	throw new IOException("Document file '" + triplesTextFile.getAbsolutePath() +
			      "' does not exist or is not readable, please check the path");
    }

    System.out.println("Indexing to directory '" +INDEX_DIR+ "'...");
    indexDoc(writer, triplesTextFile);
    writer.close();       
} catch (IOException e) {
    System.out.println(" caught a " + e.getClass() +
		       "\n with message: " + e.getMessage());
}
   }
 

/**
 * Constructor
 *
 * @param dir Directory for Lucene index.
 * @throws IOException on errors opening the lucene index
 */
public LuceneWikipediaIndexer(String dir) throws IOException {
  FSDirectory ldir = FSDirectory.open(FileSystems.getDefault().getPath(dir));
  IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer());
  index = new IndexWriter(ldir, config);
}