org.apache.lucene.store.SimpleFSDirectory Java Examples

/**
 * Acquires, then releases, all {@code write.lock} files in the given
 * shard paths. The "write.lock" file is assumed to be under the shard
 * path's "index" directory as used by Elasticsearch.
 *
 * @throws LockObtainFailedException if any of the locks could not be acquired
 */
public static void acquireFSLockForPaths(Settings indexSettings, Path... shardPaths) throws IOException {
    Lock[] locks = new Lock[shardPaths.length];
    Directory[] dirs = new Directory[shardPaths.length];
    try {
        for (int i = 0; i < shardPaths.length; i++) {
            // resolve the directory the shard actually lives in
            Path p = shardPaths[i].resolve("index");
            // open a directory (will be immediately closed) on the shard's location
            dirs[i] = new SimpleFSDirectory(p, FsDirectoryService.buildLockFactory(indexSettings));
            // create a lock for the "write.lock" file
            try {
                locks[i] = dirs[i].obtainLock(IndexWriter.WRITE_LOCK_NAME);
            } catch (IOException ex) {
                throw new LockObtainFailedException("unable to acquire " +
                        IndexWriter.WRITE_LOCK_NAME + " for " + p);
            }
        }
    } finally {
        IOUtils.closeWhileHandlingException(locks);
        IOUtils.closeWhileHandlingException(dirs);
    }
}
 

/**
 * Acquires, then releases, all {@code write.lock} files in the given
 * shard paths. The "write.lock" file is assumed to be under the shard
 * path's "index" directory as used by Elasticsearch.
 *
 * @throws LockObtainFailedException if any of the locks could not be acquired
 */
public static void acquireFSLockForPaths(IndexSettings indexSettings, Path... shardPaths) throws IOException {
    Lock[] locks = new Lock[shardPaths.length];
    Directory[] dirs = new Directory[shardPaths.length];
    try {
        for (int i = 0; i < shardPaths.length; i++) {
            // resolve the directory the shard actually lives in
            Path p = shardPaths[i].resolve("index");
            // open a directory (will be immediately closed) on the shard's location
            dirs[i] = new SimpleFSDirectory(p, indexSettings.getValue(FsDirectoryService.INDEX_LOCK_FACTOR_SETTING));
            // create a lock for the "write.lock" file
            try {
                locks[i] = dirs[i].obtainLock(IndexWriter.WRITE_LOCK_NAME);
            } catch (IOException ex) {
                throw new LockObtainFailedException("unable to acquire " +
                                IndexWriter.WRITE_LOCK_NAME + " for " + p, ex);
            }
        }
    } finally {
        IOUtils.closeWhileHandlingException(locks);
        IOUtils.closeWhileHandlingException(dirs);
    }
}
 

public static IndexReader getReader(String path) throws IOException
{
	if (!readerMap.containsKey(path))
	{
		synchronized(Indexes.class)
		{
			if (!readerMap.containsKey(path))
			{
				log.info("["+path+"] Opening...");
				if (! new File(path).exists())
					throw new ConfigurationException("Unable to find index in "+path);

				IndexReader r = IndexReader.open(new SimpleFSDirectory(new File(path)), true);
				readerMap.put(path, r);
				log.info("["+path+"] Opened. Memory: "+ExternalSortUtils.memSize(false));
			}
		}
	}
	
	return readerMap.get(path);
}
 

/**
 * Retrieves the index directory based on the type of index used.
 * @return a Directory
 * @throws IOException when the directory cannot be accessed
 * @since 3.0.0
 */
private synchronized Directory getDirectory() throws IOException {
    final File indexDir = getIndexDirectory(indexType);
    if (!indexDir.exists()) {
        if (!indexDir.mkdirs()) {
            LOGGER.error("Unable to create index directory: " + indexDir.getCanonicalPath());
            Notification.dispatch(new Notification()
                    .scope(NotificationScope.SYSTEM)
                    .group(NotificationGroup.FILE_SYSTEM)
                    .title(NotificationConstants.Title.FILE_SYSTEM_ERROR)
                    .content("Unable to create index directory: " + indexDir.getCanonicalPath())
                    .level(NotificationLevel.ERROR)
            );
        }
    }
    return new SimpleFSDirectory(indexDir.toPath());
}
 

@Transactional(readOnly = true)
public void deleteIndex(Integer contentId) throws IOException,
		ParseException {
	String path = realPathResolver.get(Constants.LUCENE_PATH);
	Directory dir = new SimpleFSDirectory(new File(path));
	deleteIndex(contentId, dir);
}
 

/**
 * Tries to open an index for the given location. This includes reading the
 * segment infos and possible corruption markers. If the index can not
 * be opened, an exception is thrown
 */
public static void tryOpenIndex(Path indexLocation, ShardId shardId, NodeEnvironment.ShardLocker shardLocker, Logger logger) throws IOException, ShardLockObtainFailedException {
    try (ShardLock lock = shardLocker.lock(shardId, "open index", TimeUnit.SECONDS.toMillis(5));
         Directory dir = new SimpleFSDirectory(indexLocation)) {
        failIfCorrupted(dir, shardId);
        SegmentInfos segInfo = Lucene.readSegmentInfos(dir);
        logger.trace("{} loaded segment info [{}]", shardId, segInfo);
    }
}
 

protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException {
    final String storeType = indexSettings.getSettings()
        .get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.FS.getSettingsKey());
    IndexModule.Type type;
    if (IndexModule.Type.FS.match(storeType)) {
        type = IndexModule.defaultStoreType(
            IndexModule.NODE_STORE_ALLOW_MMAP.getWithFallback(indexSettings.getNodeSettings()));
    } else {
        type = IndexModule.Type.fromSettingsKey(storeType);
    }
    switch (type) {
        case HYBRIDFS:
            // Use Lucene defaults
            final FSDirectory primaryDirectory = FSDirectory.open(location, lockFactory);
            if (primaryDirectory instanceof MMapDirectory) {
                MMapDirectory mMapDirectory = (MMapDirectory) primaryDirectory;
                return new HybridDirectory(lockFactory, mMapDirectory);
            } else {
                return primaryDirectory;
            }
        case MMAPFS:
            return new MMapDirectory(location, lockFactory);
        case SIMPLEFS:
            return new SimpleFSDirectory(location, lockFactory);
        case NIOFS:
            return new NIOFSDirectory(location, lockFactory);
        default:
            throw new AssertionError("unexpected built-in store type [" + type + "]");
    }
}
 

@Transactional(readOnly = true)
public List<Content> searchList(String path, String queryString,String category,String workplace,
		Integer siteId, Integer channelId, Date startDate, Date endDate,
		int first, int max) throws CorruptIndexException, IOException,
		ParseException {
	Directory dir = new SimpleFSDirectory(new File(path));
	return searchList(dir, queryString,category,workplace, siteId, channelId, startDate,
			endDate, first, max);
}
 

@Transactional(readOnly = true)
public Pagination searchPage(String path, String queryString,String category,String workplace,
		Integer siteId, Integer channelId, Date startDate, Date endDate,
		int pageNo, int pageSize) throws CorruptIndexException,
		IOException, ParseException {
	Directory dir = new SimpleFSDirectory(new File(path));
	return searchPage(dir, queryString, category,workplace,siteId, channelId, startDate,
			endDate, pageNo, pageSize);
}
 

@Override
public List<RepositoryInfo> getLoaded(final List<RepositoryInfo> repos) {
    final List<RepositoryInfo> toRet = new ArrayList<RepositoryInfo>(repos.size());
    for (final RepositoryInfo repo : repos) {
        File loc = new File(getDefaultIndexLocation(), repo.getId()); // index folder
        try {
            if (loc.exists() && new File(loc, "timestamp").exists() && DirectoryReader.indexExists(new SimpleFSDirectory(loc.toPath()))) {
                toRet.add(repo);
            }
        } catch (IOException ex) {
            LOGGER.log(Level.FINER, "Index Not Available: " +repo.getId() + " at: " + loc.getAbsolutePath(), ex);
        }
    }
    return toRet;
}
 

@Transactional(readOnly = true)
public Integer createIndex(Integer siteId, Integer channelId,
		Date startDate, Date endDate, Integer startId, Integer max)
		throws IOException, ParseException {
	String path = realPathResolver.get(Constants.LUCENE_PATH);
	Directory dir = new SimpleFSDirectory(new File(path));
	return createIndex(siteId, channelId, startDate, endDate, startId, max,
			dir);
}
 

/**
 * Runs check-index on the target shard and throws an exception if it failed
 */
public void checkIndex(ShardPath targetPath) throws IOException {
    BytesStreamOutput os = new BytesStreamOutput();
    PrintStream out = new PrintStream(os, false, Charsets.UTF_8.name());
    try (Directory directory = new SimpleFSDirectory(targetPath.resolveIndex());
        final CheckIndex checkIndex = new CheckIndex(directory)) {
        checkIndex.setInfoStream(out);
        CheckIndex.Status status = checkIndex.checkIndex();
        out.flush();
        if (!status.clean) {
            logger.warn("check index [failure]\n{}", new String(os.bytes().toBytes(), Charsets.UTF_8));
            throw new IllegalStateException("index check failure");
        }
    }
}
 

@Transactional(readOnly = true)
public void createIndex(Content content) throws IOException {
	String path = realPathResolver.get(Constants.LUCENE_PATH);
	Directory dir = new SimpleFSDirectory(new File(path));
	createIndex(content, dir);
}
 

public void updateIndex(Content content) throws IOException, ParseException {
	String path = realPathResolver.get(Constants.LUCENE_PATH);
	Directory dir = new SimpleFSDirectory(new File(path));
	updateIndex(content, dir);
}
 

private IndexSearcher getSearcher() throws IOException {
    FSDirectory directory = new SimpleFSDirectory(indexDir.toPath());
    IndexReader reader = DirectoryReader.open(directory);
    IndexSearcher searcher = new IndexSearcher(reader);
    return searcher;
}
 

@Override
	public void makeIndex(String lang, File workingDir) throws IOException
	{
		
		IndexReader articles = Indexes.getReader(RepositoryDirs.WIKIPEDIA.getPath(lang));
		Int2ObjectMap<String> bestAnchorMap = new BestAnchors(lang).getDataset();
		
		IndexWriter index = new IndexWriter(new SimpleFSDirectory(workingDir), new IndexWriterConfig(Version.LUCENE_34, new KeywordAnalyzer()));
		Document doc = new Document();
		Field fWID = new Field(FIELD_WID, "", Store.YES, Index.NOT_ANALYZED);
		Field fTitle = new Field(FIELD_TITLE, "", Store.YES, Index.NOT_ANALYZED);
		Field fAbstract = new Field(FIELD_ABSTRACT, "", Store.YES, Index.NO);
		Field fBestAnchor = new Field(FIELD_BEST_ANCHOR, "", Store.YES, Index.NO);
		doc.add(fWID);
		doc.add(fTitle);
		doc.add(fAbstract);
		doc.add(fBestAnchor);
				
		
		int max = articles.maxDoc();
		PLogger plog = new PLogger(log, Step.TEN_MINUTES, "pages", "indexed", "noBest");
		plog.setEnd(max);
		plog.start("Start indexing...");
		
		for(int i=0; i<max; i++)
		{
			plog.update(0);
			Document oldDoc = articles.document(i);
			PageType type = PageType.valueOf(oldDoc.get(WikipediaIndexer.FIELD_TYPE));
			if (type == PageType.TOPIC)
			{
				int wid = Integer.parseInt(oldDoc.get(WikipediaIndexer.FIELD_WID));
				fWID.setValue(oldDoc.get(WikipediaIndexer.FIELD_WID));
				fAbstract.setValue(oldDoc.get(WikipediaIndexer.FIELD_ABSTRACT));
				fTitle.setValue(oldDoc.get(WikipediaIndexer.FIELD_TITLE));
				
				String bestAnchor = bestAnchorMap.get(wid);
				if (bestAnchor == null || bestAnchor.length() == 0) plog.update(2);
				fBestAnchor.setValue(bestAnchor==null?"":bestAnchor);
				
				String[] cats = oldDoc.getValues(WikipediaIndexer.FIELD_CAT);
				if (cats != null) {
					for (int j=0; j<cats.length; j++)
						doc.add(new Field(FIELD_CAT, cats[j], Store.YES, Index.NOT_ANALYZED));
				}
				
				index.addDocument(doc);
				plog.update(1);
				
				doc.removeFields(FIELD_CAT);
			}
		}
		
		plog.stop();
		
		log.info("Now optimizing...");
		index.optimize();
		
		index.close();
		
		//we cannot call this because the index is still in the temporary dir
		//so TopicDocs will be created using old index
//		log.info("Index Done, now creating WID->DOC_ID map");
//		
//		TopicDocs td = new TopicDocs(lang);
//		td.forceParsing();
		
		log.info("Done.");
	}
 

/**
 * Loads the hunspell dictionary for the given local.
 *
 * @param locale       The locale of the hunspell dictionary to be loaded.
 * @param nodeSettings The node level settings
 * @param env          The node environment (from which the conf path will be resolved)
 * @return The loaded Hunspell dictionary
 * @throws Exception when loading fails (due to IO errors or malformed dictionary files)
 */
private Dictionary loadDictionary(String locale, Settings nodeSettings, Environment env) throws Exception {
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("Loading hunspell dictionary [{}]...", locale);
    }
    Path dicDir = hunspellDir.resolve(locale);
    if (FileSystemUtils.isAccessibleDirectory(dicDir, LOGGER) == false) {
        throw new ElasticsearchException(String.format(Locale.ROOT, "Could not find hunspell dictionary [%s]", locale));
    }

    // merging node settings with hunspell dictionary specific settings
    Settings dictSettings = HUNSPELL_DICTIONARY_OPTIONS.get(nodeSettings);
    nodeSettings = loadDictionarySettings(dicDir, dictSettings.getByPrefix(locale + "."));

    boolean ignoreCase = nodeSettings.getAsBoolean("ignore_case", defaultIgnoreCase);

    Path[] affixFiles = FileSystemUtils.files(dicDir, "*.aff");
    if (affixFiles.length == 0) {
        throw new ElasticsearchException(String.format(Locale.ROOT, "Missing affix file for hunspell dictionary [%s]", locale));
    }
    if (affixFiles.length != 1) {
        throw new ElasticsearchException(String.format(Locale.ROOT, "Too many affix files exist for hunspell dictionary [%s]", locale));
    }
    InputStream affixStream = null;

    Path[] dicFiles = FileSystemUtils.files(dicDir, "*.dic");
    List<InputStream> dicStreams = new ArrayList<>(dicFiles.length);
    try {

        for (int i = 0; i < dicFiles.length; i++) {
            dicStreams.add(Files.newInputStream(dicFiles[i]));
        }

        affixStream = Files.newInputStream(affixFiles[0]);

        try (Directory tmp = new SimpleFSDirectory(env.tmpFile())) {
            return new Dictionary(tmp, "hunspell", affixStream, dicStreams, ignoreCase);
        }

    } catch (Exception e) {
        LOGGER.error(() -> new ParameterizedMessage("Could not load hunspell dictionary [{}]", locale), e);
        throw e;
    } finally {
        IOUtils.close(affixStream);
        IOUtils.close(dicStreams);
    }
}
 

@Override
protected AnchorTernaryTrie parseSet() throws IOException
{
	
	File indexDir = RepositoryDirs.ANCHORS.getDir(lang);
	long indexSize = FileUtils.sizeOfDirectory(indexDir);
	long maxMemory = Runtime.getRuntime().maxMemory();
	
	IndexReader anchors;
	if (indexSize < maxMemory * 0.8){
		
		log.info("MaxMemory is enough, loading Anchor index...");
		anchors = IndexReader.open(new RAMDirectory(new SimpleFSDirectory(indexDir)), true);
		log.info("Anchor index loaded.");
		
	} else {
		log.info("Not enough memory ["+maxMemory/1000000+"Mb] to load Anchor index (about "+indexSize/1000000+"Mb)");
		anchors = Indexes.getReader(RepositoryDirs.ANCHORS.getPath(lang));
	}

	
	AnchorTernaryTrie trie = new AnchorTernaryTrie();
	
	int maxdoc = anchors.maxDoc();
	
	IntList doclist = new IntArrayList();
	for(int i=0;i<maxdoc;i++) doclist.add(i);
	Random rnd = new Random(System.currentTimeMillis());
	
	PLogger plog = new PLogger(log, Step.TEN_MINUTES, "anchors", "skipped", "duplicates");
	plog.setEnd(0, maxdoc);
	plog.start("Inserting in to trie...");
	while(!doclist.isEmpty())
	{
		int docID = doclist.removeInt(rnd.nextInt(doclist.size()));
		
		plog.update(0);
		Document doc = anchors.document(docID);
		if (doc == null){
			plog.update(1);
			continue;
		}
		
		String anchorText = doc.get(AnchorIndexer.FIELD_TEXT);
		String serial = doc.get(AnchorIndexer.FIELD_OBJECT);
		Anchor anchorObj = Anchor.deserialize(serial);
		
		if (anchorObj == null){
			plog.update(1);
			continue;
		}
		
		boolean added = trie.add(anchorText, anchorObj);
		
		if (!added) plog.update(2);
	}
	plog.stop();
	
	return trie;
}
 

protected Directory newDirectory(Path dir) throws IOException {
    return new SimpleFSDirectory(dir);
}
 

protected Directory newDirectory(Path dir) throws IOException {
    return new SimpleFSDirectory(dir);
}
 

private void doTestStoreDirectory(Index index,
                                  Path tempDir,
                                  String typeSettingValue,
                                  IndexModule.Type type) throws IOException {
    Settings.Builder settingsBuilder = Settings.builder()
        .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
    if (typeSettingValue != null) {
        settingsBuilder.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), typeSettingValue);
    }
    Settings settings = settingsBuilder.build();
    IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings);
    FsDirectoryService service = new FsDirectoryService(
        indexSettings, new ShardPath(false, tempDir, tempDir, new ShardId(index, 0)));
    try (Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) {
        switch (type) {
            case HYBRIDFS:
                assertHybridDirectory(directory);
                break;
            case NIOFS:
                assertTrue(type + " " + directory.toString(), directory instanceof NIOFSDirectory);
                break;
            case MMAPFS:
                assertTrue(type + " " + directory.toString(), directory instanceof MMapDirectory);
                break;
            case SIMPLEFS:
                assertTrue(type + " " + directory.toString(), directory instanceof SimpleFSDirectory);
                break;
            case FS:
                if (Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
                    assertHybridDirectory(directory);
                } else if (Constants.WINDOWS) {
                    assertTrue(directory.toString(), directory instanceof SimpleFSDirectory);
                } else {
                    assertTrue(directory.toString(), directory instanceof NIOFSDirectory);
                }
                break;
            default:
                fail();
        }
    }
}