org.apache.lucene.store.MMapDirectory Java Examples

@Before
public void setUp() throws Exception {
    analyzer = new StandardAnalyzer();
    tempDirectory = Files.createTempDirectory("lucene");
    directory = new MMapDirectory(tempDirectory);
    IndexWriterConfig config = new IndexWriterConfig(analyzer);
    IndexWriter iwriter = new IndexWriter(directory, config);

    Document doc = new Document();
    doc.add(new Field("contents", "name=text", TextField.TYPE_STORED));

    IntPoint intPoint = new IntPoint("intfield", 4);
    doc.add(intPoint);
    doc.add(new StoredField("intfield", 4));
    iwriter.addDocument(doc);

    iwriter.close();
    ireader = DirectoryReader.open(directory);
    isearcher = new IndexSearcher(ireader);
}
 

private static Directory setPreload(Directory directory, Path location, LockFactory lockFactory,
        Set<String> preLoadExtensions) throws IOException {
    if (preLoadExtensions.isEmpty() == false
            && directory instanceof MMapDirectory
            && ((MMapDirectory) directory).getPreload() == false) {
        if (preLoadExtensions.contains("*")) {
            ((MMapDirectory) directory).setPreload(true);
            return directory;
        }
        MMapDirectory primary = new MMapDirectory(location, lockFactory);
        primary.setPreload(true);
        return new FileSwitchDirectory(preLoadExtensions, primary, directory, true) {
            @Override
            public String[] listAll() throws IOException {
                // avoid listing twice
                return primary.listAll();
            }
        };
    }
    return directory;
}
 

public TripleIndexContext() throws IOException {
	Properties prop = new Properties();
	InputStream input = TripleIndexContext.class.getResourceAsStream("/config/agdistis.properties");
	prop.load(input);

	String envIndex = System.getenv("AGDISTIS_INDEX_BY_CONTEXT");
	String index = envIndex != null ? envIndex : prop.getProperty("index_bycontext");
	log.info("The index will be here: " + index);

	directory = new MMapDirectory(new File(index));
	ireader = DirectoryReader.open(directory);
	isearcher = new IndexSearcher(ireader);
	new UrlValidator();

	cache = CacheBuilder.newBuilder().maximumSize(50000).build();
}
 

public TripleIndex() throws IOException {
	Properties prop = new Properties();
	InputStream input = TripleIndex.class.getResourceAsStream("/config/agdistis.properties");
	prop.load(input);

	String envIndex = System.getenv("AGDISTIS_INDEX");
	String index = envIndex != null ? envIndex : prop.getProperty("index");
	log.info("The index will be here: " + index);

	directory = new MMapDirectory(new File(index));
	ireader = DirectoryReader.open(directory);
	isearcher = new IndexSearcher(ireader);
	this.urlValidator = new UrlValidator();

	cache = CacheBuilder.newBuilder().maximumSize(50000).build();
}
 

/**
 * Create a (possibly) shared NLP environment. The given data directory
 * must be created (usually from a downloaded zipfile, check the README).
 * Expect many open files and many reads. Network filesystems are known to
 * perform poorly as data directories. Strive to use a local directory if
 * possible, or at least the Lucene indices otherwise.
 * 
 * config.properties can be either in the data directory or the working
 * directory. This is to allow sharing (read-only) indices while still
 * allowing separate development configurations.  
 */
public Environment() {
	
	// Now do some per-thread setup
	db = new Database(this);
	rdf = TDBFactory.assembleDataset(
			pathMustExist("rdf/jena-lucene.ttl"));
	
	// Lucene indexes have huge overhead so avoid re-instantiating by putting them in the Environment
	IndexReader reader;
	try {
		reader = DirectoryReader.open(new MMapDirectory(Paths.get(getConfOrDie("lucene_index"))));
	} catch (IOException e) {
		e.printStackTrace();
		throw new RuntimeException("The candidate-answer Lucene index failed to open.");
	}
	lucene = new IndexSearcher(reader);
	//lucene.setSimilarity(new BM25Similarity());
}
 

public DirectoryTaxonomyWriter getTaxoWriter(int segmentNumber) throws IOException {

		Directory d;

		if (indexConfig.getIndexSettings().getStoreIndexOnDisk()) {
			d = MMapDirectory.open(getPathForFacetsIndex(segmentNumber));
		}
		else {
			String indexSegmentDbName = getIndexSegmentDbName(segmentNumber);
			String indexSegmentCollectionName = getIndexSegmentCollectionName(segmentNumber) + "_facets";
			MongoDirectory mongoDirectory = new MongoDirectory(mongo, indexSegmentDbName, indexSegmentCollectionName, clusterConfig.isSharded(),
					clusterConfig.getIndexBlockSize());
			d = new DistributedDirectory(mongoDirectory);
		}

		NRTCachingDirectory nrtCachingDirectory = new NRTCachingDirectory(d, 2, 10);

		return new DirectoryTaxonomyWriter(nrtCachingDirectory);
	}
 

/**
 * Create a new empty search index.
 */
public SongSearchIndex() {
    songs = new HashMap<>();
    try {
        analyzer = CustomAnalyzer.builder()
                .withTokenizer(StandardTokenizerFactory.class)
                .addTokenFilter(LowerCaseFilterFactory.class)
                .addTokenFilter(ASCIIFoldingFilterFactory.class)
                .build();
        index = new MMapDirectory(Files.createTempDirectory("quelea-mmap-song").toAbsolutePath());
    }
    catch(IOException ex) {
        LOGGER.log(Level.SEVERE, "Couldn't create song search index");
        throw new RuntimeException("Couldn't create song search index", ex);
    }
}
 

protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException {
    final String storeType = indexSettings.getSettings()
        .get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.FS.getSettingsKey());
    IndexModule.Type type;
    if (IndexModule.Type.FS.match(storeType)) {
        type = IndexModule.defaultStoreType(
            IndexModule.NODE_STORE_ALLOW_MMAP.getWithFallback(indexSettings.getNodeSettings()));
    } else {
        type = IndexModule.Type.fromSettingsKey(storeType);
    }
    switch (type) {
        case HYBRIDFS:
            // Use Lucene defaults
            final FSDirectory primaryDirectory = FSDirectory.open(location, lockFactory);
            if (primaryDirectory instanceof MMapDirectory) {
                MMapDirectory mMapDirectory = (MMapDirectory) primaryDirectory;
                return new HybridDirectory(lockFactory, mMapDirectory);
            } else {
                return primaryDirectory;
            }
        case MMAPFS:
            return new MMapDirectory(location, lockFactory);
        case SIMPLEFS:
            return new SimpleFSDirectory(location, lockFactory);
        case NIOFS:
            return new NIOFSDirectory(location, lockFactory);
        default:
            throw new AssertionError("unexpected built-in store type [" + type + "]");
    }
}
 

static Map<String, String> getReaderAttributes(Directory directory) {
    Directory unwrap = FilterDirectory.unwrap(directory);
    boolean defaultOffHeap = FsDirectoryService.isHybridFs(unwrap) || unwrap instanceof MMapDirectory;
    return Map.of(
        // if we are using MMAP for term dics we force all off heap
        BlockTreeTermsReader.FST_MODE_KEY,
        defaultOffHeap ? FSTLoadMode.OFF_HEAP.name() : FSTLoadMode.ON_HEAP.name()
    );
}
 

public static Type defaultStoreType(final boolean allowMmap) {
    if (allowMmap && Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
        return Type.HYBRIDFS;
    } else if (Constants.WINDOWS) {
        return Type.SIMPLEFS;
    } else {
        return Type.NIOFS;
    }
}
 

public void createIndex(List<File> files, String idxDirectory, String baseURI) {
	try {
		urlAnalyzer = new SimpleAnalyzer(LUCENE_VERSION);
		literalAnalyzer = new LiteralAnalyzer(LUCENE_VERSION);
		Map<String, Analyzer> mapping = new HashMap<String, Analyzer>();
		mapping.put(FIELD_NAME_URI, urlAnalyzer);
		mapping.put(FIELD_NAME_SURFACE_FORM, literalAnalyzer);
		mapping.put(FIELD_NAME_URI_COUNT, literalAnalyzer);
		mapping.put(FIELD_NAME_CONTEXT, literalAnalyzer);
		PerFieldAnalyzerWrapper perFieldAnalyzer = new PerFieldAnalyzerWrapper(urlAnalyzer, mapping);

		File indexDirectory = new File(idxDirectory);
		indexDirectory.mkdir();
		directory = new MMapDirectory(indexDirectory);
		IndexWriterConfig config = new IndexWriterConfig(LUCENE_VERSION, perFieldAnalyzer);
		iwriter = new IndexWriter(directory, config);
		iwriter.commit();
		for (File file : files) {
			String type = FileUtil.getFileExtension(file.getName());
			if (type.equals(TTL))
				indexTTLFile(file, baseURI);
			iwriter.commit();
		}
	} catch (Exception e) {
		log.error("Error while creating TripleIndex.", e);
	}
}
 

public void createIndex(List<File> files, String idxDirectory, String baseURI) {
	try {
		urlAnalyzer = new SimpleAnalyzer(LUCENE_VERSION);
		literalAnalyzer = new LiteralAnalyzer(LUCENE_VERSION);
		Map<String, Analyzer> mapping = new HashMap<String, Analyzer>();
		mapping.put(TripleIndex.FIELD_NAME_SUBJECT, urlAnalyzer);
		mapping.put(TripleIndex.FIELD_NAME_PREDICATE, urlAnalyzer);
		mapping.put(TripleIndex.FIELD_NAME_OBJECT_URI, urlAnalyzer);
		mapping.put(TripleIndex.FIELD_NAME_OBJECT_LITERAL, literalAnalyzer);
		PerFieldAnalyzerWrapper perFieldAnalyzer = new PerFieldAnalyzerWrapper(urlAnalyzer, mapping);

		File indexDirectory = new File(idxDirectory);
		indexDirectory.mkdir();
		directory = new MMapDirectory(indexDirectory);
		IndexWriterConfig config = new IndexWriterConfig(LUCENE_VERSION, perFieldAnalyzer);
		iwriter = new IndexWriter(directory, config);
		iwriter.commit();
		for (File file : files) {
			String type = FileUtil.getFileExtension(file.getName());
			if (type.equals(TTL))
				indexTTLFile(file, baseURI);
			if (type.equals(TSV))
				indexTSVFile(file);
			iwriter.commit();
		}
		iwriter.close();
		ireader = DirectoryReader.open(directory);
	} catch (Exception e) {
		log.error("Error while creating TripleIndex.", e);
	}
}
 

@Before
public void setUp() throws Exception {
    final Analyzer analyzer = new StandardAnalyzer();
    tempDirectory = Files.createTempDirectory("lucene");
    directory = new MMapDirectory(tempDirectory);

    IndexWriterConfig config = new IndexWriterConfig(analyzer);
    writer = new IndexWriter(directory, config);
    writer.commit();

    parser = new FiqlParser<>(SearchBean.class);
    extractor = new TikaLuceneContentExtractor(new PDFParser());
}
 

@Override
protected Directory create(String path, LockFactory lockFactory, DirContext dirContext) throws IOException {
  // we pass NoLockFactory, because the real lock factory is set later by injectLockFactory:
  MMapDirectory mapDirectory = new MMapDirectory(new File(path).toPath(), lockFactory, maxChunk);
  try {
    mapDirectory.setUseUnmap(unmapHack);
  } catch (IllegalArgumentException e) {
    log.warn("Unmap not supported on this JVM, continuing on without setting unmap", e);
  }
  mapDirectory.setPreload(preload);
  return mapDirectory;
}
 

@Override
@SuppressWarnings({"rawtypes"})
public void init(NamedList args) {
  super.init(args);
  SolrParams params = args.toSolrParams();
  maxChunk = params.getInt("maxChunkSize", MMapDirectory.DEFAULT_MAX_CHUNK_SIZE);
  if (maxChunk <= 0){
    throw new IllegalArgumentException("maxChunk must be greater than 0");
  }
  unmapHack = params.getBool("unmap", true);
  preload = params.getBool("preload", false); //default turn-off
}
 

private Directory getLuceneDir(Stargraph stargraph, KBId kbId) {
    return luceneDirs.computeIfAbsent(kbId,
            (id) -> {
                try {
                    String rootPath = stargraph.getDataRootDir();
                    Path idxPath = Paths.get(rootPath, id.getId(), id.getModel(), "idx");
                    return new MMapDirectory(idxPath);
                } catch (IOException e) {
                    throw new StarGraphException(e);
                }
            });
}
 

/**
 * Create a new empty search index.
 */
public BibleSearchIndex() {
    chapters = new HashMap<>();
    try {
        analyzer = CustomAnalyzer.builder()
                .withTokenizer(StandardTokenizerFactory.class)
                .addTokenFilter(LowerCaseFilterFactory.class)
                .addTokenFilter(ASCIIFoldingFilterFactory.class)
                .build();
        index = new MMapDirectory(Files.createTempDirectory("quelea-mmap-bible").toAbsolutePath());
    } catch (IOException ex) {
        LOGGER.log(Level.SEVERE, "Couldn't create song search index");
        throw new RuntimeException("Couldn't create song search index", ex);
    }
}
 

public void setIndex(String index) throws IOException {
	directory = new MMapDirectory(new File(index));
	ireader = DirectoryReader.open(directory);
	isearcher = new IndexSearcher(ireader);
}
 

@Nightly
public void test() throws Exception {
  assumeWorkingMMapOnWindows();
  
  MockDirectoryWrapper dir = new MockDirectoryWrapper(random(), new MMapDirectory(createTempDir("4GBStoredFields")));
  dir.setThrottling(MockDirectoryWrapper.Throttling.NEVER);

  IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
  iwc.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
  iwc.setRAMBufferSizeMB(256.0);
  iwc.setMergeScheduler(new ConcurrentMergeScheduler());
  iwc.setMergePolicy(newLogMergePolicy(false, 10));
  iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE);

  // TODO: we disable "Compressing" since it likes to pick very extreme values which will be too slow for this test.
  // maybe we should factor out crazy cases to ExtremeCompressing? then annotations can handle this stuff...
  if (random().nextBoolean()) {
    iwc.setCodec(CompressingCodec.reasonableInstance(random()));
  }

  IndexWriter w = new IndexWriter(dir, iwc);

  MergePolicy mp = w.getConfig().getMergePolicy();
  if (mp instanceof LogByteSizeMergePolicy) {
   // 1 petabyte:
   ((LogByteSizeMergePolicy) mp).setMaxMergeMB(1024*1024*1024);
  }

  final Document doc = new Document();
  final FieldType ft = new FieldType();
  ft.setStored(true);
  ft.freeze();
  final int valueLength = RandomNumbers.randomIntBetween(random(), 1 << 13, 1 << 20);
  final byte[] value = new byte[valueLength];
  for (int i = 0; i < valueLength; ++i) {
    // random so that even compressing codecs can't compress it
    value[i] = (byte) random().nextInt(256);
  }
  final Field f = new Field("fld", value, ft);
  doc.add(f);

  final int numDocs = (int) ((1L << 32) / valueLength + 100);
  for (int i = 0; i < numDocs; ++i) {
    w.addDocument(doc);
    if (VERBOSE && i % (numDocs / 10) == 0) {
      System.out.println(i + " of " + numDocs + "...");
    }
  }
  w.forceMerge(1);
  w.close();
  if (VERBOSE) {
    boolean found = false;
    for (String file : dir.listAll()) {
      if (file.endsWith(".fdt")) {
        final long fileLength = dir.fileLength(file);
        if (fileLength >= 1L << 32) {
          found = true;
        }
        System.out.println("File length of " + file + " : " + fileLength);
      }
    }
    if (!found) {
      System.out.println("No .fdt file larger than 4GB, test bug?");
    }
  }

  DirectoryReader rd = DirectoryReader.open(dir);
  Document sd = rd.document(numDocs - 1);
  assertNotNull(sd);
  assertEquals(1, sd.getFields().size());
  BytesRef valueRef = sd.getBinaryValue("fld");
  assertNotNull(valueRef);
  assertEquals(new BytesRef(value), valueRef);
  rd.close();

  dir.close();
}
 

/** Assumes that the current MMapDirectory implementation supports unmapping, so the test will not fail on Windows.
 * @see #hasWorkingMMapOnWindows()
 * */
public static void assumeWorkingMMapOnWindows() {
  assumeTrue(MMapDirectory.UNMAP_NOT_SUPPORTED_REASON, hasWorkingMMapOnWindows());
}
 

/** Returns true, if MMapDirectory supports unmapping on this platform (required for Windows), or if we are not on Windows. */
public static boolean hasWorkingMMapOnWindows() {
  return !Constants.WINDOWS || MMapDirectory.UNMAP_SUPPORTED;
}
 

@Nightly
public void testBigDocuments() throws IOException {
  assumeWorkingMMapOnWindows();
  
  // "big" as "much bigger than the chunk size"
  // for this test we force a FS dir
  // we can't just use newFSDirectory, because this test doesn't really index anything.
  // so if we get NRTCachingDir+SimpleText, we make massive stored fields and OOM (LUCENE-4484)
  Directory dir = new MockDirectoryWrapper(random(), new MMapDirectory(createTempDir("testBigDocuments")));
  IndexWriterConfig iwConf = newIndexWriterConfig(new MockAnalyzer(random()));
  iwConf.setMaxBufferedDocs(RandomNumbers.randomIntBetween(random(), 2, 30));
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConf);

  if (dir instanceof MockDirectoryWrapper) {
    ((MockDirectoryWrapper) dir).setThrottling(Throttling.NEVER);
  }

  final Document emptyDoc = new Document(); // emptyDoc
  final Document bigDoc1 = new Document(); // lot of small fields
  final Document bigDoc2 = new Document(); // 1 very big field

  final Field idField = new StringField("id", "", Store.NO);
  emptyDoc.add(idField);
  bigDoc1.add(idField);
  bigDoc2.add(idField);

  final FieldType onlyStored = new FieldType(StringField.TYPE_STORED);
  onlyStored.setIndexOptions(IndexOptions.NONE);

  final Field smallField = new Field("fld", randomByteArray(random().nextInt(10), 256), onlyStored);
  final int numFields = RandomNumbers.randomIntBetween(random(), 500000, 1000000);
  for (int i = 0; i < numFields; ++i) {
    bigDoc1.add(smallField);
  }

  final Field bigField = new Field("fld", randomByteArray(RandomNumbers.randomIntBetween(random(), 1000000, 5000000), 2), onlyStored);
  bigDoc2.add(bigField);

  final int numDocs = atLeast(5);
  final Document[] docs = new Document[numDocs];
  for (int i = 0; i < numDocs; ++i) {
    docs[i] = RandomPicks.randomFrom(random(), Arrays.asList(emptyDoc, bigDoc1, bigDoc2));
  }
  for (int i = 0; i < numDocs; ++i) {
    idField.setStringValue("" + i);
    iw.addDocument(docs[i]);
    if (random().nextInt(numDocs) == 0) {
      iw.commit();
    }
  }
  iw.commit();
  iw.forceMerge(1); // look at what happens when big docs are merged
  final DirectoryReader rd = maybeWrapWithMergingReader(DirectoryReader.open(dir));
  final IndexSearcher searcher = new IndexSearcher(rd);
  for (int i = 0; i < numDocs; ++i) {
    final Query query = new TermQuery(new Term("id", "" + i));
    final TopDocs topDocs = searcher.search(query, 1);
    assertEquals("" + i, 1, topDocs.totalHits.value);
    final Document doc = rd.document(topDocs.scoreDocs[0].doc);
    assertNotNull(doc);
    final IndexableField[] fieldValues = doc.getFields("fld");
    assertEquals(docs[i].getFields("fld").length, fieldValues.length);
    if (fieldValues.length > 0) {
      assertEquals(docs[i].getFields("fld")[0].binaryValue(), fieldValues[0].binaryValue());
    }
  }
  rd.close();
  iw.close();
  dir.close();
}
 

HybridDirectory(LockFactory lockFactory, MMapDirectory delegate) throws IOException {
    super(delegate.getDirectory(), lockFactory);
    this.delegate = delegate;
}
 

MMapDirectory getDelegate() {
    return delegate;
}
 

private void doTestStoreDirectory(Index index,
                                  Path tempDir,
                                  String typeSettingValue,
                                  IndexModule.Type type) throws IOException {
    Settings.Builder settingsBuilder = Settings.builder()
        .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
    if (typeSettingValue != null) {
        settingsBuilder.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), typeSettingValue);
    }
    Settings settings = settingsBuilder.build();
    IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings);
    FsDirectoryService service = new FsDirectoryService(
        indexSettings, new ShardPath(false, tempDir, tempDir, new ShardId(index, 0)));
    try (Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) {
        switch (type) {
            case HYBRIDFS:
                assertHybridDirectory(directory);
                break;
            case NIOFS:
                assertTrue(type + " " + directory.toString(), directory instanceof NIOFSDirectory);
                break;
            case MMAPFS:
                assertTrue(type + " " + directory.toString(), directory instanceof MMapDirectory);
                break;
            case SIMPLEFS:
                assertTrue(type + " " + directory.toString(), directory instanceof SimpleFSDirectory);
                break;
            case FS:
                if (Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
                    assertHybridDirectory(directory);
                } else if (Constants.WINDOWS) {
                    assertTrue(directory.toString(), directory instanceof SimpleFSDirectory);
                } else {
                    assertTrue(directory.toString(), directory instanceof NIOFSDirectory);
                }
                break;
            default:
                fail();
        }
    }
}
 

private void assertHybridDirectory(Directory directory) {
    assertTrue(directory.toString(), directory instanceof FsDirectoryService.HybridDirectory);
    Directory randomAccessDirectory = ((FsDirectoryService.HybridDirectory) directory).getDelegate();
    assertTrue("randomAccessDirectory:  " +  randomAccessDirectory.toString(), randomAccessDirectory instanceof MMapDirectory);
}