org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy Java Examples

The following examples show how to use org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestMixedDirectory.java    From RDFS with Apache License 2.0 6 votes vote down vote up 
public void testMixedDirectoryAndPolicy() throws IOException {
  Directory readDir = new RAMDirectory();
  updateIndex(readDir, 0, numDocsPerUpdate,
      new KeepOnlyLastCommitDeletionPolicy());

  verify(readDir, numDocsPerUpdate);

  IndexOutput out =
      readDir.createOutput("_" + (numDocsPerUpdate / maxBufferedDocs + 2)
          + ".cfs");
  out.writeInt(0);
  out.close();

  Directory writeDir = new RAMDirectory();
  Directory mixedDir = new MixedDirectory(readDir, writeDir);
  updateIndex(mixedDir, numDocsPerUpdate, numDocsPerUpdate,
      new MixedDeletionPolicy());

  verify(readDir, numDocsPerUpdate);
  verify(mixedDir, 2 * numDocsPerUpdate);
}
Example #2
Source File: TestMixedDirectory.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up 
public void testMixedDirectoryAndPolicy() throws IOException {
  Directory readDir = new RAMDirectory();
  updateIndex(readDir, 0, numDocsPerUpdate,
      new KeepOnlyLastCommitDeletionPolicy());

  verify(readDir, numDocsPerUpdate);

  IndexOutput out =
      readDir.createOutput("_" + (numDocsPerUpdate / maxBufferedDocs + 2)
          + ".cfs");
  out.writeInt(0);
  out.close();

  Directory writeDir = new RAMDirectory();
  Directory mixedDir = new MixedDirectory(readDir, writeDir);
  updateIndex(mixedDir, numDocsPerUpdate, numDocsPerUpdate,
      new MixedDeletionPolicy());

  verify(readDir, numDocsPerUpdate);
  verify(mixedDir, 2 * numDocsPerUpdate);
}
Example #3
Source File: ShardWriter.java    From linden with Apache License 2.0 5 votes vote down vote up 
/**
 * Constructor
 * @param fs
 * @param shard
 * @param tempDir
 * @param conf
 * @throws IOException
 */
public ShardWriter(FileSystem fs, Shard shard, String tempDir, Configuration conf)
    throws IOException {
  logger.info("Construct a shard writer");

  this.conf = conf;
  this.fs = fs;
  localFs = FileSystem.getLocal(conf);
  perm = new Path(shard.getDirectory());
  taxoPerm = new Path(shard.getDirectory() + ".taxonomy");
  String indexDir = tempDir + "/" + "index";
  String taxoDir = tempDir + "/" + "taxo";
  temp = new Path(indexDir);
  taxoTemp = new Path(taxoDir);

  if (localFs.exists(temp)) {
    File tempFile = new File(temp.getName());
    if (tempFile.exists()) {
      LindenReducer.deleteDir(tempFile);
    }
  }

  if (!fs.exists(perm)) {
    fs.mkdirs(perm);
  } else {
    moveToTrash(conf, perm);
    fs.mkdirs(perm);
  }

  if (!fs.exists(taxoPerm)) {
    fs.mkdirs(taxoPerm);
  } else {
    moveToTrash(conf, taxoPerm);
    fs.mkdirs(taxoPerm);
  }
  IndexWriterConfig config = new IndexWriterConfig(Version.LATEST, null);
  config.setIndexDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
  writer = new IndexWriter(FSDirectory.open(new File(indexDir)), config);
  taxoWriter = new DirectoryTaxonomyWriter(FSDirectory.open(new File(taxoDir)));
}
Example #4
Source File: IndexRevisionTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up 
@Test
public void testNoSnapshotDeletionPolicy() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(null);
  conf.setIndexDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
  IndexWriter writer = new IndexWriter(dir, conf);
  // should fail when IndexDeletionPolicy is not Snapshot
  expectThrows(IllegalArgumentException.class, () -> {
    new IndexRevision(writer);
  });

  writer.close();
  IOUtils.close(dir);
}
Example #5
Source File: IntermediateForm.java    From RDFS with Apache License 2.0 5 votes vote down vote up 
private IndexWriter createWriter() throws IOException {
  IndexWriter writer =
      new IndexWriter(dir, false, null,
          new KeepOnlyLastCommitDeletionPolicy());
  writer.setUseCompoundFile(false);

  if (iconf != null) {
    int maxFieldLength = iconf.getIndexMaxFieldLength();
    if (maxFieldLength > 0) {
      writer.setMaxFieldLength(maxFieldLength);
    }
  }

  return writer;
}
Example #6
Source File: ShardWriter.java    From RDFS with Apache License 2.0 5 votes vote down vote up 
/**
 * Constructor
 * @param fs
 * @param shard
 * @param tempDir
 * @param iconf
 * @throws IOException
 */
public ShardWriter(FileSystem fs, Shard shard, String tempDir,
    IndexUpdateConfiguration iconf) throws IOException {
  LOG.info("Construct a shard writer");

  this.fs = fs;
  localFs = FileSystem.getLocal(iconf.getConfiguration());
  perm = new Path(shard.getDirectory());
  temp = new Path(tempDir);

  long initGeneration = shard.getGeneration();
  if (!fs.exists(perm)) {
    assert (initGeneration < 0);
    fs.mkdirs(perm);
  } else {
    restoreGeneration(fs, perm, initGeneration);
  }
  dir =
      new MixedDirectory(fs, perm, localFs, fs.startLocalOutput(perm, temp),
          iconf.getConfiguration());

  // analyzer is null because we only use addIndexes, not addDocument
  writer =
      new IndexWriter(dir, false, null,
          initGeneration < 0 ? new KeepOnlyLastCommitDeletionPolicy()
              : new MixedDeletionPolicy());
  setParameters(iconf);
}
Example #7
Source File: IntermediateForm.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up 
private IndexWriter createWriter() throws IOException {
  IndexWriter writer =
      new IndexWriter(dir, false, null,
          new KeepOnlyLastCommitDeletionPolicy());
  writer.setUseCompoundFile(false);

  if (iconf != null) {
    int maxFieldLength = iconf.getIndexMaxFieldLength();
    if (maxFieldLength > 0) {
      writer.setMaxFieldLength(maxFieldLength);
    }
  }

  return writer;
}
Example #8
Source File: ShardWriter.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up 
/**
 * Constructor
 * @param fs
 * @param shard
 * @param tempDir
 * @param iconf
 * @throws IOException
 */
public ShardWriter(FileSystem fs, Shard shard, String tempDir,
    IndexUpdateConfiguration iconf) throws IOException {
  LOG.info("Construct a shard writer");

  this.fs = fs;
  localFs = FileSystem.getLocal(iconf.getConfiguration());
  perm = new Path(shard.getDirectory());
  temp = new Path(tempDir);

  long initGeneration = shard.getGeneration();
  if (!fs.exists(perm)) {
    assert (initGeneration < 0);
    fs.mkdirs(perm);
  } else {
    restoreGeneration(fs, perm, initGeneration);
  }
  dir =
      new MixedDirectory(fs, perm, localFs, fs.startLocalOutput(perm, temp),
          iconf.getConfiguration());

  // analyzer is null because we only use addIndexes, not addDocument
  writer =
      new IndexWriter(dir, false, null,
          initGeneration < 0 ? new KeepOnlyLastCommitDeletionPolicy()
              : new MixedDeletionPolicy());
  setParameters(iconf);
}
Example #9
Source File: IntermediateForm.java    From linden with Apache License 2.0 4 votes vote down vote up 
private void createWriter() throws IOException {
  IndexWriterConfig config = new IndexWriterConfig(Version.LATEST, null);
  config.setIndexDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
  writer = new IndexWriter(dir, config);
  taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
}
Example #10
Source File: BaseDirectoryTestSuite.java    From incubator-retired-blur with Apache License 2.0 4 votes vote down vote up 
@Test
public void testCreateIndex() throws IOException {
  long s = System.nanoTime();
  IndexWriterConfig conf = new IndexWriterConfig(LuceneVersionConstant.LUCENE_VERSION, new KeywordAnalyzer());
  IndexDeletionPolicyReader indexDeletionPolicy = new IndexDeletionPolicyReader(
      new KeepOnlyLastCommitDeletionPolicy());
  conf.setIndexDeletionPolicy(indexDeletionPolicy);
  FSDirectory control = FSDirectory.open(fileControl);
  Directory dir = getControlDir(control, directory);
  // The serial merge scheduler can be useful for debugging.
  // conf.setMergeScheduler(new SerialMergeScheduler());
  IndexWriter writer = new IndexWriter(dir, conf);
  int numDocs = 1000;
  DirectoryReader reader = null;
  long gen = 0;
  for (int i = 0; i < 100; i++) {
    if (reader == null) {
      reader = DirectoryReader.open(writer, true);
      gen = reader.getIndexCommit().getGeneration();
      indexDeletionPolicy.register(gen);
    } else {
      DirectoryReader old = reader;
      reader = DirectoryReader.openIfChanged(old, writer, true);
      if (reader == null) {
        reader = old;
      } else {
        long newGen = reader.getIndexCommit().getGeneration();
        indexDeletionPolicy.register(newGen);
        indexDeletionPolicy.unregister(gen);
        old.close();
        gen = newGen;
      }
    }
    assertEquals(i * numDocs, reader.numDocs());
    IndexSearcher searcher = new IndexSearcher(reader);
    NumericRangeQuery<Integer> query = NumericRangeQuery.newIntRange("id", 42, 42, true, true);
    TopDocs topDocs = searcher.search(query, 10);
    assertEquals(i, topDocs.totalHits);
    addDocuments(writer, numDocs);
  }
  writer.close(false);
  reader.close();
  long e = System.nanoTime();
  System.out.println("Total time [" + (e - s) / 1000000.0 + " ms]");
}