Java Code Examples for org.apache.lucene.index.IndexWriterConfig#setIndexDeletionPolicy()

@Override
@Before
public void setUp() throws Exception {
  super.setUp();
  publishIndexDir = newDirectory();
  publishTaxoDir = newDirectory();
  handlerIndexDir = newMockDirectory();
  handlerTaxoDir = newMockDirectory();
  clientWorkDir = createTempDir("replicationClientTest");
  sourceDirFactory = new PerSessionDirectoryFactory(clientWorkDir);
  replicator = new LocalReplicator();
  callback = new IndexAndTaxonomyReadyCallback(handlerIndexDir, handlerTaxoDir);
  handler = new IndexAndTaxonomyReplicationHandler(handlerIndexDir, handlerTaxoDir, callback);
  client = new ReplicationClient(replicator, handler, sourceDirFactory);
  
  IndexWriterConfig conf = newIndexWriterConfig(null);
  conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
  publishIndexWriter = new IndexWriter(publishIndexDir, conf);
  publishTaxoWriter = new SnapshotDirectoryTaxonomyWriter(publishTaxoDir);
  config = new FacetsConfig();
  config.setHierarchical("A", true);
}
 

@Test
public void testNoCommit() throws Exception {
  Directory indexDir = newDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(null);
  conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
  IndexWriter indexWriter = new IndexWriter(indexDir, conf);
  
  Directory taxoDir = newDirectory();
  SnapshotDirectoryTaxonomyWriter taxoWriter = new SnapshotDirectoryTaxonomyWriter(taxoDir);
  // should fail when there are no commits to snapshot
  expectThrows(IllegalStateException.class, () -> {
    new IndexAndTaxonomyRevision(indexWriter, taxoWriter);
  });

  indexWriter.close();
  IOUtils.close(taxoWriter, taxoDir, indexDir);
}
 

@Test
public void testSegmentsFileLast() throws Exception {
  Directory indexDir = newDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(null);
  conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
  IndexWriter indexWriter = new IndexWriter(indexDir, conf);
  
  Directory taxoDir = newDirectory();
  SnapshotDirectoryTaxonomyWriter taxoWriter = new SnapshotDirectoryTaxonomyWriter(taxoDir);
  try {
    indexWriter.addDocument(newDocument(taxoWriter));
    indexWriter.commit();
    taxoWriter.commit();
    Revision rev = new IndexAndTaxonomyRevision(indexWriter, taxoWriter);
    Map<String,List<RevisionFile>> sourceFiles = rev.getSourceFiles();
    assertEquals(2, sourceFiles.size());
    for (List<RevisionFile> files : sourceFiles.values()) {
      String lastFile = files.get(files.size() - 1).fileName;
      assertTrue(lastFile.startsWith(IndexFileNames.SEGMENTS));
    }
    indexWriter.close();
  } finally {
    IOUtils.close(indexWriter, taxoWriter, taxoDir, indexDir);
  }
}
 

@Before
@Override
public void setUp() throws Exception {
  super.setUp();
  if (VERBOSE) {
    System.setProperty("org.eclipse.jetty.LEVEL", "DEBUG"); // sets stderr logging to DEBUG level
  }
  clientWorkDir = createTempDir("httpReplicatorTest");
  handlerIndexDir = newDirectory();
  serverIndexDir = newDirectory();
  serverReplicator = new LocalReplicator();
  startServer();
  
  IndexWriterConfig conf = newIndexWriterConfig(null);
  conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
  writer = new IndexWriter(serverIndexDir, conf);
  reader = DirectoryReader.open(writer);
}
 

@Override
@Before
public void setUp() throws Exception {
  super.setUp();
  publishDir = newMockDirectory();
  handlerDir = newMockDirectory();
  sourceDirFactory = new PerSessionDirectoryFactory(createTempDir("replicationClientTest"));
  replicator = new LocalReplicator();
  callback = new IndexReadyCallback(handlerDir);
  handler = new IndexReplicationHandler(handlerDir, callback);
  client = new ReplicationClient(replicator, handler, sourceDirFactory);
  
  IndexWriterConfig conf = newIndexWriterConfig(null);
  conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
  publishWriter = new IndexWriter(publishDir, conf);
}
 

@Test
public void testSegmentsFileLast() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(null);
  conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
  IndexWriter writer = new IndexWriter(dir, conf);
  try {
    writer.addDocument(new Document());
    writer.commit();
    Revision rev = new IndexRevision(writer);
    @SuppressWarnings("unchecked")
    Map<String, List<RevisionFile>> sourceFiles = rev.getSourceFiles();
    assertEquals(1, sourceFiles.size());
    List<RevisionFile> files = sourceFiles.values().iterator().next();
    String lastFile = files.get(files.size() - 1).fileName;
    assertTrue(lastFile.startsWith(IndexFileNames.SEGMENTS));
    writer.close();
  } finally {
    IOUtils.close(dir);
  }
}
 

@Test
public void testRevisionRelease() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(null);
  conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
  IndexWriter writer = new IndexWriter(dir, conf);
  try {
    writer.addDocument(new Document());
    writer.commit();
    Revision rev1 = new IndexRevision(writer);
    // releasing that revision should not delete the files
    rev1.release();
    assertTrue(slowFileExists(dir, IndexFileNames.SEGMENTS + "_1"));
    
    rev1 = new IndexRevision(writer); // create revision again, so the files are snapshotted
    writer.addDocument(new Document());
    writer.commit();
    assertNotNull(new IndexRevision(writer));
    rev1.release(); // this release should trigger the delete of segments_1
    assertFalse(slowFileExists(dir, IndexFileNames.SEGMENTS + "_1"));
  } finally {
    IOUtils.close(writer, dir);
  }
}
 

@Test
public void testNoSnapshotDeletionPolicy() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(null);
  conf.setIndexDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
  IndexWriter writer = new IndexWriter(dir, conf);
  // should fail when IndexDeletionPolicy is not Snapshot
  expectThrows(IllegalArgumentException.class, () -> {
    new IndexRevision(writer);
  });

  writer.close();
  IOUtils.close(dir);
}
 

/**
 * This method deletes index files of the {@linkplain IndexCommit} for the specified generation number.
 *
 * @param core The Solr core
 * @param dir The index directory storing the snapshot.
 * @throws IOException in case of I/O errors.
 */

@SuppressWarnings({"try", "unused"})
private static void deleteSnapshotIndexFiles(SolrCore core, Directory dir, IndexDeletionPolicy delPolicy) throws IOException {
  IndexWriterConfig conf = core.getSolrConfig().indexConfig.toIndexWriterConfig(core);
  conf.setOpenMode(OpenMode.APPEND);
  conf.setMergePolicy(NoMergePolicy.INSTANCE);//Don't want to merge any commits here!
  conf.setIndexDeletionPolicy(delPolicy);
  conf.setCodec(core.getCodec());
  try (SolrIndexWriter iw = new SolrIndexWriter("SolrSnapshotCleaner", dir, conf)) {
    // Do nothing. The only purpose of opening index writer is to invoke the Lucene IndexDeletionPolicy#onInit
    // method so that we can cleanup the files associated with specified index commit.
    // Note the index writer creates a new commit during the close() operation (which is harmless).
  }
}
 

private Path createIndex() throws IOException {
  Path indexDir = createTempDir();

  Directory dir = newFSDirectory(indexDir);

  IndexWriterConfig config = new IndexWriterConfig(new MockAnalyzer(random()));
  config.setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE);
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, config);

  Document doc1 = new Document();
  doc1.add(newStringField("f1", "1", Field.Store.NO));
  writer.addDocument(doc1);

  writer.commit();

  Document doc2 = new Document();
  doc2.add(newStringField("f1", "2", Field.Store.NO));
  writer.addDocument(doc2);

  Document doc3 = new Document();
  doc3.add(newStringField("f1", "3", Field.Store.NO));
  writer.addDocument(doc3);

  writer.commit();

  writer.close();
  dir.close();

  return indexDir;
}
 

@Test
public void testSerialization() throws IOException {
  Directory dir = newDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(null);
  conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
  IndexWriter writer = new IndexWriter(dir, conf);
  writer.addDocument(new Document());
  writer.commit();
  Revision rev = new IndexRevision(writer);
  
  SessionToken session1 = new SessionToken("17", rev);
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  session1.serialize(new DataOutputStream(baos));
  byte[] b = baos.toByteArray();
  SessionToken session2 = new SessionToken(new DataInputStream(new ByteArrayInputStream(b)));
  assertEquals(session1.id, session2.id);
  assertEquals(session1.version, session2.version);
  assertEquals(1, session2.sourceFiles.size());
  assertEquals(session1.sourceFiles.size(), session2.sourceFiles.size());
  assertEquals(session1.sourceFiles.keySet(), session2.sourceFiles.keySet());
  List<RevisionFile> files1 = session1.sourceFiles.values().iterator().next();
  List<RevisionFile> files2 = session2.sourceFiles.values().iterator().next();
  assertEquals(files1, files2);

  writer.close();
  IOUtils.close(dir);
}
 

@Test
public void testOpen() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(null);
  conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
  IndexWriter writer = new IndexWriter(dir, conf);
  try {
    writer.addDocument(new Document());
    writer.commit();
    Revision rev = new IndexRevision(writer);
    @SuppressWarnings("unchecked")
    Map<String, List<RevisionFile>> sourceFiles = rev.getSourceFiles();
    String source = sourceFiles.keySet().iterator().next();
    for (RevisionFile file : sourceFiles.values().iterator().next()) {
      IndexInput src = dir.openInput(file.fileName, IOContext.READONCE);
      InputStream in = rev.open(source, file.fileName);
      assertEquals(src.length(), in.available());
      byte[] srcBytes = new byte[(int) src.length()];
      byte[] inBytes = new byte[(int) src.length()];
      int offset = 0;
      if (random().nextBoolean()) {
        int skip = random().nextInt(10);
        if (skip >= src.length()) {
          skip = 0;
        }
        in.skip(skip);
        src.seek(skip);
        offset = skip;
      }
      src.readBytes(srcBytes, offset, srcBytes.length - offset);
      in.read(inBytes, offset, inBytes.length - offset);
      assertArrayEquals(srcBytes, inBytes);
      IOUtils.close(src, in);
    }
    writer.close();
  } finally {
    IOUtils.close(dir);
  }
}
 

@Test
public void testNoCommit() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(null);
  conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
  IndexWriter writer = new IndexWriter(dir, conf);
  // should fail when there are no commits to snapshot"
  expectThrows(IllegalStateException.class, () -> {
    new IndexRevision(writer);
  });

  writer.close();
  IOUtils.close(dir);
}
 

@Before
@Override
public void setUp() throws Exception {
  super.setUp();
  sourceDir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(null);
  conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
  sourceWriter = new IndexWriter(sourceDir, conf);
  replicator = new LocalReplicator();
}
 

@Test
public void testRevisionRelease() throws Exception {
  Directory indexDir = newDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(null);
  conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
  IndexWriter indexWriter = new IndexWriter(indexDir, conf);
  
  Directory taxoDir = newDirectory();
  SnapshotDirectoryTaxonomyWriter taxoWriter = new SnapshotDirectoryTaxonomyWriter(taxoDir);
  try {
    indexWriter.addDocument(newDocument(taxoWriter));
    indexWriter.commit();
    taxoWriter.commit();
    Revision rev1 = new IndexAndTaxonomyRevision(indexWriter, taxoWriter);
    // releasing that revision should not delete the files
    rev1.release();
    assertTrue(slowFileExists(indexDir, IndexFileNames.SEGMENTS + "_1"));
    assertTrue(slowFileExists(taxoDir, IndexFileNames.SEGMENTS + "_1"));
    
    rev1 = new IndexAndTaxonomyRevision(indexWriter, taxoWriter); // create revision again, so the files are snapshotted
    indexWriter.addDocument(newDocument(taxoWriter));
    indexWriter.commit();
    taxoWriter.commit();
    assertNotNull(new IndexAndTaxonomyRevision(indexWriter, taxoWriter));
    rev1.release(); // this release should trigger the delete of segments_1
    assertFalse(slowFileExists(indexDir, IndexFileNames.SEGMENTS + "_1"));
    indexWriter.close();
  } finally {
    IOUtils.close(indexWriter, taxoWriter, taxoDir, indexDir);
  }
}
 

@Override
protected IndexWriterConfig createIndexWriterConfig(OpenMode openMode) {
  IndexWriterConfig conf = super.createIndexWriterConfig(openMode);
  sdp = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy());
  conf.setIndexDeletionPolicy(sdp);
  return conf;
}
 

/**
 * Constructor
 * @param fs
 * @param shard
 * @param tempDir
 * @param conf
 * @throws IOException
 */
public ShardWriter(FileSystem fs, Shard shard, String tempDir, Configuration conf)
    throws IOException {
  logger.info("Construct a shard writer");

  this.conf = conf;
  this.fs = fs;
  localFs = FileSystem.getLocal(conf);
  perm = new Path(shard.getDirectory());
  taxoPerm = new Path(shard.getDirectory() + ".taxonomy");
  String indexDir = tempDir + "/" + "index";
  String taxoDir = tempDir + "/" + "taxo";
  temp = new Path(indexDir);
  taxoTemp = new Path(taxoDir);

  if (localFs.exists(temp)) {
    File tempFile = new File(temp.getName());
    if (tempFile.exists()) {
      LindenReducer.deleteDir(tempFile);
    }
  }

  if (!fs.exists(perm)) {
    fs.mkdirs(perm);
  } else {
    moveToTrash(conf, perm);
    fs.mkdirs(perm);
  }

  if (!fs.exists(taxoPerm)) {
    fs.mkdirs(taxoPerm);
  } else {
    moveToTrash(conf, taxoPerm);
    fs.mkdirs(taxoPerm);
  }
  IndexWriterConfig config = new IndexWriterConfig(Version.LATEST, null);
  config.setIndexDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
  writer = new IndexWriter(FSDirectory.open(new File(indexDir)), config);
  taxoWriter = new DirectoryTaxonomyWriter(FSDirectory.open(new File(taxoDir)));
}
 

@Test
public void testCreateIndex() throws IOException {
  long s = System.nanoTime();
  IndexWriterConfig conf = new IndexWriterConfig(LuceneVersionConstant.LUCENE_VERSION, new KeywordAnalyzer());
  IndexDeletionPolicyReader indexDeletionPolicy = new IndexDeletionPolicyReader(
      new KeepOnlyLastCommitDeletionPolicy());
  conf.setIndexDeletionPolicy(indexDeletionPolicy);
  FSDirectory control = FSDirectory.open(fileControl);
  Directory dir = getControlDir(control, directory);
  // The serial merge scheduler can be useful for debugging.
  // conf.setMergeScheduler(new SerialMergeScheduler());
  IndexWriter writer = new IndexWriter(dir, conf);
  int numDocs = 1000;
  DirectoryReader reader = null;
  long gen = 0;
  for (int i = 0; i < 100; i++) {
    if (reader == null) {
      reader = DirectoryReader.open(writer, true);
      gen = reader.getIndexCommit().getGeneration();
      indexDeletionPolicy.register(gen);
    } else {
      DirectoryReader old = reader;
      reader = DirectoryReader.openIfChanged(old, writer, true);
      if (reader == null) {
        reader = old;
      } else {
        long newGen = reader.getIndexCommit().getGeneration();
        indexDeletionPolicy.register(newGen);
        indexDeletionPolicy.unregister(gen);
        old.close();
        gen = newGen;
      }
    }
    assertEquals(i * numDocs, reader.numDocs());
    IndexSearcher searcher = new IndexSearcher(reader);
    NumericRangeQuery<Integer> query = NumericRangeQuery.newIntRange("id", 42, 42, true, true);
    TopDocs topDocs = searcher.search(query, 10);
    assertEquals(i, topDocs.totalHits);
    addDocuments(writer, numDocs);
  }
  writer.close(false);
  reader.close();
  long e = System.nanoTime();
  System.out.println("Total time [" + (e - s) / 1000000.0 + " ms]");
}
 

@Test
public void testOpen() throws Exception {
  Directory indexDir = newDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(null);
  conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
  IndexWriter indexWriter = new IndexWriter(indexDir, conf);
  
  Directory taxoDir = newDirectory();
  SnapshotDirectoryTaxonomyWriter taxoWriter = new SnapshotDirectoryTaxonomyWriter(taxoDir);
  try {
    indexWriter.addDocument(newDocument(taxoWriter));
    indexWriter.commit();
    taxoWriter.commit();
    Revision rev = new IndexAndTaxonomyRevision(indexWriter, taxoWriter);
    for (Entry<String,List<RevisionFile>> e : rev.getSourceFiles().entrySet()) {
      String source = e.getKey();
      @SuppressWarnings("resource") // silly, both directories are closed in the end
      Directory dir = source.equals(IndexAndTaxonomyRevision.INDEX_SOURCE) ? indexDir : taxoDir;
      for (RevisionFile file : e.getValue()) {
        IndexInput src = dir.openInput(file.fileName, IOContext.READONCE);
        InputStream in = rev.open(source, file.fileName);
        assertEquals(src.length(), in.available());
        byte[] srcBytes = new byte[(int) src.length()];
        byte[] inBytes = new byte[(int) src.length()];
        int offset = 0;
        if (random().nextBoolean()) {
          int skip = random().nextInt(10);
          if (skip >= src.length()) {
            skip = 0;
          }
          in.skip(skip);
          src.seek(skip);
          offset = skip;
        }
        src.readBytes(srcBytes, offset, srcBytes.length - offset);
        in.read(inBytes, offset, inBytes.length - offset);
        assertArrayEquals(srcBytes, inBytes);
        IOUtils.close(src, in);
      }
    }
    indexWriter.close();
  } finally {
    IOUtils.close(indexWriter, taxoWriter, taxoDir, indexDir);
  }
}
 

private void createWriter() throws IOException {
  IndexWriterConfig config = new IndexWriterConfig(Version.LATEST, null);
  config.setIndexDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
  writer = new IndexWriter(dir, config);
  taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
}