org.apache.lucene.store.FilterDirectory Java Examples

/**
 * Tries to extract the leaf of the {@link Directory} if the directory is a {@link FilterDirectory} and cast
 * it to the given target class or returns the given default value, if the leaf is not assignable to the target class.
 * If the given {@link Directory} is a concrete directory it will treated as a leaf and the above applies.
 */
public static <T extends Directory> T getLeaf(Directory dir, Class<T> targetClass, T defaultValue) {
    Directory d = dir;
    if (dir instanceof FilterDirectory) {
        d = getLeafDirectory((FilterDirectory) dir, targetClass);
    }
    if (d instanceof FileSwitchDirectory) {
        T leaf = getLeaf(((FileSwitchDirectory) d).getPrimaryDir(), targetClass);
        if (leaf == null) {
            d = getLeaf(((FileSwitchDirectory) d).getSecondaryDir(), targetClass, defaultValue);
        } else {
            d = leaf;
        }
    }

    if (d != null && targetClass.isAssignableFrom(d.getClass())) {
        return targetClass.cast(d);
    } else {
        return defaultValue;
    }
}
 

public void testPassIOContext() throws IOException {
  final String testfile = "_123.test";
  final IOContext myContext = new IOContext();

  Directory dir = new FilterDirectory(newDirectory()) {
    @Override
    public IndexOutput createOutput(String name, IOContext context) throws IOException {
      assertSame(myContext, context);
      return super.createOutput(name, context);
    }
  };
  SegmentInfo si = newSegmentInfo(dir, "_123");
  try (IndexOutput out = dir.createOutput(testfile, myContext)) {
    CodecUtil.writeIndexHeader(out, "Foo", 0, si.getId(), "suffix");
    out.writeInt(3);
    CodecUtil.writeFooter(out);
  }
  
  si.setFiles(Collections.singleton(testfile));
  si.getCodec().compoundFormat().write(dir, si, myContext);
  dir.close();
}
 

public static boolean hasWindowsFS(Directory dir) {
  dir = FilterDirectory.unwrap(dir);
  if (dir instanceof FSDirectory) {
    Path path = ((FSDirectory) dir).getDirectory();
    FileSystem fs = path.getFileSystem();
    while (fs instanceof FilterFileSystem) {
      FilterFileSystem ffs = (FilterFileSystem) fs;
      if (ffs.getParent() instanceof WindowsFS) {
        return true;
      }
      fs = ffs.getDelegate();
    }
  }

  return false;
}
 

/** Returns true if VirusCheckingFS is in use and was in fact already enabled */
public static boolean disableVirusChecker(Directory in) {
  Directory dir = FilterDirectory.unwrap(in);
  if (dir instanceof FSDirectory) {

    FileSystem fs = ((FSDirectory) dir).getDirectory().getFileSystem();
    while (fs instanceof FilterFileSystem) {
      FilterFileSystem ffs = (FilterFileSystem) fs;
      if (ffs.getParent() instanceof VirusCheckingFS) {
        VirusCheckingFS vfs = (VirusCheckingFS) ffs.getParent();
        boolean isEnabled = vfs.isEnabled();
        vfs.disable();
        return isEnabled;
      }
      fs = ffs.getDelegate();
    }
  }

  return false;
}
 

public static void enableVirusChecker(Directory in) {
  Directory dir = FilterDirectory.unwrap(in);
  if (dir instanceof FSDirectory) {

    FileSystem fs = ((FSDirectory) dir).getDirectory().getFileSystem();
    while (fs instanceof FilterFileSystem) {
      FilterFileSystem ffs = (FilterFileSystem) fs;
      if (ffs.getParent() instanceof VirusCheckingFS) {
        VirusCheckingFS vfs = (VirusCheckingFS) ffs.getParent();
        vfs.enable();
        return;
      }
      fs = ffs.getDelegate();
    }
  }
}
 

/** Make sure corruption on the incoming (unsorted) file is caught, even if the corruption didn't confuse OfflineSorter! */
public void testBitFlippedOnInput1() throws Exception {

  try (Directory dir0 = newMockDirectory()) {

    Directory dir = new FilterDirectory(dir0) {
      @Override
      public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
        IndexOutput out = in.createTempOutput(prefix, suffix, context);
        if (prefix.equals("unsorted")) {
          return new CorruptingIndexOutput(dir0, 22, out);
        } else {
          return out;
        }
      }
    };

    IndexOutput unsorted = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
    writeAll(unsorted, generateFixed(10*1024));

    CorruptIndexException e = expectThrows(CorruptIndexException.class, () -> {
        new OfflineSorter(dir, "foo").sort(unsorted.getName());
      });
    assertTrue(e.getMessage().contains("checksum failed (hardware problem?)"));
  }
}
 

public void testPendingDeletionsRollbackWithReader() throws IOException {
  // irony: currently we don't emulate windows well enough to work on windows!
  assumeFalse("windows is not supported", Constants.WINDOWS);

  Path path = createTempDir();

  // Use WindowsFS to prevent open files from being deleted:
  FileSystem fs = new WindowsFS(path.getFileSystem()).getFileSystem(URI.create("file:///"));
  Path root = new FilterPath(path, fs);
  try (FSDirectory _dir = new NIOFSDirectory(root)) {
    Directory dir = new FilterDirectory(_dir) {};

    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
    IndexWriter w = new IndexWriter(dir, iwc);
    Document d = new Document();
    d.add(new StringField("id", "1", Field.Store.YES));
    d.add(new NumericDocValuesField("numval", 1));
    w.addDocument(d);
    w.commit();
    w.addDocument(d);
    w.flush();
    DirectoryReader reader = DirectoryReader.open(w);
    w.rollback();

    // try-delete superfluous files (some will fail due to windows-fs)
    IndexWriterConfig iwc2 = new IndexWriterConfig(new MockAnalyzer(random()));
    new IndexWriter(dir, iwc2).close();

    // test that we can index on top of pending deletions
    IndexWriterConfig iwc3 = new IndexWriterConfig(new MockAnalyzer(random()));
    w = new IndexWriter(dir, iwc3);
    w.addDocument(d);
    w.commit();

    reader.close();
    w.close();
  }
}
 

static Map<String, String> getReaderAttributes(Directory directory) {
    Directory unwrap = FilterDirectory.unwrap(directory);
    boolean defaultOffHeap = FsDirectoryService.isHybridFs(unwrap) || unwrap instanceof MMapDirectory;
    return Map.of(
        // if we are using MMAP for term dics we force all off heap
        BlockTreeTermsReader.FST_MODE_KEY,
        defaultOffHeap ? FSTLoadMode.OFF_HEAP.name() : FSTLoadMode.ON_HEAP.name()
    );
}
 

protected Directory getBaseDir(Directory dir) {
  Directory baseDir = dir;
  while (baseDir instanceof FilterDirectory) {
    baseDir = ((FilterDirectory)baseDir).getDelegate();
  } 
  
  return baseDir;
}
 

/** Make sure corruption on a temp file (partition) is caught, even if the corruption didn't confuse OfflineSorter! */
public void testBitFlippedOnPartition1() throws Exception {

  try (Directory dir0 = newMockDirectory()) {

    Directory dir = new FilterDirectory(dir0) {

      boolean corrupted;

      @Override
      public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
        IndexOutput out = in.createTempOutput(prefix, suffix, context);
        if (corrupted == false && suffix.equals("sort")) {
          corrupted = true;
          return new CorruptingIndexOutput(dir0, 544677, out);
        } else {
          return out;
        }
      }
    };

    IndexOutput unsorted = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
    writeAll(unsorted, generateFixed((int) (OfflineSorter.MB * 3)));

    CorruptIndexException e = expectThrows(CorruptIndexException.class, () -> {
        new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), 10, -1, null, 0).sort(unsorted.getName());
      });
    assertTrue(e.getMessage().contains("checksum failed (hardware problem?)"));
  }
}
 

static final <T extends Directory> Directory getLeafDirectory(FilterDirectory dir, Class<T> targetClass) {
    Directory current = dir.getDelegate();
    while (true) {
        if ((current instanceof FilterDirectory)) {
            if (targetClass != null && targetClass.isAssignableFrom(current.getClass())) {
                break;
            }
            current = ((FilterDirectory) current).getDelegate();
        } else {
            break;
        }
    }
    return current;
}
 

/** If the dir is an {@link FSDirectory} or wraps one via possibly
 *  nested {@link FilterDirectory} or {@link FileSwitchDirectory},
 *  this returns {@link #spins(Path)} for the wrapped directory,
 *  else, true.
 *
 *  @throws IOException if {@code path} does not exist.
 *
 *  @lucene.internal */
public static boolean spins(Directory dir) throws IOException {
  dir = FilterDirectory.unwrap(dir);
  if (dir instanceof FileSwitchDirectory) {
    FileSwitchDirectory fsd = (FileSwitchDirectory) dir;
    // Spinning is contagious:
    return spins(fsd.getPrimaryDir()) || spins(fsd.getSecondaryDir());
  } else if (dir instanceof ByteBuffersDirectory) {
    return false;
  } else if (dir instanceof FSDirectory) {
    return spins(((FSDirectory) dir).getDirectory());
  } else {
    return true;
  }
}
 

public static boolean hasVirusChecker(Directory dir) {
  dir = FilterDirectory.unwrap(dir);
  if (dir instanceof FSDirectory) {
    return hasVirusChecker(((FSDirectory) dir).getDirectory());
  } else {
    return false;
  }
}
 

/** Make sure corruption on the incoming (unsorted) file is caught, if the corruption did confuse OfflineSorter! */
public void testBitFlippedOnInput2() throws Exception {

  try (Directory dir0 = newMockDirectory()) {

    Directory dir = new FilterDirectory(dir0) {
      @Override
      public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
        IndexOutput out = in.createTempOutput(prefix, suffix, context);
        if (prefix.equals("unsorted")) {
          return new CorruptingIndexOutput(dir0, 22, out) {
            @Override
            protected void corruptFile() throws IOException {
              String newTempName;
              try(IndexOutput tmpOut = dir0.createTempOutput("tmp", "tmp", IOContext.DEFAULT);
                  IndexInput in = dir0.openInput(out.getName(), IOContext.DEFAULT)) {
                newTempName = tmpOut.getName();
                // Replace length at the end with a too-long value:
                short v = in.readShort();
                assertEquals(256, v);
                tmpOut.writeShort(Short.MAX_VALUE);
                tmpOut.copyBytes(in, in.length()-Short.BYTES);
              }

              // Delete original and copy corrupt version back:
              dir0.deleteFile(out.getName());
              dir0.copyFrom(dir0, newTempName, out.getName(), IOContext.DEFAULT);
              dir0.deleteFile(newTempName);
            }
          };
        } else {
          return out;
        }
      }
    };

    IndexOutput unsorted = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
    writeAll(unsorted, generateFixed(5*1024));

    // This corruption made OfflineSorter fail with its own exception, but we verify and throw a CorruptIndexException
    // instead when checksums don't match.
    CorruptIndexException e = expectThrows(CorruptIndexException.class, () -> {
        new OfflineSorter(dir, "foo").sort(unsorted.getName());
      });
    assertTrue(e.getMessage().contains("checksum failed (hardware problem?)"));
  }
}
 

/** 
 * LUCENE-6299: Test if addindexes(Dir[]) prevents exceeding max docs.
 */
// TODO: can we use the setter to lower the amount of docs to be written here?
@Nightly
public void testAddTooManyIndexesDir() throws Exception {
  // we cheat and add the same one over again... IW wants a write lock on each
  Directory dir = newDirectory(random(), NoLockFactory.INSTANCE);
  Document doc = new Document();
  IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));
  for (int i = 0; i < 100000; i++) {
    w.addDocument(doc);
  }
  w.forceMerge(1);
  w.commit();
  w.close();
  
  // wrap this with disk full, so test fails faster and doesn't fill up real disks.
  MockDirectoryWrapper dir2 = newMockDirectory();
  w = new IndexWriter(dir2, new IndexWriterConfig(null));
  w.commit(); // don't confuse checkindex
  dir2.setMaxSizeInBytes(dir2.sizeInBytes() + 65536); // 64KB
  Directory dirs[] = new Directory[1 + (IndexWriter.MAX_DOCS / 100000)];
  for (int i = 0; i < dirs.length; i++) {
    // bypass iw check for duplicate dirs
    dirs[i] = new FilterDirectory(dir) {};
  }

  try {
    w.addIndexes(dirs);
    fail("didn't get expected exception");
  } catch (IllegalArgumentException expected) {
    // pass
  } catch (IOException fakeDiskFull) {
    final Exception e;
    if (fakeDiskFull.getMessage() != null && fakeDiskFull.getMessage().startsWith("fake disk full")) {
      e = new RuntimeException("test failed: IW checks aren't working and we are executing addIndexes");
      e.addSuppressed(fakeDiskFull);
    } else {
      e = fakeDiskFull;
    }
    throw e;
  }
  
  w.close();
  dir.close();
  dir2.close();
}
 

/** Make sure corruption on a temp file (partition) is caught, if the corruption did confuse OfflineSorter! */
public void testBitFlippedOnPartition2() throws Exception {

  try (Directory dir0 = newMockDirectory()) {

    Directory dir = new FilterDirectory(dir0) {

      boolean corrupted;

      @Override
      public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
        IndexOutput out = in.createTempOutput(prefix, suffix, context);
        if (corrupted == false && suffix.equals("sort")) {
          corrupted = true;
          return new CorruptingIndexOutput(dir0, 544677, out) {
            @Override
            protected void corruptFile() throws IOException {
              String newTempName;
              try(IndexOutput tmpOut = dir0.createTempOutput("tmp", "tmp", IOContext.DEFAULT);
                  IndexInput in = dir0.openInput(out.getName(), IOContext.DEFAULT)) {
                newTempName = tmpOut.getName();
                tmpOut.copyBytes(in, 1025905);
                short v = in.readShort();
                assertEquals(254, v);
                tmpOut.writeShort(Short.MAX_VALUE);
                tmpOut.copyBytes(in, in.length()-1025905-Short.BYTES);
              }

              // Delete original and copy corrupt version back:
              dir0.deleteFile(out.getName());
              dir0.copyFrom(dir0, newTempName, out.getName(), IOContext.DEFAULT);
              dir0.deleteFile(newTempName);
            }
          };
        } else {
          return out;
        }
      }
    };

    IndexOutput unsorted = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
    writeAll(unsorted, generateFixed((int) (OfflineSorter.MB * 3)));

    CorruptIndexException e = expectThrows(CorruptIndexException.class, () -> {
        new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), 10, -1, null, 0).sort(unsorted.getName());
      });
    assertTrue(e.getMessage().contains("checksum failed (hardware problem?)"));
  }
}
 

/** Make sure corruption on an input sort file is caught, even if BKDWriter doesn't get angry */
public void testBitFlippedOnPartition1() throws Exception {

  // Generate fixed data set:
  int numDocs = atLeast(10000);
  int numBytesPerDim = 4;
  int numDims = 3;

  byte[][][] docValues = new byte[numDocs][][];
  byte counter = 0;

  for(int docID=0;docID<numDocs;docID++) {
    byte[][] values = new byte[numDims][];
    for(int dim=0;dim<numDims;dim++) {
      values[dim] = new byte[numBytesPerDim];
      for(int i=0;i<values[dim].length;i++) {
        values[dim][i] = counter;
        counter++;
      }
    }
    docValues[docID] = values;
  }

  try (Directory dir0 = newMockDirectory()) {

    Directory dir = new FilterDirectory(dir0) {
      boolean corrupted;
      @Override
      public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
        IndexOutput out = in.createTempOutput(prefix, suffix, context);
        if (corrupted == false && prefix.equals("_0") && suffix.equals("bkd_left0")) {
          corrupted = true;
          return new CorruptingIndexOutput(dir0, 22, out);
        } else {
          return out;
        }
      }
    };

    CorruptIndexException e = expectThrows(CorruptIndexException.class, () -> {
        verify(dir, docValues, null, numDims, numDims, numBytesPerDim, 50, 0.1);
      });
    assertTrue(e.getMessage().contains("checksum failed (hardware problem?)"));
  }
}
 

/** Make sure corruption on a recursed partition is caught, when BKDWriter does get angry */
public void testBitFlippedOnPartition2() throws Exception {

  // Generate fixed data set:
  int numDocs = atLeast(10000);
  int numBytesPerDim = 4;
  int numDims = 3;

  byte[][][] docValues = new byte[numDocs][][];
  byte counter = 0;

  for(int docID=0;docID<numDocs;docID++) {
    byte[][] values = new byte[numDims][];
    for(int dim=0;dim<numDims;dim++) {
      values[dim] = new byte[numBytesPerDim];
      for(int i=0;i<values[dim].length;i++) {
        values[dim][i] = counter;
        counter++;
      }
    }
    docValues[docID] = values;
  }

  try (Directory dir0 = newMockDirectory()) {

    Directory dir = new FilterDirectory(dir0) {
      boolean corrupted;
      @Override
      public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
        IndexOutput out = in.createTempOutput(prefix, suffix, context);
        //System.out.println("prefix=" + prefix + " suffix=" + suffix);
        if (corrupted == false && suffix.equals("bkd_left0")) {
          //System.out.println("now corrupt byte=" + x + " prefix=" + prefix + " suffix=" + suffix);
          corrupted = true;
          return new CorruptingIndexOutput(dir0, 22072, out);
        } else {
          return out;
        }
      }
    };

    Throwable t = expectThrows(CorruptIndexException.class, () -> {
        verify(dir, docValues, null, numDims, numDims, numBytesPerDim, 50, 0.1);
      });
    assertCorruptionDetected(t);
  }
}
 

public static boolean isHybridFs(Directory directory) {
    Directory unwrap = FilterDirectory.unwrap(directory);
    return unwrap instanceof HybridDirectory;
}