Java Code Examples for org.apache.lucene.store.IndexOutput#close()

public void testCheckFooterValid() throws Exception {
  ByteBuffersDataOutput out = new ByteBuffersDataOutput();
  IndexOutput output = new ByteBuffersIndexOutput(out, "temp", "temp");
  CodecUtil.writeHeader(output, "FooBar", 5);
  output.writeString("this is the data");
  CodecUtil.writeFooter(output);
  output.close();
  
  ChecksumIndexInput input = new BufferedChecksumIndexInput(new ByteBuffersIndexInput(out.toDataInput(), "temp"));
  Exception mine = new RuntimeException("fake exception");
  RuntimeException expected = expectThrows(RuntimeException.class, () -> {
    CodecUtil.checkFooter(input, mine);
  });
  assertEquals("fake exception", expected.getMessage());
  Throwable suppressed[] = expected.getSuppressed();
  assertEquals(1, suppressed.length);
  assertTrue(suppressed[0].getMessage().contains("checksum passed"));
  input.close();
}
 

/** test exception is delivered if you add the wrong number of values */
public void testNotEnoughValues() throws Exception {
  Directory dir = newDirectory();
  int bitsPerValue = DirectWriter.bitsRequired(2);
  IndexOutput output = dir.createOutput("foo", IOContext.DEFAULT);
  DirectWriter writer = DirectWriter.getInstance(output, 5, bitsPerValue);
  writer.add(1);
  writer.add(0);
  writer.add(2);
  writer.add(1);
  IllegalStateException expected = expectThrows(IllegalStateException.class, () -> {
    writer.finish();
  });
  assertTrue(expected.getMessage().startsWith("Wrong number of values added"));

  output.close();
  dir.close();
}
 

private static Directory copyFilesLocally(Configuration configuration, Directory dir, String table, Path shardDir,
    Path localCachePath, Collection<String> files, String segmentName) throws IOException {
  LOG.info("Copying files need to local cache for faster reads [{0}].", shardDir);
  Path localShardPath = new Path(new Path(new Path(localCachePath, table), shardDir.getName()), segmentName);
  HdfsDirectory localDir = new HdfsDirectory(configuration, localShardPath, null);
  for (String name : files) {
    if (!isValidFileToCache(name)) {
      continue;
    }
    LOG.info("Valid file for local copy [{0}].", name);
    if (!isValid(localDir, dir, name)) {
      LastModified lastModified = (LastModified) dir;
      long fileModified = lastModified.getFileModified(name);

      IndexInput input = dir.openInput(name, IOContext.READONCE);
      IndexOutput output = localDir.createOutput(name, IOContext.READONCE);
      output.copyBytes(input, input.length());
      output.close();
      IndexOutput lastMod = localDir.createOutput(name + LASTMOD, IOContext.DEFAULT);
      lastMod.writeLong(fileModified);
      lastMod.close();
    }
  }
  return localDir;
}
 

public void testReadBogusCRC() throws Exception {
  ByteBuffersDataOutput out = new ByteBuffersDataOutput();
  IndexOutput output = new ByteBuffersIndexOutput(out, "temp", "temp");
  output.writeLong(-1L); // bad
  output.writeLong(1L << 32); // bad
  output.writeLong(-(1L << 32)); // bad
  output.writeLong((1L << 32) - 1); // ok
  output.close();
  IndexInput input = new BufferedChecksumIndexInput(new ByteBuffersIndexInput(out.toDataInput(), "temp"));
  // read 3 bogus values
  for (int i = 0; i < 3; i++) {
    expectThrows(CorruptIndexException.class, () -> {
      CodecUtil.readCRC(input);
    });
  }
  // good value
  CodecUtil.readCRC(input);
}
 

public void testWriteVeryLongSuffix() throws Exception {
  StringBuilder justLongEnough = new StringBuilder();
  for (int i = 0; i < 255; i++) {
    justLongEnough.append('a');
  }
  ByteBuffersDataOutput out = new ByteBuffersDataOutput();
  IndexOutput output = new ByteBuffersIndexOutput(out, "temp", "temp");
  byte[] id = StringHelper.randomId();
  CodecUtil.writeIndexHeader(output, "foobar", 5, id, justLongEnough.toString());
  output.close();
  
  IndexInput input = new ByteBuffersIndexInput(out.toDataInput(), "temp");
  CodecUtil.checkIndexHeader(input, "foobar", 5, 5, id, justLongEnough.toString());
  assertEquals(input.getFilePointer(), input.length());
  assertEquals(input.getFilePointer(), CodecUtil.indexHeaderLength("foobar", justLongEnough.toString()));
  input.close();
}
 

@Test
public void test2() throws IOException {
  Cache cache = CacheIndexInputTest.getCache();
  RAMDirectory directory = new RAMDirectory();
  RAMDirectory directory2 = new RAMDirectory();

  Random random = new Random(seed);

  String name = "test2";
  long size = (10 * 1024 * 1024) + 13;

  IndexOutput output = directory.createOutput(name, IOContext.DEFAULT);
  CacheIndexOutput cacheIndexOutput = new CacheIndexOutput(null, name, cache, directory2, IOContext.DEFAULT);
  CacheIndexInputTest.writeRandomData(size, random, output, cacheIndexOutput);
  output.close();
  cacheIndexOutput.close();

  IndexInput input = directory.openInput(name, IOContext.DEFAULT);
  IndexInput testInput = directory2.openInput(name, IOContext.DEFAULT);
  CacheIndexInputTest.readRandomData(input, testInput, random, sampleSize, maxBufSize, maxOffset);
  testInput.close();
  input.close();
  directory.close();
  directory2.close();
}
 

@Test
public void testMultiInstancesHdfsDirectoryTest1() throws IOException, InterruptedException {
  HdfsDirectory dir1 = new HdfsDirectory(_configuration, new Path(_root, "dir"));
  

  IndexOutput output = dir1.createOutput("a", IOContext.DEFAULT);
  output.writeInt(1234);
  output.close();
  
  HdfsDirectory dir2 = new HdfsDirectory(_configuration, new Path(_root, "dir"));

  IndexInput input = dir2.openInput("a", IOContext.READ);
  assertEquals(4, input.length());
  assertEquals(1234, input.readInt());
  input.close();

  dir1.close();
  dir2.close();
}
 

public void testLeftoverTempFiles() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter w = new IndexWriter(dir, iwc);
  w.close();

  IndexOutput out = dir.createTempOutput("_0", "bkd", IOContext.DEFAULT);
  String tempName = out.getName();
  out.close();
  iwc = new IndexWriterConfig(new MockAnalyzer(random()));
  w = new IndexWriter(dir, iwc);

  // Make sure IW deleted the unref'd file:
  try {
    dir.openInput(tempName, IOContext.DEFAULT);
    fail("did not hit exception");
  } catch (FileNotFoundException | NoSuchFileException e) {
    // expected
  }
  w.close();
  dir.close();
}
 

public void testOtherFiles() throws Throwable {
  Directory dir = newDirectory();
  IndexWriter iw = new IndexWriter(dir,
      newIndexWriterConfig(new MockAnalyzer(random())));
  iw.addDocument(new Document());
  iw.close();
  try {
    // Create my own random file:
    IndexOutput out = dir.createOutput("myrandomfile", newIOContext(random()));
    out.writeByte((byte) 42);
    out.close();

    new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))).close();

    assertTrue(slowFileExists(dir, "myrandomfile"));
  } finally {
    dir.close();
  }
}
 

/** 
 * Convert a WordSpace text matrix to a bin WordSpace file
 * Text matrix format:
 * - the first line contains the matrix dimensions N
 * - each line contains the word vector information: word d1 d2 ... dN
 * Text2Bin text_matrix_file bin_matrix_file
 * @param args the command line arguments
 */
public static void main(String[] args) {
    try {
        BufferedReader in = new BufferedReader(new FileReader(args[0]));
        File file = new File(args[1]);
        FSDirectory fs = FSDirectory.open(file.getParentFile());
        IndexOutput output = fs.createOutput(file.getName());
        String header = in.readLine();
        output.writeString("-dimensions");
        output.writeInt(Integer.parseInt(header));
        while (in.ready()) {
            String line = in.readLine();
            String[] split = line.split("\t");
            output.writeString(split[0]);
            for (int i=1;i<split.length;i++) {
                output.writeInt(Float.floatToIntBits(Float.parseFloat(split[i])));
            }
        }
        in.close();
        output.close();
    } catch (IOException ex) {
        Logger.getLogger(Text2Bin.class.getName()).log(Level.SEVERE, null, ex);
    }
}
 

public void testMixedDirectoryAndPolicy() throws IOException {
  Directory readDir = new RAMDirectory();
  updateIndex(readDir, 0, numDocsPerUpdate,
      new KeepOnlyLastCommitDeletionPolicy());

  verify(readDir, numDocsPerUpdate);

  IndexOutput out =
      readDir.createOutput("_" + (numDocsPerUpdate / maxBufferedDocs + 2)
          + ".cfs");
  out.writeInt(0);
  out.close();

  Directory writeDir = new RAMDirectory();
  Directory mixedDir = new MixedDirectory(readDir, writeDir);
  updateIndex(mixedDir, numDocsPerUpdate, numDocsPerUpdate,
      new MixedDeletionPolicy());

  verify(readDir, numDocsPerUpdate);
  verify(mixedDir, 2 * numDocsPerUpdate);
}
 

/** simple encode/decode */
public void testSimple() throws Exception {
  Directory dir = newDirectory();
  int bitsPerValue = DirectWriter.bitsRequired(2);
  IndexOutput output = dir.createOutput("foo", IOContext.DEFAULT);
  DirectWriter writer = DirectWriter.getInstance(output, 5, bitsPerValue);
  writer.add(1);
  writer.add(0);
  writer.add(2);
  writer.add(1);
  writer.add(2);
  writer.finish();
  output.close();
  IndexInput input = dir.openInput("foo", IOContext.DEFAULT);
  LongValues reader = DirectReader.getInstance(input.randomAccessSlice(0, input.length()), bitsPerValue, 0);
  assertEquals(1, reader.get(0));
  assertEquals(0, reader.get(1));
  assertEquals(2, reader.get(2));
  assertEquals(1, reader.get(3));
  assertEquals(2, reader.get(4));
  input.close();
  dir.close();
}
 

private void write(Directory directory) throws IOException {

    long nextGeneration = getNextPendingGeneration();
    String segmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.PENDING_SEGMENTS,
                                                                   "",
                                                                   nextGeneration);

    // Always advance the generation on write:
    generation = nextGeneration;
    
    IndexOutput segnOutput = null;
    boolean success = false;

    try {
      segnOutput = directory.createOutput(segmentFileName, IOContext.DEFAULT);
      write(segnOutput);
      segnOutput.close();
      directory.sync(Collections.singleton(segmentFileName));
      success = true;
    } finally {
      if (success) {
        pendingCommit = true;
      } else {
        // We hit an exception above; try to close the file
        // but suppress any exception:
        IOUtils.closeWhileHandlingException(segnOutput);
        // Try not to leave a truncated segments_N file in
        // the index:
        IOUtils.deleteFilesIgnoringExceptions(directory, segmentFileName);
      }
    }
  }
 

public void create(DocIdSetIterator it) throws IOException {
  String fileName = getFileName();
  if (_directory.fileExists(getFileName())) {
    LOG.warn("Filter [{0}] in directory [{1}] being recreated due to incorrect size.", fileName, _directory);
    _directory.deleteFile(fileName);
  }
  IndexOutput output = _directory.createOutput(fileName, IOContext.READ);
  int index;
  int currentWordNum = 0;
  long wordValue = 0;
  while ((index = it.nextDoc()) < _numBits) {
    int wordNum = index >> 6; // div 64
    if (currentWordNum > wordNum) {
      throw new IOException("We got a problem here!");
    }
    while (currentWordNum < wordNum) {
      output.writeLong(wordValue);
      currentWordNum++;
      wordValue = 0;
    }
    int bit = index & 0x3f; // mod 64
    long bitmask = 1L << bit;
    wordValue |= bitmask;
  }
  if (_numBits > 0) {
    int totalWords = (_numBits / 64) + 1;
    while (currentWordNum < totalWords) {
      output.writeLong(wordValue);
      currentWordNum++;
      wordValue = 0;
    }
  }
  output.close();
}
 

@Ignore // memory hole
public void testOverflow() throws IOException {
  BaseDirectoryWrapper dir = newFSDirectory(createTempDir("testOverflow"));
  if (dir instanceof MockDirectoryWrapper) {
    ((MockDirectoryWrapper)dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
  }
  final int blockBits = TestUtil.nextInt(random(), 14, 28);
  final int blockSize = 1 << blockBits;
  byte[] arr = new byte[TestUtil.nextInt(random(), blockSize / 2, blockSize * 2)];
  for (int i = 0; i < arr.length; ++i) {
    arr[i] = (byte) i;
  }
  final long numBytes = (1L << 31) + TestUtil.nextInt(random(), 1, blockSize * 3);
  final PagedBytes p = new PagedBytes(blockBits);
  final IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT);
  for (long i = 0; i < numBytes; ) {
    assertEquals(i, out.getFilePointer());
    final int len = (int) Math.min(arr.length, numBytes - i);
    out.writeBytes(arr, len);
    i += len;
  }
  assertEquals(numBytes, out.getFilePointer());
  out.close();
  final IndexInput in = dir.openInput("foo", IOContext.DEFAULT);
  p.copy(in, numBytes);
  final PagedBytes.Reader reader = p.freeze(random().nextBoolean());

  for (long offset : new long[] {0L, Integer.MAX_VALUE, numBytes - 1,
      TestUtil.nextLong(random(), 1, numBytes - 2)}) {
    BytesRef b = new BytesRef();
    reader.fillSlice(b, offset, 1);
    assertEquals(arr[(int) (offset % arr.length)], b.bytes[b.offset]);
  }
  in.close();
  dir.close();
}
 

public void testTruncatedFileThrowsCorruptIndexException() throws IOException {
  ByteBuffersDataOutput out = new ByteBuffersDataOutput();
  IndexOutput output = new ByteBuffersIndexOutput(out, "temp", "temp");
  output.close();

  IndexInput input = new ByteBuffersIndexInput(out.toDataInput(), "temp");
  
  CorruptIndexException e = expectThrows(CorruptIndexException.class,
      () -> CodecUtil.checksumEntireFile(input));
  assertTrue(e.getMessage(), e.getMessage().contains("misplaced codec footer (file truncated?): length=0 but footerLength==16 (resource"));

  e = expectThrows(CorruptIndexException.class,
      () -> CodecUtil.retrieveChecksum(input));
  assertTrue(e.getMessage(), e.getMessage().contains("misplaced codec footer (file truncated?): length=0 but footerLength==16 (resource"));
}
 

@Override
public void write(Directory dir, SegmentInfo si, FieldInfos fis, IOContext ioContext) throws IOException {
  final String fileName = IndexFileNames.segmentFileName(si.name, "", Blur022SegmentInfoFormat.SI_EXTENSION);
  si.addFile(fileName);

  final IndexOutput output = dir.createOutput(fileName, ioContext);

  boolean success = false;
  try {
    CodecUtil.writeHeader(output, Blur022SegmentInfoFormat.CODEC_NAME, Blur022SegmentInfoFormat.VERSION_CURRENT);
    output.writeString(si.getVersion());
    output.writeInt(si.getDocCount());

    output.writeByte((byte) (si.getUseCompoundFile() ? SegmentInfo.YES : SegmentInfo.NO));
    output.writeStringStringMap(si.getDiagnostics());
    Map<String, String> attributes = si.attributes();
    TreeMap<String, String> newAttributes = new TreeMap<String, String>();
    if (attributes != null) {
      newAttributes.putAll(attributes);
    }
    newAttributes.put(Blur022StoredFieldsFormat.STORED_FIELDS_FORMAT_CHUNK_SIZE,
        Integer.toString(_compressionChunkSize));
    newAttributes.put(Blur022StoredFieldsFormat.STORED_FIELDS_FORMAT_COMPRESSION_MODE, _compressionMode);
    output.writeStringStringMap(newAttributes);
    output.writeStringSet(si.files());

    success = true;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(output);
      si.dir.deleteFile(fileName);
    } else {
      output.close();
    }
  }
}
 

private void trainIncrementalDocVectors() throws IOException {
  int numdocs = luceneUtils.getNumDocs();

  // Open file and write headers.
  File vectorFile = new File(
      VectorStoreUtils.getStoreFileName(flagConfig.docvectorsfile(), flagConfig));
  String parentPath = vectorFile.getParent();
  if (parentPath == null) parentPath = "";
  FSDirectory fsDirectory = FSDirectory.open(FileSystems.getDefault().getPath(parentPath));

  java.nio.file.Files.deleteIfExists(vectorFile.toPath());
  
  IndexOutput outputStream = fsDirectory.createOutput(vectorFile.getName(), IOContext.DEFAULT);

  VerbatimLogger.info("Writing vectors incrementally to file " + vectorFile + " ... ");

  // Write header giving number of dimension for all vectors.
  outputStream.writeString(VectorStoreWriter.generateHeaderString(flagConfig));

  // Iterate through documents.
  for (int dc = 0; dc < numdocs; dc++) {
    // Output progress counter.
    if ((dc > 0) && ((dc % 10000 == 0) || (dc < 10000 && dc % 1000 == 0))) {
      VerbatimLogger.info("Processed " + dc + " documents ... ");
    }

    // Get filename and path to be used as document vector ID, defaulting to doc number only if
    // docidfield is not populated.
    String docID = luceneUtils.getExternalDocId(dc);

    Vector docVector = VectorFactory.createZeroVector(flagConfig.vectortype(), flagConfig.dimension());

    for (String fieldName : flagConfig.contentsfields()) {
      Terms terms = luceneUtils.getTermVector(dc, fieldName);

      if (terms == null) {
        VerbatimLogger.fine(
            String.format(
                "When building document vectors, no term vector for field: '%s' in document '%s'.",
                fieldName, docID));
        continue;
      }

      TermsEnum termsEnum = terms.iterator();
      BytesRef bytes;
      while ((bytes = termsEnum.next()) != null) {
        Term term = new Term(fieldName, bytes);
        String termString = term.text();
        PostingsEnum docs = termsEnum.postings(null);
        docs.nextDoc();
        int freq = docs.freq();

        try {
          Vector termVector = termVectorData.getVector(termString);
          if (termVector != null && termVector.getDimension() > 0) {
            float localweight = luceneUtils.getLocalTermWeight(freq);
            float globalweight = luceneUtils.getGlobalTermWeight(new Term(fieldName, termString));
            float fieldweight = 1;

            if (flagConfig.fieldweight()) {
              //field weight: 1/sqrt(number of terms in field)
              fieldweight = (float) (1 / Math.sqrt(terms.size()));
            }

            // Add contribution from this term, excluding terms that
            // are not represented in termVectorData.
            docVector.superpose(termVector, localweight * globalweight * fieldweight, null);
          }
        } catch (NullPointerException npe) {
          // Don't normally print anything - too much data!
          logger.finest("term " + termString + " not represented");
        }
      }
    }

    if (docVector.isZeroVector()) {
      logger.warning(String.format(
          "Outputting zero vector for document '%s'. This probably means that none of " +
              "the -contentsfields were populated, or all terms failed the LuceneUtils termsfilter." +
              " You may want to investigate.",
          docID));
    }

    // All fields in document have been processed. Write out documentID and normalized vector.
    docVector.normalize();
    outputStream.writeString(docID);
    docVector.writeToLuceneStream(outputStream);
  } // Finish iterating through documents.

  VerbatimLogger.info("Finished writing vectors.\n");
  outputStream.close();
  fsDirectory.close();
}
 

public void testEndPointer() throws IOException {
  final Directory dir = newDirectory();
  final int valueCount = RandomNumbers.randomIntBetween(random(), 1, 1000);
  final IndexOutput out = dir.createOutput("tests.bin", newIOContext(random()));
  for (int i = 0; i < valueCount; ++i) {
    out.writeLong(0);
  }
  out.close();
  final IndexInput in = dir.openInput("tests.bin", newIOContext(random()));
  for (int version = PackedInts.VERSION_START; version <= PackedInts.VERSION_CURRENT; ++version) {
    for (int bpv = 1; bpv <= 64; ++bpv) {
      for (PackedInts.Format format : PackedInts.Format.values()) {
        if (!format.isSupported(bpv)) {
          continue;
        }
        final long byteCount = format.byteCount(version, valueCount, bpv);
        String msg = "format=" + format + ",version=" + version + ",valueCount=" + valueCount + ",bpv=" + bpv;

        // test iterator
        in.seek(0L);
        final PackedInts.ReaderIterator it = PackedInts.getReaderIteratorNoHeader(in, format, version, valueCount, bpv, RandomNumbers.randomIntBetween(random(), 1, 1<<16));
        for (int i = 0; i < valueCount; ++i) {
          it.next();
        }
        assertEquals(msg, byteCount, in.getFilePointer());

        // test direct reader
        in.seek(0L);
        final PackedInts.Reader directReader = PackedInts.getDirectReaderNoHeader(in, format, version, valueCount, bpv);
        directReader.get(valueCount - 1);
        assertEquals(msg, byteCount, in.getFilePointer());

        // test reader
        in.seek(0L);
        PackedInts.getReaderNoHeader(in, format, version, valueCount, bpv);
        assertEquals(msg, byteCount, in.getFilePointer());
       }
    }
  }
  in.close();
  dir.close();
}
 

public void testPackedInputOutput() throws IOException {
  final long[] longs = new long[random().nextInt(8192)];
  final int[] bitsPerValues = new int[longs.length];
  final boolean[] skip = new boolean[longs.length];
  for (int i = 0; i < longs.length; ++i) {
    final int bpv = RandomNumbers.randomIntBetween(random(), 1, 64);
    bitsPerValues[i] = random().nextBoolean() ? bpv : TestUtil.nextInt(random(), bpv, 64);
    if (bpv == 64) {
      longs[i] = random().nextLong();
    } else {
      longs[i] = TestUtil.nextLong(random(), 0, PackedInts.maxValue(bpv));
    }
    skip[i] = rarely();
  }

  final Directory dir = newDirectory();
  final IndexOutput out = dir.createOutput("out.bin", IOContext.DEFAULT);
  PackedDataOutput pout = new PackedDataOutput(out);
  long totalBits = 0;
  for (int i = 0; i < longs.length; ++i) {
    pout.writeLong(longs[i], bitsPerValues[i]);
    totalBits += bitsPerValues[i];
    if (skip[i]) {
      pout.flush();
      totalBits = 8 * (long) Math.ceil((double) totalBits / 8);
    }
  }
  pout.flush();
  assertEquals((long) Math.ceil((double) totalBits / 8), out.getFilePointer());
  out.close();
  final IndexInput in = dir.openInput("out.bin", IOContext.READONCE);
  final PackedDataInput pin = new PackedDataInput(in);
  for (int i = 0; i < longs.length; ++i) {
    assertEquals("" + i, longs[i], pin.readLong(bitsPerValues[i]));
    if (skip[i]) {
      pin.skipToNextByte();
    }
  }
  assertEquals((long) Math.ceil((double) totalBits / 8), in.getFilePointer());
  in.close();
  dir.close();
}