Java Code Examples for org.apache.lucene.store.IndexOutput#writeLong()

private void writeValues(NumericDocValues values, int numBytesPerValue, IndexOutput out) throws IOException, AssertionError {
  for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
    long value = values.longValue();
    switch (numBytesPerValue) {
      case 1:
        out.writeByte((byte) value);
        break;
      case 2:
        out.writeShort((short) value);
        break;
      case 4:
        out.writeInt((int) value);
        break;
      case 8:
        out.writeLong(value);
        break;
      default:
        throw new AssertionError();
    }
  }
}
 

public void testCheckFooterInvalid() throws Exception {
  ByteBuffersDataOutput out = new ByteBuffersDataOutput();
  IndexOutput output = new ByteBuffersIndexOutput(out, "temp", "temp");
  CodecUtil.writeHeader(output, "FooBar", 5);
  output.writeString("this is the data");
  output.writeInt(CodecUtil.FOOTER_MAGIC);
  output.writeInt(0);
  output.writeLong(1234567); // write a bogus checksum
  output.close();

  ChecksumIndexInput input = new BufferedChecksumIndexInput(new ByteBuffersIndexInput(out.toDataInput(), "temp"));
  CodecUtil.checkHeader(input, "FooBar", 5, 5);
  assertEquals("this is the data", input.readString());
  Exception mine = new RuntimeException("fake exception");
  CorruptIndexException expected = expectThrows(CorruptIndexException.class, () -> {
    CodecUtil.checkFooter(input, mine);
  });
  assertTrue(expected.getMessage().contains("checksum failed"));
  Throwable suppressed[] = expected.getSuppressed();
  assertEquals(1, suppressed.length);
  assertEquals("fake exception", suppressed[0].getMessage());
  input.close();
}
 

public void testReadBogusCRC() throws Exception {
  ByteBuffersDataOutput out = new ByteBuffersDataOutput();
  IndexOutput output = new ByteBuffersIndexOutput(out, "temp", "temp");
  output.writeLong(-1L); // bad
  output.writeLong(1L << 32); // bad
  output.writeLong(-(1L << 32)); // bad
  output.writeLong((1L << 32) - 1); // ok
  output.close();
  IndexInput input = new BufferedChecksumIndexInput(new ByteBuffersIndexInput(out.toDataInput(), "temp"));
  // read 3 bogus values
  for (int i = 0; i < 3; i++) {
    expectThrows(CorruptIndexException.class, () -> {
      CodecUtil.readCRC(input);
    });
  }
  // good value
  CodecUtil.readCRC(input);
}
 

@Override
/**
 * Writes vector out to object output stream.  Converts to dense format if necessary.
 */
public void writeToLuceneStream(IndexOutput outputStream) {
  if (isSparse) {
    elementalToSemantic();
  }
  long[] bitArray = bitSet.getBits();

  for (int i = 0; i < bitArray.length; i++) {
    try {
      outputStream.writeLong(bitArray[i]);
    } catch (IOException e) {
      logger.severe("Couldn't write binary vector to lucene output stream.");
      e.printStackTrace();
    }
  }
}
 

/**
 * Writes vector out to object output stream.  Converts to dense format if necessary. Truncates to length k.
 */
public void writeToLuceneStream(IndexOutput outputStream, int k) {
  if (isSparse) {
    elementalToSemantic();
  }
  long[] bitArray = bitSet.getBits();

  for (int i = 0; i < k/64; i++) {
    try {
      outputStream.writeLong(bitArray[i]);
    } catch (IOException e) {
      logger.severe("Couldn't write binary vector to lucene output stream.");
      e.printStackTrace();
    }
  }
}
 

private static Directory copyFilesLocally(Configuration configuration, Directory dir, String table, Path shardDir,
    Path localCachePath, Collection<String> files, String segmentName) throws IOException {
  LOG.info("Copying files need to local cache for faster reads [{0}].", shardDir);
  Path localShardPath = new Path(new Path(new Path(localCachePath, table), shardDir.getName()), segmentName);
  HdfsDirectory localDir = new HdfsDirectory(configuration, localShardPath, null);
  for (String name : files) {
    if (!isValidFileToCache(name)) {
      continue;
    }
    LOG.info("Valid file for local copy [{0}].", name);
    if (!isValid(localDir, dir, name)) {
      LastModified lastModified = (LastModified) dir;
      long fileModified = lastModified.getFileModified(name);

      IndexInput input = dir.openInput(name, IOContext.READONCE);
      IndexOutput output = localDir.createOutput(name, IOContext.READONCE);
      output.copyBytes(input, input.length());
      output.close();
      IndexOutput lastMod = localDir.createOutput(name + LASTMOD, IOContext.DEFAULT);
      lastMod.writeLong(fileModified);
      lastMod.close();
    }
  }
  return localDir;
}
 

/**
 * Writes CRC32 value as a 64-bit long to the output.
 * @throws IllegalStateException if CRC is formatted incorrectly (wrong bits set)
 * @throws IOException if an i/o error occurs
 */
static void writeCRC(IndexOutput output) throws IOException {
  long value = output.getChecksum();
  if ((value & 0xFFFFFFFF00000000L) != 0) {
    throw new IllegalStateException("Illegal CRC-32 checksum: " + value + " (resource=" + output + ")");
  }
  output.writeLong(value);
}
 

@Test
public void test1() throws IOException {
  IndexOutput output = _cacheDirectory.createOutput("test.file", IOContext.DEFAULT);
  output.writeLong(0);
  output.writeLong(1);
  output.writeLong(2);
  output.close();

  IndexInput input = _cacheDirectory.openInput("test.file", IOContext.DEFAULT);
  assertEquals(0, input.readLong());
  assertEquals(1, input.readLong());
  assertEquals(2, input.readLong());
  input.close();
}
 

@Test
public void testSymlink() throws IOException {
  HdfsDirectory dir1 = new HdfsDirectory(_configuration, new Path(_base, "dir1"));
  IndexOutput output = dir1.createOutput("file1", IOContext.DEFAULT);
  output.writeLong(12345);
  output.close();

  assertTrue(dir1.fileExists("file1"));

  HdfsDirectory dir2 = new HdfsDirectory(_configuration, new Path(_base, "dir2"));
  dir1.copy(dir2, "file1", "file2", IOContext.DEFAULT);

  assertTrue(dir2.fileExists("file2"));
  assertEquals(8, dir2.fileLength("file2"));

  String[] listAll = dir2.listAll();
  assertEquals(1, listAll.length);
  assertEquals("file2", listAll[0]);

  IndexInput input = dir2.openInput("file2", IOContext.DEFAULT);
  assertEquals(12345, input.readLong());
  input.close();

  dir2.deleteFile("file2");

  assertFalse(dir2.fileExists("file2"));
  assertTrue(dir1.fileExists("file1"));

  dir2.close();
  dir1.close();
}
 

@Test
public void test4() throws IOException {
  Path path = new Path(_root, "dir4");
  {
    HdfsDirectory dir1 = new HdfsDirectory(_configuration, path);
    createFile(dir1, "file1");
    createFile(dir1, "file2");
    createFile(dir1, "file3");
    IndexOutput output = dir1.createOutput("file2", IOContext.DEFAULT);
    output.writeLong(1L);
    output.close();
    dir1.close();
  }

  HdfsDirectory dir2 = new HdfsDirectory(_configuration, path);
  String[] listAll = dir2.listAll();
  assertEquals(3, listAll.length);
  assertEquals(new HashSet<String>(Arrays.asList("file1", "file2", "file3")),
      new HashSet<String>(Arrays.asList(listAll)));
  for (String f : listAll) {
    if (f.equals("file2")) {
      assertEquals(8, dir2.fileLength(f));
    } else {
      assertEquals(4, dir2.fileLength(f));
    }
  }
  dir2.close();
}
 

public void create(DocIdSetIterator it) throws IOException {
  String fileName = getFileName();
  if (_directory.fileExists(getFileName())) {
    LOG.warn("Filter [{0}] in directory [{1}] being recreated due to incorrect size.", fileName, _directory);
    _directory.deleteFile(fileName);
  }
  IndexOutput output = _directory.createOutput(fileName, IOContext.READ);
  int index;
  int currentWordNum = 0;
  long wordValue = 0;
  while ((index = it.nextDoc()) < _numBits) {
    int wordNum = index >> 6; // div 64
    if (currentWordNum > wordNum) {
      throw new IOException("We got a problem here!");
    }
    while (currentWordNum < wordNum) {
      output.writeLong(wordValue);
      currentWordNum++;
      wordValue = 0;
    }
    int bit = index & 0x3f; // mod 64
    long bitmask = 1L << bit;
    wordValue |= bitmask;
  }
  if (_numBits > 0) {
    int totalWords = (_numBits / 64) + 1;
    while (currentWordNum < totalWords) {
      output.writeLong(wordValue);
      currentWordNum++;
      wordValue = 0;
    }
  }
  output.close();
}
 

private void writeTrailer(IndexOutput out, long dirStart) throws IOException {
  out.writeLong(dirStart);
}
 

/** Writes the terms file trailer. */
private void writeTrailer(IndexOutput out, long dirStart) throws IOException {
  out.writeLong(dirStart);    
}
 

/** Writes the index file trailer. */
private void writeIndexTrailer(IndexOutput indexOut, long dirStart) throws IOException {
  indexOut.writeLong(dirStart);    
}
 

public void testEndPointer() throws IOException {
  final Directory dir = newDirectory();
  final int valueCount = RandomNumbers.randomIntBetween(random(), 1, 1000);
  final IndexOutput out = dir.createOutput("tests.bin", newIOContext(random()));
  for (int i = 0; i < valueCount; ++i) {
    out.writeLong(0);
  }
  out.close();
  final IndexInput in = dir.openInput("tests.bin", newIOContext(random()));
  for (int version = PackedInts.VERSION_START; version <= PackedInts.VERSION_CURRENT; ++version) {
    for (int bpv = 1; bpv <= 64; ++bpv) {
      for (PackedInts.Format format : PackedInts.Format.values()) {
        if (!format.isSupported(bpv)) {
          continue;
        }
        final long byteCount = format.byteCount(version, valueCount, bpv);
        String msg = "format=" + format + ",version=" + version + ",valueCount=" + valueCount + ",bpv=" + bpv;

        // test iterator
        in.seek(0L);
        final PackedInts.ReaderIterator it = PackedInts.getReaderIteratorNoHeader(in, format, version, valueCount, bpv, RandomNumbers.randomIntBetween(random(), 1, 1<<16));
        for (int i = 0; i < valueCount; ++i) {
          it.next();
        }
        assertEquals(msg, byteCount, in.getFilePointer());

        // test direct reader
        in.seek(0L);
        final PackedInts.Reader directReader = PackedInts.getDirectReaderNoHeader(in, format, version, valueCount, bpv);
        directReader.get(valueCount - 1);
        assertEquals(msg, byteCount, in.getFilePointer());

        // test reader
        in.seek(0L);
        PackedInts.getReaderNoHeader(in, format, version, valueCount, bpv);
        assertEquals(msg, byteCount, in.getFilePointer());
       }
    }
  }
  in.close();
  dir.close();
}