Java Code Examples for org.apache.lucene.store.IndexOutput#writeInt()

public void testMixedDirectoryAndPolicy() throws IOException {
  Directory readDir = new RAMDirectory();
  updateIndex(readDir, 0, numDocsPerUpdate,
      new KeepOnlyLastCommitDeletionPolicy());

  verify(readDir, numDocsPerUpdate);

  IndexOutput out =
      readDir.createOutput("_" + (numDocsPerUpdate / maxBufferedDocs + 2)
          + ".cfs");
  out.writeInt(0);
  out.close();

  Directory writeDir = new RAMDirectory();
  Directory mixedDir = new MixedDirectory(readDir, writeDir);
  updateIndex(mixedDir, numDocsPerUpdate, numDocsPerUpdate,
      new MixedDeletionPolicy());

  verify(readDir, numDocsPerUpdate);
  verify(mixedDir, 2 * numDocsPerUpdate);
}
 

public void testMixedDirectoryAndPolicy() throws IOException {
  Directory readDir = new RAMDirectory();
  updateIndex(readDir, 0, numDocsPerUpdate,
      new KeepOnlyLastCommitDeletionPolicy());

  verify(readDir, numDocsPerUpdate);

  IndexOutput out =
      readDir.createOutput("_" + (numDocsPerUpdate / maxBufferedDocs + 2)
          + ".cfs");
  out.writeInt(0);
  out.close();

  Directory writeDir = new RAMDirectory();
  Directory mixedDir = new MixedDirectory(readDir, writeDir);
  updateIndex(mixedDir, numDocsPerUpdate, numDocsPerUpdate,
      new MixedDeletionPolicy());

  verify(readDir, numDocsPerUpdate);
  verify(mixedDir, 2 * numDocsPerUpdate);
}
 

/** 
 * Convert a WordSpace text matrix to a bin WordSpace file
 * Text matrix format:
 * - the first line contains the matrix dimensions N
 * - each line contains the word vector information: word d1 d2 ... dN
 * Text2Bin text_matrix_file bin_matrix_file
 * @param args the command line arguments
 */
public static void main(String[] args) {
    try {
        BufferedReader in = new BufferedReader(new FileReader(args[0]));
        File file = new File(args[1]);
        FSDirectory fs = FSDirectory.open(file.getParentFile());
        IndexOutput output = fs.createOutput(file.getName());
        String header = in.readLine();
        output.writeString("-dimensions");
        output.writeInt(Integer.parseInt(header));
        while (in.ready()) {
            String line = in.readLine();
            String[] split = line.split("\t");
            output.writeString(split[0]);
            for (int i=1;i<split.length;i++) {
                output.writeInt(Float.floatToIntBits(Float.parseFloat(split[i])));
            }
        }
        in.close();
        output.close();
    } catch (IOException ex) {
        Logger.getLogger(Text2Bin.class.getName()).log(Level.SEVERE, null, ex);
    }
}
 

/**
 * Writes vector out in dense format.  If vector is originally sparse, writes out a copy so
 * that vector remains sparse. Truncates to length k. 
 */
public void writeToLuceneStream(IndexOutput outputStream, int k) {
  float[] coordsToWrite;
  if (isSparse) {
    RealVector copy = copy();
    copy.sparseToDense();
    coordsToWrite = copy.coordinates;
  } else {
    coordsToWrite = coordinates;
  }
  for (int i = 0; i < k; ++i) {
    try {
      outputStream.writeInt(Float.floatToIntBits(coordsToWrite[i]));
    } catch (IOException e) {
      e.printStackTrace();
    }
  }
}
 

@Override
/**
 * Writes vector out in dense format.  If vector is originally sparse, writes out a copy so
 * that vector remains sparse.
 */
public void writeToLuceneStream(IndexOutput outputStream) {
  float[] coordsToWrite;
  if (isSparse) {
    RealVector copy = copy();
    copy.sparseToDense();
    coordsToWrite = copy.coordinates;
  } else {
    coordsToWrite = coordinates;
  }
  for (int i = 0; i < dimension; ++i) {
    try {
      outputStream.writeInt(Float.floatToIntBits(coordsToWrite[i]));
    } catch (IOException e) {
      e.printStackTrace();
    }
  }
}
 

@Override
/**
 * Writes vector out in dense format.
 */
public void writeToLuceneStream(IndexOutput outputStream) {
  int[] coordsToWrite;
  
    coordsToWrite = coordinates;
  
  for (int i = 0; i < dimension; ++i) {
    try {
      outputStream.writeInt((coordsToWrite[i]));
    } catch (IOException e) {
      e.printStackTrace();
    }
  }
}
 

/**
 * Transforms vector to cartesian form and writes vector out in dense format, truncating the
 * vectors to the assigned dimensionality
 */
public void writeToLuceneStream(IndexOutput outputStream, int k) {
  toCartesian();
  for (int i = 0; i < k * 2; ++i) {
    try {
      outputStream.writeInt(Float.floatToIntBits(coordinates[i]));
    } catch (IOException e) {
      e.printStackTrace();
    }
  }
    
    
    /* DORMANT CODE!
  assert(opMode != MODE.POLAR_SPARSE);
  if (opMode == MODE.CARTESIAN) {
    cartesianToDensePolar();
  }
  for (int i = 0; i < dimension; ++i) {
    try {
      outputStream.writeInt((int)(phaseAngles[i]));
    } catch (IOException e) {
      e.printStackTrace();
    }
  }
     */
}
 

public void testRename() throws IOException {
  String[] listAll = directory.listAll();
  for (String file : listAll) {
    directory.deleteFile(file);
  }
  
  IndexOutput output = directory.createOutput("testing.test", new IOContext());
  output.writeInt(12345);
  output.close();
  directory.rename("testing.test", "testing.test.renamed");
  assertFalse(slowFileExists(directory, "testing.test"));
  assertTrue(slowFileExists(directory, "testing.test.renamed"));
  IndexInput input = directory.openInput("testing.test.renamed", new IOContext());
  assertEquals(12345, input.readInt());
  assertEquals(input.getFilePointer(), input.length());
  input.close();
  directory.deleteFile("testing.test.renamed");
  assertFalse(slowFileExists(directory, "testing.test.renamed"));
}
 

public void testCheckFooterInvalid() throws Exception {
  ByteBuffersDataOutput out = new ByteBuffersDataOutput();
  IndexOutput output = new ByteBuffersIndexOutput(out, "temp", "temp");
  CodecUtil.writeHeader(output, "FooBar", 5);
  output.writeString("this is the data");
  output.writeInt(CodecUtil.FOOTER_MAGIC);
  output.writeInt(0);
  output.writeLong(1234567); // write a bogus checksum
  output.close();

  ChecksumIndexInput input = new BufferedChecksumIndexInput(new ByteBuffersIndexInput(out.toDataInput(), "temp"));
  CodecUtil.checkHeader(input, "FooBar", 5, 5);
  assertEquals("this is the data", input.readString());
  Exception mine = new RuntimeException("fake exception");
  CorruptIndexException expected = expectThrows(CorruptIndexException.class, () -> {
    CodecUtil.checkFooter(input, mine);
  });
  assertTrue(expected.getMessage().contains("checksum failed"));
  Throwable suppressed[] = expected.getSuppressed();
  assertEquals(1, suppressed.length);
  assertEquals("fake exception", suppressed[0].getMessage());
  input.close();
}
 

@Test
public void testMultiInstancesHdfsDirectoryTest1() throws IOException, InterruptedException {
  HdfsDirectory dir1 = new HdfsDirectory(_configuration, new Path(_root, "dir"));
  

  IndexOutput output = dir1.createOutput("a", IOContext.DEFAULT);
  output.writeInt(1234);
  output.close();
  
  HdfsDirectory dir2 = new HdfsDirectory(_configuration, new Path(_root, "dir"));

  IndexInput input = dir2.openInput("a", IOContext.READ);
  assertEquals(4, input.length());
  assertEquals(1234, input.readInt());
  input.close();

  dir1.close();
  dir2.close();
}
 

public void testMakeLockDisabled() throws IOException {
  final String testfile = "_123.test";

  Directory dir = newDirectory();
  IndexOutput out = dir.createOutput(testfile, IOContext.DEFAULT);
  out.writeInt(3);
  out.close();
 
  SegmentInfo si = newSegmentInfo(dir, "_123");
  si.setFiles(Collections.emptyList());
  si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
  Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
  expectThrows(UnsupportedOperationException.class, () -> {
    cfs.obtainLock("foobar");
  });

  cfs.close();
  dir.close();
}
 

public void testSyncDisabled() throws IOException {
  final String testfile = "_123.test";

  Directory dir = newDirectory();
  IndexOutput out = dir.createOutput(testfile, IOContext.DEFAULT);
  out.writeInt(3);
  out.close();
 
  SegmentInfo si = newSegmentInfo(dir, "_123");
  si.setFiles(Collections.emptyList());
  si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
  Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
  expectThrows(UnsupportedOperationException.class, () -> {
    cfs.sync(Collections.singleton(testfile));
  });

  cfs.close();
  dir.close();
}
 

public void testDeleteFileDisabled() throws IOException {
  final String testfile = "_123.test";

  Directory dir = newDirectory();
  IndexOutput out = dir.createOutput(testfile, IOContext.DEFAULT);
  out.writeInt(3);
  out.close();
 
  SegmentInfo si = newSegmentInfo(dir, "_123");
  si.setFiles(Collections.emptyList());
  si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
  Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
  expectThrows(UnsupportedOperationException.class, () -> {
    cfs.deleteFile(testfile);
  });

  cfs.close();
  dir.close();
}
 

@Test
public void testWritingAndReadingAFile() throws IOException {
  String[] listAll = directory.listAll();
  for (String file : listAll) {
    directory.deleteFile(file);
  }
  
  IndexOutput output = directory.createOutput("testing.test", new IOContext());
  output.writeInt(12345);
  output.close();

  IndexInput input = directory.openInput("testing.test", new IOContext());
  assertEquals(12345, input.readInt());
  input.close();

  listAll = directory.listAll();
  assertEquals(1, listAll.length);
  assertEquals("testing.test", listAll[0]);

  assertEquals(4, directory.fileLength("testing.test"));

  IndexInput input1 = directory.openInput("testing.test", new IOContext());

  IndexInput input2 = input1.clone();
  assertEquals(12345, input2.readInt());
  input2.close();

  assertEquals(12345, input1.readInt());
  input1.close();

  assertFalse(slowFileExists(directory, "testing.test.other"));
  assertTrue(slowFileExists(directory, "testing.test"));
  directory.deleteFile("testing.test");
  assertFalse(slowFileExists(directory, "testing.test"));
}
 

public void testReadHeaderWrongMagic() throws Exception {
  ByteBuffersDataOutput out = new ByteBuffersDataOutput();
  IndexOutput output = new ByteBuffersIndexOutput(out, "temp", "temp");
  output.writeInt(1234);
  output.close();
  
  IndexInput input = new ByteBuffersIndexInput(out.toDataInput(), "temp");
  expectThrows(CorruptIndexException.class, () -> {
    CodecUtil.checkHeader(input, "bogus", 1, 1);
  });
}
 

@Override
/**
 * Transforms vector to cartesian form and writes vector out in dense format.
 */
public void writeToLuceneStream(IndexOutput outputStream) {
  toCartesian();
  for (int i = 0; i < dimension * 2; ++i) {
    try {
      outputStream.writeInt(Float.floatToIntBits(coordinates[i]));
    } catch (IOException e) {
      e.printStackTrace();
    }
  }
}
 

private static short flushBlockJumps(int[] jumps, int blockCount, IndexOutput out, long origo) throws IOException {
  if (blockCount == 2) { // Jumps with a single real entry + NO_MORE_DOCS is just wasted space so we ignore that
    blockCount = 0;
  }
  for (int i = 0 ; i < blockCount ; i++) {
    out.writeInt(jumps[i*2]); // index
    out.writeInt(jumps[i*2+1]); // offset
  }
  // As there are at most 32k blocks, the count is a short
  // The jumpTableOffset will be at lastPos - (blockCount * Long.BYTES)
  return (short)blockCount;
}
 

@Test
public void testWritingAndReadingAFile() throws IOException {

  IndexOutput output = directory.createOutput("testing.test", IOContext.DEFAULT);
  output.writeInt(12345);
  output.flush();
  output.close();

  IndexInput input = directory.openInput("testing.test", IOContext.DEFAULT);
  assertEquals(12345, input.readInt());
  input.close();

  String[] listAll = directory.listAll();
  assertEquals(1, listAll.length);
  assertEquals("testing.test", listAll[0]);

  assertEquals(4, directory.fileLength("testing.test"));

  IndexInput input1 = directory.openInput("testing.test", IOContext.DEFAULT);

  IndexInput input2 = (IndexInput) input1.clone();
  assertEquals(12345, input2.readInt());
  input2.close();

  assertEquals(12345, input1.readInt());
  input1.close();

  assertFalse(directory.fileExists("testing.test.other"));
  assertTrue(directory.fileExists("testing.test"));
  directory.deleteFile("testing.test");
  assertFalse(directory.fileExists("testing.test"));
}
 

private void createFile(HdfsDirectory dir, String name) throws IOException {
  IndexOutput output = dir.createOutput(name, IOContext.DEFAULT);
  output.writeInt(1);
  output.close();
}
 

/**
 * Writes a codec footer, which records both a checksum
 * algorithm ID and a checksum. This footer can
 * be parsed and validated with 
 * {@link #checkFooter(ChecksumIndexInput) checkFooter()}.
 * <p>
 * CodecFooter --&gt; Magic,AlgorithmID,Checksum
 * <ul>
 *    <li>Magic --&gt; {@link DataOutput#writeInt Uint32}. This
 *        identifies the start of the footer. It is always {@value #FOOTER_MAGIC}.
 *    <li>AlgorithmID --&gt; {@link DataOutput#writeInt Uint32}. This
 *        indicates the checksum algorithm used. Currently this is always 0,
 *        for zlib-crc32.
 *    <li>Checksum --&gt; {@link DataOutput#writeLong Uint64}. The
 *        actual checksum value for all previous bytes in the stream, including
 *        the bytes from Magic and AlgorithmID.
 * </ul>
 * 
 * @param out Output stream
 * @throws IOException If there is an I/O error writing to the underlying medium.
 */
public static void writeFooter(IndexOutput out) throws IOException {
  out.writeInt(FOOTER_MAGIC);
  out.writeInt(0);
  writeCRC(out);
}