Java Code Examples for org.apache.lucene.store.IndexInput#clone()

/**
 * Loads a {@link NRTSuggester} from {@link org.apache.lucene.store.IndexInput} on or off-heap
 * depending on the provided <code>fstLoadMode</code>
 */
public static NRTSuggester load(IndexInput input, FSTLoadMode fstLoadMode) throws IOException {
  final FST<Pair<Long, BytesRef>> fst;
  if (shouldLoadFSTOffHeap(input, fstLoadMode)) {
    OffHeapFSTStore store = new OffHeapFSTStore();
    IndexInput clone = input.clone();
    clone.seek(input.getFilePointer());
    fst = new FST<>(clone, clone, new PairOutputs<>(
        PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton()), store);
    input.seek(clone.getFilePointer() + store.size());
  } else {
    fst = new FST<>(input, input, new PairOutputs<>(
        PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton()));
  }

  /* read some meta info */
  int maxAnalyzedPathsPerOutput = input.readVInt();
  /*
   * Label used to denote the end of an input in the FST and
   * the beginning of dedup bytes
   */
  int endByte = input.readVInt();
  int payloadSep = input.readVInt();
  return new NRTSuggester(fst, maxAnalyzedPathsPerOutput, payloadSep);
}
 

public void testClonedStreamsClosing() throws IOException {
  Directory dir = newDirectory();
  Directory cr = createLargeCFS(dir);
  
  // basic clone
  IndexInput expected = dir.openInput("_123.f11", newIOContext(random()));
  
  IndexInput one = cr.openInput("_123.f11", newIOContext(random()));
  
  IndexInput two = one.clone();
  
  assertSameStreams("basic clone one", expected, one);
  expected.seek(0);
  assertSameStreams("basic clone two", expected, two);
  
  // Now close the compound reader
  cr.close();
  expected.close();
  dir.close();
}
 

@Test
public void test2() throws IOException {
  Cache cache = getCache();
  RAMDirectory directory = new RAMDirectory();
  Random random = new Random(seed);

  String name = "test2";
  long size = (10 * 1024 * 1024) + 13;

  IndexOutput output = directory.createOutput(name, IOContext.DEFAULT);
  writeRandomData(size, random, output);
  output.close();

  IndexInput input = directory.openInput(name, IOContext.DEFAULT);
  IndexInput testInput = new CacheIndexInput(null, name, input.clone(), cache);
  readRandomData(input, testInput, random, sampleSize, maxBufSize, maxOffset);
  readRandomDataShort(input, testInput, random, sampleSize);
  readRandomDataInt(input, testInput, random, sampleSize);
  readRandomDataLong(input, testInput, random, sampleSize);
  testInput.close();
  input.close();
  directory.close();
}
 

public FieldIndexData(IndexInput in, FieldInfo fieldInfo, long indexStart) throws IOException {
  IndexInput clone = in.clone();
  clone.seek(indexStart);
  fst = new FST<>(clone, clone, fstOutputs);
  clone.close();

  /*
  final String dotFileName = segment + "_" + fieldInfo.name + ".dot";
  Writer w = new OutputStreamWriter(new FileOutputStream(dotFileName));
  Util.toDot(fst, w, false, false);
  System.out.println("FST INDEX: SAVED to " + dotFileName);
  w.close();
  */
}
 

@Test
public void testWritingAndReadingAFile() throws IOException {
  String[] listAll = directory.listAll();
  for (String file : listAll) {
    directory.deleteFile(file);
  }
  
  IndexOutput output = directory.createOutput("testing.test", new IOContext());
  output.writeInt(12345);
  output.close();

  IndexInput input = directory.openInput("testing.test", new IOContext());
  assertEquals(12345, input.readInt());
  input.close();

  listAll = directory.listAll();
  assertEquals(1, listAll.length);
  assertEquals("testing.test", listAll[0]);

  assertEquals(4, directory.fileLength("testing.test"));

  IndexInput input1 = directory.openInput("testing.test", new IOContext());

  IndexInput input2 = input1.clone();
  assertEquals(12345, input2.readInt());
  input2.close();

  assertEquals(12345, input1.readInt());
  input1.close();

  assertFalse(slowFileExists(directory, "testing.test.other"));
  assertTrue(slowFileExists(directory, "testing.test"));
  directory.deleteFile("testing.test");
  assertFalse(slowFileExists(directory, "testing.test"));
}
 

private BinaryDocValues getFixedBinary(FieldInfo field, final BinaryEntry bytes) {
  final IndexInput data = this.data.clone();

  return new LongBinaryDocValues() {

    private final ThreadValue<IndexInput> in = new ThreadValue<IndexInput>() {
      @Override
      protected IndexInput initialValue() {
        return data.clone();
      }
    };

    @Override
    public void get(long id, BytesRef result) {
      long address = bytes.offset + id * bytes.maxLength;
      try {
        IndexInput indexInput = in.get();
        indexInput.seek(address);
        // NOTE: we could have one buffer, but various consumers (e.g.
        // FieldComparatorSource)
        // assume "they" own the bytes after calling this!
        final byte[] buffer = new byte[bytes.maxLength];
        indexInput.readBytes(buffer, 0, buffer.length);
        result.bytes = buffer;
        result.offset = 0;
        result.length = buffer.length;
      } catch (IOException e) {
        throw new RuntimeException(e);
      }
    }
  };
}
 

DirectPacked64SingleBlockReader(int bitsPerValue, int valueCount,
    IndexInput input) {
  super(valueCount, bitsPerValue);
  this.in = new ThreadLocal<IndexInput>() {
    @Override
    protected IndexInput initialValue() {
      return input.clone();
    }      
  };
  startPointer = input.getFilePointer();
  valuesPerBlock = 64 / bitsPerValue;
  mask = ~(~0L << bitsPerValue);
}
 

public DirectPackedReader(int bitsPerValue, int valueCount, IndexInput input) {
  super(valueCount, bitsPerValue);
  this.in = new ThreadLocal<IndexInput>() {
    @Override
    protected IndexInput initialValue() {
      return input.clone();
    }      
  };
  startPointer = input.getFilePointer();
}
 

@Test
public void testWritingAndReadingAFile() throws IOException {

  IndexOutput output = directory.createOutput("testing.test", IOContext.DEFAULT);
  output.writeInt(12345);
  output.flush();
  output.close();

  IndexInput input = directory.openInput("testing.test", IOContext.DEFAULT);
  assertEquals(12345, input.readInt());
  input.close();

  String[] listAll = directory.listAll();
  assertEquals(1, listAll.length);
  assertEquals("testing.test", listAll[0]);

  assertEquals(4, directory.fileLength("testing.test"));

  IndexInput input1 = directory.openInput("testing.test", IOContext.DEFAULT);

  IndexInput input2 = (IndexInput) input1.clone();
  assertEquals(12345, input2.readInt());
  input2.close();

  assertEquals(12345, input1.readInt());
  input1.close();

  assertFalse(directory.fileExists("testing.test.other"));
  assertTrue(directory.fileExists("testing.test"));
  directory.deleteFile("testing.test");
  assertFalse(directory.fileExists("testing.test"));
}
 

@Test
public void testLongReadAndClone() throws IOException {
  FSDirectory control = FSDirectory.open(fileControl);
  Directory dir = getControlDir(control, directory);
  String name = writeFile(dir,10*1000*1000);
  IndexInput input = dir.openInput(name, IOContext.DEFAULT);
  readFile(input,1000*1000);
  IndexInput clone = input.clone();
  clone.readByte();
  input.close();
}
 

public BrowserSupplier(IndexInput dictionaryInput, long dictionaryStartFP, BlockDecoder blockDecoder, boolean isFSTOnHeap) throws IOException {
  this.dictionaryInput = dictionaryInput.clone();
  this.dictionaryInput.seek(dictionaryStartFP);
  this.blockDecoder = blockDecoder;
  this.isFSTOnHeap = isFSTOnHeap;
}
 

/** This test opens two files from a compound stream and verifies that
 *  their file positions are independent of each other.
 */
public void testRandomAccessClones() throws IOException {
  Directory dir = newDirectory();
  Directory cr = createLargeCFS(dir);
  
  // Open two files
  IndexInput e1 = cr.openInput("_123.f11", newIOContext(random()));
  IndexInput e2 = cr.openInput("_123.f3", newIOContext(random()));
  
  IndexInput a1 = e1.clone();
  IndexInput a2 = e2.clone();
  
  // Seek the first pair
  e1.seek(100);
  a1.seek(100);
  assertEquals(100, e1.getFilePointer());
  assertEquals(100, a1.getFilePointer());
  byte be1 = e1.readByte();
  byte ba1 = a1.readByte();
  assertEquals(be1, ba1);
  
  // Now seek the second pair
  e2.seek(1027);
  a2.seek(1027);
  assertEquals(1027, e2.getFilePointer());
  assertEquals(1027, a2.getFilePointer());
  byte be2 = e2.readByte();
  byte ba2 = a2.readByte();
  assertEquals(be2, ba2);
  
  // Now make sure the first one didn't move
  assertEquals(101, e1.getFilePointer());
  assertEquals(101, a1.getFilePointer());
  be1 = e1.readByte();
  ba1 = a1.readByte();
  assertEquals(be1, ba1);
  
  // Now more the first one again, past the buffer length
  e1.seek(1910);
  a1.seek(1910);
  assertEquals(1910, e1.getFilePointer());
  assertEquals(1910, a1.getFilePointer());
  be1 = e1.readByte();
  ba1 = a1.readByte();
  assertEquals(be1, ba1);
  
  // Now make sure the second set didn't move
  assertEquals(1028, e2.getFilePointer());
  assertEquals(1028, a2.getFilePointer());
  be2 = e2.readByte();
  ba2 = a2.readByte();
  assertEquals(be2, ba2);
  
  // Move the second set back, again cross the buffer size
  e2.seek(17);
  a2.seek(17);
  assertEquals(17, e2.getFilePointer());
  assertEquals(17, a2.getFilePointer());
  be2 = e2.readByte();
  ba2 = a2.readByte();
  assertEquals(be2, ba2);
  
  // Finally, make sure the first set didn't move
  // Now make sure the first one didn't move
  assertEquals(1911, e1.getFilePointer());
  assertEquals(1911, a1.getFilePointer());
  be1 = e1.readByte();
  ba1 = a1.readByte();
  assertEquals(be1, ba1);
  
  e1.close();
  e2.close();
  a1.close();
  a2.close();
  cr.close();
  dir.close();
}
 

private BinaryDocValues getVariableBinary(FieldInfo field, final BinaryEntry bytes) throws IOException {
  final IndexInput data = this.data.clone();

  Tracer trace = Trace.trace("getSorted - BlockPackedReader - create");
  final MonotonicBlockPackedReader addresses;
  try {
    data.seek(bytes.addressesOffset);
    addresses = new MonotonicBlockPackedReader(data, bytes.packedIntsVersion, bytes.blockSize, bytes.count, true);
  } finally {
    trace.done();
  }
  return new LongBinaryDocValues() {
    
    private final ThreadValue<IndexInput> _input = new ThreadValue<IndexInput>() {
      @Override
      protected IndexInput initialValue() {
        return data.clone();
      }
    };
    
    @Override
    public void get(long id, BytesRef result) {
      long startAddress = bytes.offset + (id == 0 ? 0 : addresses.get(id - 1));
      long endAddress = bytes.offset + addresses.get(id);
      int length = (int) (endAddress - startAddress);
      try {
        IndexInput indexInput = _input.get();
        indexInput.seek(startAddress);
        // NOTE: we could have one buffer, but various consumers (e.g.
        // FieldComparatorSource)
        // assume "they" own the bytes after calling this!
        final byte[] buffer = new byte[length];
        indexInput.readBytes(buffer, 0, buffer.length);
        result.bytes = buffer;
        result.offset = 0;
        result.length = length;
      } catch (IOException e) {
        throw new RuntimeException(e);
      }
    }
  };
}