Java Code Examples for org.apache.lucene.store.IndexInput#getFilePointer()

@Override
public boolean isCompressed(IndexInput in) throws IOException {
    long currentPointer = in.getFilePointer();
    // since we have some metdata before the first compressed header, we check on our specific header
    if (in.length() - currentPointer < (LUCENE_HEADER.length)) {
        return false;
    }
    for (int i = 0; i < LUCENE_HEADER.length; i++) {
        if (in.readByte() != LUCENE_HEADER[i]) {
            in.seek(currentPointer);
            return false;
        }
    }
    in.seek(currentPointer);
    return true;
}
 

public CompressedIndexInput(IndexInput in) throws IOException {
    super("compressed(" + in.toString() + ")");
    this.in = in;
    readHeader(in);
    this.version = in.readInt();
    long metaDataPosition = in.readLong();
    long headerLength = in.getFilePointer();
    in.seek(metaDataPosition);
    this.totalUncompressedLength = in.readVLong();
    int size = in.readVInt();
    offsets = BigArrays.NON_RECYCLING_INSTANCE.newLongArray(size);
    for (int i = 0; i < size; i++) {
        offsets.set(i, in.readVLong());
    }
    this.currentOffsetIdx = -1;
    this.currentUncompressedChunkPointer = 0;
    in.seek(headerLength);
}
 

protected static void assertSameStreams(String msg, IndexInput expected, IndexInput test) throws IOException {
  assertNotNull(msg + " null expected", expected);
  assertNotNull(msg + " null test", test);
  assertEquals(msg + " length", expected.length(), test.length());
  assertEquals(msg + " position", expected.getFilePointer(), test.getFilePointer());
  
  byte expectedBuffer[] = new byte[512];
  byte testBuffer[] = new byte[expectedBuffer.length];
  
  long remainder = expected.length() - expected.getFilePointer();
  while (remainder > 0) {
    int readLen = (int) Math.min(remainder, expectedBuffer.length);
    expected.readBytes(expectedBuffer, 0, readLen);
    test.readBytes(testBuffer, 0, readLen);
    assertEqualArrays(msg + ", remainder " + remainder, expectedBuffer, testBuffer, 0, readLen);
    remainder -= readLen;
  }
}
 

private static void validateFooter(IndexInput in) throws IOException {
  long remaining = in.length() - in.getFilePointer();
  long expected = footerLength();
  if (remaining < expected) {
    throw new CorruptIndexException("misplaced codec footer (file truncated?): remaining=" + remaining + ", expected=" + expected + ", fp=" + in.getFilePointer(), in);
  } else if (remaining > expected) {
    throw new CorruptIndexException("misplaced codec footer (file extended?): remaining=" + remaining + ", expected=" + expected + ", fp=" + in.getFilePointer(), in);
  }
  
  final int magic = in.readInt();
  if (magic != FOOTER_MAGIC) {
    throw new CorruptIndexException("codec footer mismatch (file truncated?): actual footer=" + magic + " vs expected footer=" + FOOTER_MAGIC, in);
  }
  
  final int algorithmID = in.readInt();
  if (algorithmID != 0) {
    throw new CorruptIndexException("codec footer mismatch: unknown algorithmID: " + algorithmID, in);
  }
}
 

/**
 * Gets the next doc.
 *
 * @param field
 *          the field
 * @param previousDocId
 *          the previous doc id
 * @return the next doc
 */
public IndexDoc getNextDoc(String field, int previousDocId) {
  if (fieldReferences.containsKey(field)) {
    FieldReferences fr = fieldReferences.get(field);
    try {
      if (previousDocId < 0) {
        return new IndexDoc(fr.refIndexDoc);
      } else {
        int nextDocId = previousDocId + 1;
        IndexInput inIndexDocId = indexInputList.get("indexDocId");
        ArrayList<MtasTreeHit<?>> list = CodecSearchTree.advanceMtasTree(
            nextDocId, inIndexDocId, fr.refIndexDocId, fr.refIndexDoc);
        if (list.size() == 1) {
          IndexInput inDoc = indexInputList.get("doc");
          inDoc.seek(list.get(0).ref);
          return new IndexDoc(inDoc.getFilePointer());
        }
      }
    } catch (IOException e) {
      log.debug(e);
      return null;
    }
  }
  return null;
}
 

public InputStreamIndexInput(IndexInput indexInput, long limit) {
    this.indexInput = indexInput;
    this.limit = limit;
    if ((indexInput.length() - indexInput.getFilePointer()) > limit) {
        actualSizeToRead = limit;
    } else {
        actualSizeToRead = indexInput.length() - indexInput.getFilePointer();
    }
}
 

public DirectPackedReader(int bitsPerValue, int valueCount, IndexInput input) {
  super(valueCount, bitsPerValue);
  this.in = new ThreadLocal<IndexInput>() {
    @Override
    protected IndexInput initialValue() {
      return input.clone();
    }      
  };
  startPointer = input.getFilePointer();
}
 

DirectPacked64SingleBlockReader(int bitsPerValue, int valueCount,
    IndexInput input) {
  super(valueCount, bitsPerValue);
  this.in = new ThreadLocal<IndexInput>() {
    @Override
    protected IndexInput initialValue() {
      return input.clone();
    }      
  };
  startPointer = input.getFilePointer();
  valuesPerBlock = 64 / bitsPerValue;
  mask = ~(~0L << bitsPerValue);
}
 

@Override
public SegmentInfo read(Directory dir, String segment, IOContext context) throws IOException {
  final String fileName = IndexFileNames.segmentFileName(segment, "", Blur022SegmentInfoFormat.SI_EXTENSION);
  final IndexInput input = dir.openInput(fileName, context);
  boolean success = false;
  try {
    CodecUtil.checkHeader(input, Blur022SegmentInfoFormat.CODEC_NAME, Blur022SegmentInfoFormat.VERSION_START,
        Blur022SegmentInfoFormat.VERSION_CURRENT);
    final String version = input.readString();
    final int docCount = input.readInt();
    if (docCount < 0) {
      throw new CorruptIndexException("invalid docCount: " + docCount + " (resource=" + input + ")");
    }
    final boolean isCompoundFile = input.readByte() == SegmentInfo.YES;
    final Map<String, String> diagnostics = input.readStringStringMap();
    final Map<String, String> attributes = input.readStringStringMap();
    final Set<String> files = input.readStringSet();

    if (input.getFilePointer() != input.length()) {
      throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read "
          + input.getFilePointer() + " vs size " + input.length() + " (resource: " + input + ")");
    }

    final SegmentInfo si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics,
        Collections.unmodifiableMap(attributes));
    si.setFiles(files);

    success = true;

    return si;

  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(input);
    } else {
      input.close();
    }
  }
}
 

@Override
public void decompress(final DataInput in, final int originalLength, final int offset, final int length,
    final BytesRef bytes) throws IOException {
  if (in instanceof IndexInput) {
    IndexInput indexInput = (IndexInput) in;
    String name = indexInput.toString();
    long filePointer = indexInput.getFilePointer();

    Entry entry = _entry.get();
    if (!entry.isValid(indexInput, name, filePointer)) {
      entry.setup(indexInput, name, filePointer);
      entry._cache.grow(originalLength + 7);
      _decompressor.decompress(indexInput, originalLength, 0, originalLength, entry._cache);
      entry._cache.length = originalLength;
      entry._cache.offset = 0;
      _entry.set(entry);
    }
    if (bytes.bytes.length < originalLength + 7) {
      bytes.bytes = new byte[ArrayUtil.oversize(originalLength + 7, 1)];
    }
    System.arraycopy(entry._cache.bytes, entry._cache.offset, bytes.bytes, 0, length + offset);
    bytes.offset = offset;
    bytes.length = length;
  } else {
    _decompressor.decompress(in, originalLength, offset, length, bytes);
  }
}
 

/**
 * Gets the prefixes.
 *
 * @param field
 *          the field
 * @return the prefixes
 */
private LinkedHashMap<String, Long> getPrefixRefs(String field) {
  if (fieldReferences.containsKey(field)) {
    FieldReferences fr = fieldReferences.get(field);
    if (!prefixReferences.containsKey(field)) {
      LinkedHashMap<String, Long> refs = new LinkedHashMap<String, Long>();
      try {
        IndexInput inPrefix = indexInputList.get("prefix");
        inPrefix.seek(fr.refPrefix);
        for (int i = 0; i < fr.numberOfPrefixes; i++) {
          Long ref = inPrefix.getFilePointer();
          String prefix = inPrefix.readString();
          refs.put(prefix, ref);
        }
      } catch (Exception e) {
        log.error(e);
        refs.clear();
      }
      prefixReferences.put(field, refs);
      return refs;
    } else {
      return prefixReferences.get(field);
    }
  } else {
    return null;
  }
}
 

DirectPackedReader(int bitsPerValue, int valueCount, IndexInput in) {
  super(valueCount);
  this.in = in;
  this.bitsPerValue = bitsPerValue;

  startPointer = in.getFilePointer();
  if (bitsPerValue == 64) {
    valueMask = -1L;
  } else {
    valueMask = (1L << bitsPerValue) - 1;
  }
}
 

private MonotonicBlockPackedReader(IndexInput in, int packedIntsVersion, int blockSize, long valueCount, boolean direct) throws IOException {
  this.valueCount = valueCount;
  blockShift = checkBlockSize(blockSize, MIN_BLOCK_SIZE, MAX_BLOCK_SIZE);
  blockMask = blockSize - 1;
  final int numBlocks = numBlocks(valueCount, blockSize);
  minValues = new long[numBlocks];
  averages = new float[numBlocks];
  subReaders = new PackedInts.Reader[numBlocks];
  long sumBPV = 0;
  for (int i = 0; i < numBlocks; ++i) {
    minValues[i] = in.readZLong();
    averages[i] = Float.intBitsToFloat(in.readInt());
    final int bitsPerValue = in.readVInt();
    sumBPV += bitsPerValue;
    if (bitsPerValue > 64) {
      throw new IOException("Corrupted");
    }
    if (bitsPerValue == 0) {
      subReaders[i] = new PackedInts.NullReader(blockSize);
    } else {
      final int size = (int) Math.min(blockSize, valueCount - (long) i * blockSize);
      if (direct) {
        final long pointer = in.getFilePointer();
        subReaders[i] = PackedInts.getDirectReaderNoHeader(in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue);
        in.seek(pointer + PackedInts.Format.PACKED.byteCount(packedIntsVersion, size, bitsPerValue));
      } else {
        subReaders[i] = PackedInts.getReaderNoHeader(in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue);
      }
    }
  }
  this.sumBPV = sumBPV;
}
 

/** Sole constructor. */
public BlockPackedReader(IndexInput in, int packedIntsVersion, int blockSize, long valueCount, boolean direct) throws IOException {
  this.valueCount = valueCount;
  blockShift = checkBlockSize(blockSize, MIN_BLOCK_SIZE, MAX_BLOCK_SIZE);
  blockMask = blockSize - 1;
  final int numBlocks = numBlocks(valueCount, blockSize);
  long[] minValues = null;
  subReaders = new PackedInts.Reader[numBlocks];
  long sumBPV = 0;
  for (int i = 0; i < numBlocks; ++i) {
    final int token = in.readByte() & 0xFF;
    final int bitsPerValue = token >>> BPV_SHIFT;
    sumBPV += bitsPerValue;
    if (bitsPerValue > 64) {
      throw new CorruptIndexException("Corrupted Block#" + i, in);
    }
    if ((token & MIN_VALUE_EQUALS_0) == 0) {
      if (minValues == null) {
        minValues = new long[numBlocks];
      }
      minValues[i] = zigZagDecode(1L + readVLong(in));
    }
    if (bitsPerValue == 0) {
      subReaders[i] = new PackedInts.NullReader(blockSize);
    } else {
      final int size = (int) Math.min(blockSize, valueCount - (long) i * blockSize);
      if (direct) {
        final long pointer = in.getFilePointer();
        subReaders[i] = PackedInts.getDirectReaderNoHeader(in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue);
        in.seek(pointer + PackedInts.Format.PACKED.byteCount(packedIntsVersion, size, bitsPerValue));
      } else {
        subReaders[i] = PackedInts.getReaderNoHeader(in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue);
      }
    }
  }
  this.minValues = minValues;
  this.sumBPV = sumBPV;
}
 

DirectPacked64SingleBlockReader(int bitsPerValue, int valueCount,
    IndexInput in) {
  super(valueCount);
  this.in = in;
  this.bitsPerValue = bitsPerValue;
  startPointer = in.getFilePointer();
  valuesPerBlock = 64 / bitsPerValue;
  mask = ~(~0L << bitsPerValue);
}
 

public InputStreamIndexInput(IndexInput indexInput, long limit) {
    this.indexInput = indexInput;
    this.limit = limit;
    if ((indexInput.length() - indexInput.getFilePointer()) > limit) {
        actualSizeToRead = limit;
    } else {
        actualSizeToRead = indexInput.length() - indexInput.getFilePointer();
    }
}
 

SkipBuffer(IndexInput input, int length) throws IOException {
  super("SkipBuffer on " + input);
  data = new byte[length];
  pointer = input.getFilePointer();
  input.readBytes(data, 0, length);
}
 

private void loadTerms() throws IOException {
  PositiveIntOutputs posIntOutputs = PositiveIntOutputs.getSingleton();
  final FSTCompiler<PairOutputs.Pair<Long,PairOutputs.Pair<Long,Long>>> fstCompiler;
  final PairOutputs<Long,Long> outputsInner = new PairOutputs<>(posIntOutputs, posIntOutputs);
  final PairOutputs<Long,PairOutputs.Pair<Long,Long>> outputs = new PairOutputs<>(posIntOutputs,
      outputsInner);
  fstCompiler = new FSTCompiler<>(FST.INPUT_TYPE.BYTE1, outputs);
  IndexInput in = SimpleTextFieldsReader.this.in.clone();
  in.seek(termsStart);
  final BytesRefBuilder lastTerm = new BytesRefBuilder();
  long lastDocsStart = -1;
  int docFreq = 0;
  long totalTermFreq = 0;
  FixedBitSet visitedDocs = new FixedBitSet(maxDoc);
  final IntsRefBuilder scratchIntsRef = new IntsRefBuilder();
  while(true) {
    SimpleTextUtil.readLine(in, scratch);
    if (scratch.get().equals(END) || StringHelper.startsWith(scratch.get(), FIELD)) {
      if (lastDocsStart != -1) {
        fstCompiler.add(Util.toIntsRef(lastTerm.get(), scratchIntsRef),
            outputs.newPair(lastDocsStart,
                outputsInner.newPair((long) docFreq, totalTermFreq)));
        sumTotalTermFreq += totalTermFreq;
      }
      break;
    } else if (StringHelper.startsWith(scratch.get(), DOC)) {
      docFreq++;
      sumDocFreq++;
      totalTermFreq++;
      scratchUTF16.copyUTF8Bytes(scratch.bytes(), DOC.length, scratch.length()-DOC.length);
      int docID = ArrayUtil.parseInt(scratchUTF16.chars(), 0, scratchUTF16.length());
      visitedDocs.set(docID);
    } else if (StringHelper.startsWith(scratch.get(), FREQ)) {
      scratchUTF16.copyUTF8Bytes(scratch.bytes(), FREQ.length, scratch.length()-FREQ.length);
      totalTermFreq += ArrayUtil.parseInt(scratchUTF16.chars(), 0, scratchUTF16.length()) - 1;
    } else if (StringHelper.startsWith(scratch.get(), TERM)) {
      if (lastDocsStart != -1) {
        fstCompiler.add(Util.toIntsRef(lastTerm.get(), scratchIntsRef), outputs.newPair(lastDocsStart,
            outputsInner.newPair((long) docFreq, totalTermFreq)));
      }
      lastDocsStart = in.getFilePointer();
      final int len = scratch.length() - TERM.length;
      lastTerm.grow(len);
      System.arraycopy(scratch.bytes(), TERM.length, lastTerm.bytes(), 0, len);
      lastTerm.setLength(len);
      docFreq = 0;
      sumTotalTermFreq += totalTermFreq;
      totalTermFreq = 0;
      termCount++;
    }
  }
  docCount = visitedDocs.cardinality();
  fst = fstCompiler.compile();
  /*
  PrintStream ps = new PrintStream("out.dot");
  fst.toDot(ps);
  ps.close();
  System.out.println("SAVED out.dot");
  */
  //System.out.println("FST " + fst.sizeInBytes());
}