Java Code Examples for org.apache.hadoop.hbase.KeyValue#KeyOnlyKeyValue

public Map<byte[], byte[]> loadFileInfo() throws IOException {
  Map<byte [], byte []> fi = reader.getHFileInfo();

  byte[] b = fi.get(BLOOM_FILTER_TYPE_KEY);
  if (b != null) {
    bloomFilterType = BloomType.valueOf(Bytes.toString(b));
  }

  byte[] p = fi.get(BLOOM_FILTER_PARAM_KEY);
  if (bloomFilterType ==  BloomType.ROWPREFIX_FIXED_LENGTH) {
    prefixLength = Bytes.toInt(p);
  }

  lastBloomKey = fi.get(LAST_BLOOM_KEY);
  if(bloomFilterType == BloomType.ROWCOL) {
    lastBloomKeyOnlyKV = new KeyValue.KeyOnlyKeyValue(lastBloomKey, 0, lastBloomKey.length);
  }
  byte[] cnt = fi.get(DELETE_FAMILY_COUNT);
  if (cnt != null) {
    deleteFamilyCnt = Bytes.toLong(cnt);
  }

  return fi;
}
 

@Override
public Cell getKey() {
  assertSeeked();
  // Create a new object so that this getKey is cached as firstKey, lastKey
  ObjectIntPair<ByteBuffer> keyPair = new ObjectIntPair<>();
  blockBuffer.asSubByteBuffer(blockBuffer.position() + KEY_VALUE_LEN_SIZE, currKeyLen, keyPair);
  ByteBuffer keyBuf = keyPair.getFirst();
  if (keyBuf.hasArray()) {
    return new KeyValue.KeyOnlyKeyValue(keyBuf.array(), keyBuf.arrayOffset()
        + keyPair.getSecond(), currKeyLen);
  } else {
    // Better to do a copy here instead of holding on to this BB so that
    // we could release the blocks referring to this key. This key is specifically used
    // in HalfStoreFileReader to get the firstkey and lastkey by creating a new scanner
    // every time. So holding onto the BB (incase of DBB) is not advised here.
    byte[] key = new byte[currKeyLen];
    ByteBufferUtils.copyFromBufferToArray(key, keyBuf, keyPair.getSecond(), 0, currKeyLen);
    return new KeyValue.KeyOnlyKeyValue(key, 0, currKeyLen);
  }
}
 

/**
 * In case of top half store, the passed key will be with the start key of the daughter region.
 * But in the actual HFiles, the key will be with the start key of the old parent region. In
 * order to make the real seek in the HFiles, we need to build the old key.
 *
 * The logic here is just replace daughter region start key with parent region start key
 * in the key part.
 *
 * @param key
 *
 */
private KeyValue getKeyPresentInHFiles(Cell keyValue) {
    int rowLength = keyValue.getRowLength();
    int rowOffset = keyValue.getRowOffset();

    short length = (short) (rowLength - reader.getSplitRow().length + reader.getOffset());
    byte[] replacedKey =
            new byte[length + keyValue.getRowArray().length - (rowOffset + rowLength) + ROW_LENGTH_SIZE];
    System.arraycopy(Bytes.toBytes(length), 0, replacedKey, 0, ROW_LENGTH_SIZE);
    System.arraycopy(reader.getRegionStartKeyInHFile(), 0, replacedKey, ROW_LENGTH_SIZE, reader.getOffset());
    System.arraycopy(keyValue.getRowArray(), keyValue.getRowOffset() + reader.getSplitRow().length,
        replacedKey, reader.getOffset() + ROW_LENGTH_SIZE, rowLength
                - reader.getSplitRow().length);
    System.arraycopy(keyValue.getRowArray(), rowOffset + rowLength, replacedKey,
        reader.getOffset() + keyValue.getRowLength() - reader.getSplitRow().length
                + ROW_LENGTH_SIZE, keyValue.getRowArray().length - (rowOffset + rowLength));
    return new KeyValue.KeyOnlyKeyValue(replacedKey);
}
 

/**
 * 
 * @param kv
 * @param isSeek pass true for seek, false for reseek.
 * @return 
 * @throws IOException
 */
public boolean seekOrReseek(Cell cell, boolean isSeek) throws IOException{
    Cell keyToSeek = cell;
    KeyValue splitKeyValue = new KeyValue.KeyOnlyKeyValue(reader.getSplitkey());
    if (reader.isTop()) {
        if(this.comparator.compare(cell, splitKeyValue, true) < 0){
            if(!isSeek && realSeekDone()) {
                return true;
            }
            return seekOrReseekToProperKey(isSeek, keyToSeek);
        }
        keyToSeek = getKeyPresentInHFiles(cell);
        return seekOrReseekToProperKey(isSeek, keyToSeek);
    } else {
        if (this.comparator.compare(cell, splitKeyValue, true) >= 0) {
            close();
            return false;
        }
        if(!isSeek && reader.getRegionInfo().getStartKey().length == 0 && reader.getSplitRow().length > reader.getRegionStartKeyInHFile().length) {
            keyToSeek = getKeyPresentInHFiles(cell);
        }
    }
    return seekOrReseekToProperKey(isSeek, keyToSeek);
}
 

private void loadMetaInfo(HFileBlock.BlockIterator blockIter, HFileContext hfileContext)
    throws IOException {
  read(blockIter.nextBlockWithBlockType(BlockType.FILE_INFO).getByteStream());
  byte[] creationTimeBytes = get(HFileInfo.CREATE_TIME_TS);
  hfileContext.setFileCreateTime(creationTimeBytes == null ?
      0 : Bytes.toLong(creationTimeBytes));
  byte[] tmp = get(HFileInfo.MAX_TAGS_LEN);
  // max tag length is not present in the HFile means tags were not at all written to file.
  if (tmp != null) {
    hfileContext.setIncludesTags(true);
    tmp = get(HFileInfo.TAGS_COMPRESSED);
    if (tmp != null && Bytes.toBoolean(tmp)) {
      hfileContext.setCompressTags(true);
    }
  }
  // parse meta info
  if (get(HFileInfo.LASTKEY) != null) {
    lastKeyCell = new KeyValue.KeyOnlyKeyValue(get(HFileInfo.LASTKEY));
  }
  avgKeyLen = Bytes.toInt(get(HFileInfo.AVG_KEY_LEN));
  avgValueLen = Bytes.toInt(get(HFileInfo.AVG_VALUE_LEN));
  byte [] keyValueFormatVersion = get(HFileWriterImpl.KEY_VALUE_VERSION);
  includesMemstoreTS = keyValueFormatVersion != null &&
      Bytes.toInt(keyValueFormatVersion) == HFileWriterImpl.KEY_VALUE_VER_WITH_MEMSTORE;
  hfileContext.setIncludesMvcc(includesMemstoreTS);
  if (includesMemstoreTS) {
    decodeMemstoreTS = Bytes.toLong(get(HFileWriterImpl.MAX_MEMSTORE_TS_KEY)) > 0;
  }
}
 

/**
 * Adds a new entry in the root block index. Only used when reading.
 *
 * @param key Last key in the block
 * @param offset file offset where the block is stored
 * @param dataSize the uncompressed data size
 */
@Override
protected void add(final byte[] key, final long offset, final int dataSize) {
  blockOffsets[rootCount] = offset;
  // Create the blockKeys as Cells once when the reader is opened
  blockKeys[rootCount] = new KeyValue.KeyOnlyKeyValue(key, 0, key.length);
  blockDataSizes[rootCount] = dataSize;
  rootCount++;
}
 

protected Cell getFirstKeyCellInBlock(HFileBlock curBlock) {
  ByteBuff buffer = curBlock.getBufferWithoutHeader();
  // It is safe to manipulate this buffer because we own the buffer object.
  buffer.rewind();
  int klen = buffer.getInt();
  buffer.skip(Bytes.SIZEOF_INT);// Skip value len part
  ByteBuffer keyBuff = buffer.asSubByteBuffer(klen);
  if (keyBuff.hasArray()) {
    return new KeyValue.KeyOnlyKeyValue(keyBuff.array(), keyBuff.arrayOffset()
        + keyBuff.position(), klen);
  } else {
    return new ByteBufferKeyOnlyKeyValue(keyBuff, keyBuff.position(), klen);
  }
}
 

/**
 * Creates a half file reader for a hfile referred to by an hfilelink.
 * @param context Reader context info
 * @param fileInfo HFile info
 * @param cacheConf CacheConfig
 * @param r original reference file (contains top or bottom)
 * @param refCount reference count
 * @param conf Configuration
 */
public HalfStoreFileReader(final ReaderContext context, final HFileInfo fileInfo,
    final CacheConfig cacheConf, final Reference r,
    AtomicInteger refCount, final Configuration conf) throws IOException {
  super(context, fileInfo, cacheConf, refCount, conf);
  // This is not actual midkey for this half-file; its just border
  // around which we split top and bottom.  Have to look in files to find
  // actual last and first keys for bottom and top halves.  Half-files don't
  // have an actual midkey themselves. No midkey is how we indicate file is
  // not splittable.
  this.splitkey = r.getSplitKey();
  this.splitCell = new KeyValue.KeyOnlyKeyValue(this.splitkey, 0, this.splitkey.length);
  // Is it top or bottom half?
  this.top = Reference.isTopFileRegion(r.getFileRegion());
}
 

protected Cell createFirstKeyCell(ByteBuffer key, int keyLength) {
  if (key.hasArray()) {
    return new KeyValue.KeyOnlyKeyValue(key.array(), key.arrayOffset()
        + key.position(), keyLength);
  } else {
    return new ByteBufferKeyOnlyKeyValue(key, key.position(), keyLength);
  }
}
 

@Override
public Cell getKey() {
  if (current.keyBuffer.hasArray()) {
    return new KeyValue.KeyOnlyKeyValue(current.keyBuffer.array(),
        current.keyBuffer.arrayOffset() + current.keyBuffer.position(),
        current.keyLength);
  } else {
    byte[] key = new byte[current.keyLength];
    ByteBufferUtils.copyFromBufferToArray(key, current.keyBuffer,
        current.keyBuffer.position(), 0, current.keyLength);
    return new KeyValue.KeyOnlyKeyValue(key, 0, current.keyLength);
  }
}
 

@Override
public Cell midkey(CachingBlockReader cachingBlockReader) throws IOException {
  if (rootCount == 0)
    throw new IOException("HFile empty");

  Cell targetMidKey = this.midKey.get();
  if (targetMidKey != null) {
    return targetMidKey;
  }

  if (midLeafBlockOffset >= 0) {
    if (cachingBlockReader == null) {
      throw new IOException("Have to read the middle leaf block but " +
          "no block reader available");
    }

    // Caching, using pread, assuming this is not a compaction.
    HFileBlock midLeafBlock = cachingBlockReader.readBlock(
        midLeafBlockOffset, midLeafBlockOnDiskSize, true, true, false, true,
        BlockType.LEAF_INDEX, null);
    try {
      ByteBuff b = midLeafBlock.getBufferWithoutHeader();
      int numDataBlocks = b.getIntAfterPosition(0);
      int keyRelOffset = b.getIntAfterPosition(Bytes.SIZEOF_INT * (midKeyEntry + 1));
      int keyLen = b.getIntAfterPosition(Bytes.SIZEOF_INT * (midKeyEntry + 2)) - keyRelOffset
          - SECONDARY_INDEX_ENTRY_OVERHEAD;
      int keyOffset =
          Bytes.SIZEOF_INT * (numDataBlocks + 2) + keyRelOffset
              + SECONDARY_INDEX_ENTRY_OVERHEAD;
      byte[] bytes = b.toBytes(keyOffset, keyLen);
      targetMidKey = new KeyValue.KeyOnlyKeyValue(bytes, 0, bytes.length);
    } finally {
      midLeafBlock.release();
    }
  } else {
    // The middle of the root-level index.
    targetMidKey = blockKeys[rootCount / 2];
  }

  this.midKey.set(targetMidKey);
  return targetMidKey;
}
 

private void readIndex(boolean useTags) throws IOException {
  long fileSize = fs.getFileStatus(path).getLen();
  LOG.info("Size of {}: {} compression={}", path, fileSize, compr.toString());

  FSDataInputStream istream = fs.open(path);
  HFileContext meta = new HFileContextBuilder()
                      .withHBaseCheckSum(true)
                      .withIncludesMvcc(includesMemstoreTS)
                      .withIncludesTags(useTags)
                      .withCompression(compr)
                      .build();
  ReaderContext context = new ReaderContextBuilder().withFileSystemAndPath(fs, path).build();
  HFileBlock.FSReader blockReader = new HFileBlock.FSReaderImpl(context, meta,
      ByteBuffAllocator.HEAP);

  BlockReaderWrapper brw = new BlockReaderWrapper(blockReader);
  HFileBlockIndex.BlockIndexReader indexReader =
      new HFileBlockIndex.CellBasedKeyBlockIndexReader(
          CellComparatorImpl.COMPARATOR, numLevels);

  indexReader.readRootIndex(blockReader.blockRange(rootIndexOffset,
      fileSize).nextBlockWithBlockType(BlockType.ROOT_INDEX), numRootEntries);

  long prevOffset = -1;
  int i = 0;
  int expectedHitCount = 0;
  int expectedMissCount = 0;
  LOG.info("Total number of keys: " + keys.size());
  for (byte[] key : keys) {
    assertTrue(key != null);
    assertTrue(indexReader != null);
    KeyValue.KeyOnlyKeyValue keyOnlyKey = new KeyValue.KeyOnlyKeyValue(key, 0, key.length);
    HFileBlock b =
        indexReader.seekToDataBlock(keyOnlyKey, null, true,
          true, false, null, brw);
    if (PrivateCellUtil.compare(CellComparatorImpl.COMPARATOR, keyOnlyKey, firstKeyInFile, 0,
      firstKeyInFile.length) < 0) {
      assertTrue(b == null);
      ++i;
      continue;
    }

    String keyStr = "key #" + i + ", " + Bytes.toStringBinary(key);

    assertTrue("seekToDataBlock failed for " + keyStr, b != null);

    if (prevOffset == b.getOffset()) {
      assertEquals(++expectedHitCount, brw.hitCount);
    } else {
      LOG.info("First key in a new block: " + keyStr + ", block offset: "
          + b.getOffset() + ")");
      assertTrue(b.getOffset() > prevOffset);
      assertEquals(++expectedMissCount, brw.missCount);
      prevOffset = b.getOffset();
    }
    ++i;
  }

  istream.close();
}
 

@Override
public Cell getKey() {
  byte[] key = new byte[current.keyLength];
  System.arraycopy(current.keyBuffer, 0, key, 0, current.keyLength);
  return new KeyValue.KeyOnlyKeyValue(key);
}