org.apache.hadoop.hbase.CellComparator Java Examples

public void testBasicLoadValue() throws Exception {
  KeyValue [] kvs = genKVs(row, family, value, 1, 100);

  Arrays.sort(kvs, CellComparator.getInstance());

  Result r = Result.create(kvs);
  ByteBuffer loadValueBuffer = ByteBuffer.allocate(1024);

  for (int i = 0; i < 100; ++i) {
    final byte[] qf = Bytes.toBytes(i);

    loadValueBuffer.clear();
    r.loadValue(family, qf, loadValueBuffer);
    loadValueBuffer.flip();
    assertEquals(loadValueBuffer, ByteBuffer.wrap(Bytes.add(value, Bytes.toBytes(i))));
    assertEquals(ByteBuffer.wrap(Bytes.add(value, Bytes.toBytes(i))),
        r.getValueAsByteBuffer(family, qf));
  }
}
 

public CompactingMemStore(Configuration conf, CellComparator c,
    HStore store, RegionServicesForStores regionServices,
    MemoryCompactionPolicy compactionPolicy) throws IOException {
  super(conf, c, regionServices);
  this.store = store;
  this.regionServices = regionServices;
  this.pipeline = new CompactionPipeline(getRegionServices());
  this.compactor = createMemStoreCompactor(compactionPolicy);
  if (conf.getBoolean(MemStoreLAB.USEMSLAB_KEY, MemStoreLAB.USEMSLAB_DEFAULT)) {
    // if user requested to work with MSLABs (whether on- or off-heap), then the
    // immutable segments are going to use CellChunkMap as their index
    indexType = IndexType.CHUNK_MAP;
  } else {
    indexType = IndexType.ARRAY_MAP;
  }
  // initialization of the flush size should happen after initialization of the index type
  // so do not transfer the following method
  initInmemoryFlushSize(conf);
  LOG.info("Store={}, in-memory flush size threshold={}, immutable segments index type={}, " +
          "compactor={}", this.store.getColumnFamilyName(),
      StringUtils.byteDesc(this.inmemoryFlushSize), this.indexType,
      (this.compactor == null? "NULL": this.compactor.toString()));
}
 

@Test
public void testScanSameTimestamp() throws IOException {
  // returns only 1 of these 2 even though same timestamp
  KeyValue [] kvs = new KeyValue[] {
      create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
      create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
  };
  List<KeyValueScanner> scanners = Arrays.asList(
    new KeyValueScanner[] {new KeyValueScanFixture(CellComparator.getInstance(), kvs)});

  Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1"));
  // this only uses maxVersions (default=1) and TimeRange (default=all)
  try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, null, scanners)) {
    List<Cell> results = new ArrayList<>();
    assertEquals(true, scan.next(results));
    assertEquals(1, results.size());
    assertEquals(1, scan.memstoreOnlyReads);
    assertEquals(kvs[0], results.get(0));
  }
}
 

@Override
public boolean equals(final Object obj) {
  if (this == obj) {
    return true;
  }
  if (obj == null) {
    return false;
  }
  if (getClass() != obj.getClass()) {
    return false;
  }
  final PartialCellEquality other = (PartialCellEquality) obj;
  return CellComparator.equalsFamily(cell, other.cell)
      && CellComparator.equalsQualifier(cell, other.cell)
      && (!includeTags || tagsEqual(cell, other.cell));
}
 

@Override
public int rootBlockContainingKey(byte[] key, int offset, int length, CellComparator comp) {
  int pos = Bytes.binarySearch(blockKeys, key, offset, length);
  // pos is between -(blockKeys.length + 1) to blockKeys.length - 1, see
  // binarySearch's javadoc.

  if (pos >= 0) {
    // This means this is an exact match with an element of blockKeys.
    assert pos < blockKeys.length;
    return pos;
  }

  // Otherwise, pos = -(i + 1), where blockKeys[i - 1] < key < blockKeys[i],
  // and i is in [0, blockKeys.length]. We are returning j = i - 1 such that
  // blockKeys[j] <= key < blockKeys[j + 1]. In particular, j = -1 if
  // key < blockKeys[0], meaning the file does not contain the given key.

  int i = -pos - 1;
  assert 0 <= i && i <= blockKeys.length;
  return i - 1;
}
 

/**
 * Ensure that expired delete family markers don't override valid puts
 */
@Test
public void testExpiredDeleteFamily() throws Exception {
  long now = System.currentTimeMillis();
  KeyValue[] kvs = new KeyValue[] {
    new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now-1000,
      KeyValue.Type.DeleteFamily),
    create("R1", "cf", "a", now-10, KeyValue.Type.Put,
      "dont-care"),
  };
  List<KeyValueScanner> scanners = scanFixture(kvs);
  Scan scan = new Scan();
  scan.readVersions(1);
  // scanner with ttl equal to 500
  ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE,
      HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
  try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) {
    List<Cell> results = new ArrayList<>();
    assertEquals(true, scanner.next(results));
    assertEquals(1, results.size());
    assertEquals(kvs[1], results.get(0));
    results.clear();

    assertEquals(false, scanner.next(results));
  }
}
 

protected Segment(CellComparator comparator, List<ImmutableSegment> segments,
    TimeRangeTracker trt) {
  long dataSize = 0;
  long heapSize = 0;
  long OffHeapSize = 0;
  int cellsCount = 0;
  for (Segment segment : segments) {
    MemStoreSize memStoreSize = segment.getMemStoreSize();
    dataSize += memStoreSize.getDataSize();
    heapSize += memStoreSize.getHeapSize();
    OffHeapSize += memStoreSize.getOffHeapSize();
    cellsCount += memStoreSize.getCellsCount();
  }
  this.comparator = comparator;
  this.updatesLock = new ReentrantReadWriteLock();
  // Do we need to be thread safe always? What if ImmutableSegment?
  // DITTO for the TimeRangeTracker below.
  this.memStoreSizing = new ThreadSafeMemStoreSizing(dataSize, heapSize, OffHeapSize, cellsCount);
  this.timeRangeTracker = trt;
}
 

public void testBasicGetColumn() throws Exception {
  KeyValue [] kvs = genKVs(row, family, value, 1, 100);

  Arrays.sort(kvs, CellComparator.getInstance());

  Result r = Result.create(kvs);

  for (int i = 0; i < 100; ++i) {
    final byte[] qf = Bytes.toBytes(i);

    List<Cell> ks = r.getColumnCells(family, qf);
    assertEquals(1, ks.size());
    assertTrue(CellUtil.matchingQualifier(ks.get(0), qf));
    assertEquals(ks.get(0), r.getColumnLatestCell(family, qf));
  }
}
 

@SuppressWarnings("unchecked")
private static Class<? extends CellComparator> getComparatorClass(String comparatorClassName)
  throws IOException {
  Class<? extends CellComparator> comparatorKlass;
  // for BC
  if (comparatorClassName.equals(KeyValue.COMPARATOR.getLegacyKeyComparatorName())
    || comparatorClassName.equals(KeyValue.COMPARATOR.getClass().getName())
    || (comparatorClassName.equals("org.apache.hadoop.hbase.CellComparator"))) {
    comparatorKlass = CellComparatorImpl.class;
  } else if (comparatorClassName.equals(KeyValue.META_COMPARATOR.getLegacyKeyComparatorName())
    || comparatorClassName.equals(KeyValue.META_COMPARATOR.getClass().getName())
    || (comparatorClassName
    .equals("org.apache.hadoop.hbase.CellComparator$MetaCellComparator"))) {
    comparatorKlass = MetaCellComparator.class;
  } else if (comparatorClassName.equals("org.apache.hadoop.hbase.KeyValue$RawBytesComparator")
    || comparatorClassName.equals("org.apache.hadoop.hbase.util.Bytes$ByteArrayComparator")) {
    // When the comparator to be used is Bytes.BYTES_RAWCOMPARATOR, we just return null from here
    // Bytes.BYTES_RAWCOMPARATOR is not a CellComparator
    comparatorKlass = null;
  } else {
    // if the name wasn't one of the legacy names, maybe its a legit new kind of comparator.
    try {
      comparatorKlass = (Class<? extends CellComparator>) Class.forName(comparatorClassName);
    } catch (ClassNotFoundException e) {
      throw new IOException(e);
    }
  }
  return comparatorKlass;
}
 

public StripeStoreFileManager(
    CellComparator kvComparator, Configuration conf, StripeStoreConfig config) {
  this.cellComparator = kvComparator;
  this.config = config;
  this.blockingFileCount = conf.getInt(
      HStore.BLOCKING_STOREFILES_KEY, HStore.DEFAULT_BLOCKING_STOREFILE_COUNT);
}
 

public MemStoreCompactorSegmentsIterator(List<ImmutableSegment> segments,
    CellComparator comparator, int compactionKVMax, HStore store) throws IOException {
  super(compactionKVMax);

  List<KeyValueScanner> scanners = new ArrayList<KeyValueScanner>();
  AbstractMemStore.addToScanners(segments, Long.MAX_VALUE, scanners);
  // build the scanner based on Query Matcher
  // reinitialize the compacting scanner for each instance of iterator
  compactingScanner = createScanner(store, scanners);
  refillKVS();
}
 

@Override
@Before
public void setUp() throws Exception {
  compactingSetUp();
  this.memstore = new MyCompactingMemStore(HBaseConfiguration.create(), CellComparator.getInstance(),
      store, regionServicesForStores, MemoryCompactionPolicy.EAGER);
  ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.ARRAY_MAP);
}
 

/**
 * Searches for the latest value for the specified column.
 *
 * @param kvs the array to search
 * @param family family name
 * @param foffset family offset
 * @param flength family length
 * @param qualifier column qualifier
 * @param qoffset qualifier offset
 * @param qlength qualifier length
 *
 * @return the index where the value was found, or -1 otherwise
 */
protected int binarySearch(final Cell [] kvs,
    final byte [] family, final int foffset, final int flength,
    final byte [] qualifier, final int qoffset, final int qlength) {

  double keyValueSize = (double)
      KeyValue.getKeyValueDataStructureSize(kvs[0].getRowLength(), flength, qlength, 0);

  byte[] buffer = localBuffer.get();
  if (buffer == null || keyValueSize > buffer.length) {
    // pad to the smallest multiple of the pad width
    buffer = new byte[(int) Math.ceil(keyValueSize / PAD_WIDTH) * PAD_WIDTH];
    localBuffer.set(buffer);
  }

  Cell searchTerm = KeyValueUtil.createFirstOnRow(buffer, 0,
      kvs[0].getRowArray(), kvs[0].getRowOffset(), kvs[0].getRowLength(),
      family, foffset, flength,
      qualifier, qoffset, qlength);

  // pos === ( -(insertion point) - 1)
  int pos = Arrays.binarySearch(kvs, searchTerm, CellComparator.getInstance());
  // never will exact match
  if (pos < 0) {
    pos = (pos+1) * -1;
    // pos is now insertion point
  }
  if (pos == kvs.length) {
    return -1; // doesn't exist
  }
  return pos;
}
 

/**
 * @param chunkByteSizeHint
 *          each chunk's size in bytes. The real chunk size might be different
 *          as required by the fold factor.
 * @param errorRate
 *          target false positive rate
 * @param hashType
 *          hash function type to use
 * @param maxFold
 *          maximum degree of folding allowed
 * @param bloomType
 *          the bloom type
 */
public CompoundBloomFilterWriter(int chunkByteSizeHint, float errorRate,
    int hashType, int maxFold, boolean cacheOnWrite,
    CellComparator comparator, BloomType bloomType) {
  chunkByteSize = BloomFilterUtil.computeFoldableByteSize(
      chunkByteSizeHint * 8L, maxFold);

  this.errorRate = errorRate;
  this.hashType = hashType;
  this.maxFold = maxFold;
  this.cacheOnWrite = cacheOnWrite;
  this.comparator = comparator;
  this.bloomType = bloomType;
}
 

public void testBasicGetValue() throws Exception {
  KeyValue [] kvs = genKVs(row, family, value, 1, 100);

  Arrays.sort(kvs, CellComparator.getInstance());

  Result r = Result.create(kvs);

  for (int i = 0; i < 100; ++i) {
    final byte[] qf = Bytes.toBytes(i);

    assertArrayEquals(Bytes.add(value, Bytes.toBytes(i)), r.getValue(family, qf));
    assertTrue(r.containsColumn(family, qf));
  }
}
 

public ImmutableSegment createImmutableSegmentByMerge(final Configuration conf,
    final CellComparator comparator, MemStoreSegmentsIterator iterator, int numOfCells,
    List<ImmutableSegment> segments, CompactingMemStore.IndexType idxType,
    MemStoreCompactionStrategy.Action action)
    throws IOException {

  MemStoreLAB memStoreLAB = getMergedMemStoreLAB(conf, segments);
  return
      createImmutableSegment(
          conf,comparator,iterator,memStoreLAB,numOfCells,action,idxType);

}
 

private ScanInfo getScanInfo() {
  int oldMaxVersions = 1;
  int oldMinVersions = 0;
  long oldTTL = 10000;

  return new ScanInfo(conf, Bytes.toBytes("cf"), oldMinVersions, oldMaxVersions, oldTTL,
  KeepDeletedCells.FALSE, HConstants.FOREVER, 1000,
    CellComparator.getInstance(), true);
}
 

public void setComparatorClass(Class<? extends CellComparator> klass) {
  // Is the comparator instantiable?
  try {
    // If null, it should be the Bytes.BYTES_RAWCOMPARATOR
    if (klass != null) {
      CellComparator comp = klass.getDeclaredConstructor().newInstance();
      // if the name wasn't one of the legacy names, maybe its a legit new
      // kind of comparator.
      this.comparatorClassName = klass.getName();
    }
  } catch (Exception e) {
    throw new RuntimeException("Comparator class " + klass.getName() + " is not instantiable", e);
  }
}
 

public static List<KeyValueScanner> scanFixture(KeyValue[] ... kvArrays) {
  ArrayList<KeyValueScanner> scanners = new ArrayList<>();
  for (KeyValue [] kvs : kvArrays) {
    scanners.add(new KeyValueScanFixture(CellComparator.getInstance(), kvs));
  }
  return scanners;
}
 

public MemStoreMergerSegmentsIterator(List<ImmutableSegment> segments, CellComparator comparator,
    int compactionKVMax) throws IOException {
  super(compactionKVMax);
  // create the list of scanners to traverse over all the data
  // no dirty reads here as these are immutable segments
  AbstractMemStore.addToScanners(segments, Long.MAX_VALUE, scanners);
  heap = new KeyValueHeap(scanners, comparator);
}
 

/**
 * Creates the mob store engine.
 */
@Override
protected StoreEngine<?, ?, ?, ?> createStoreEngine(HStore store, Configuration conf,
    CellComparator cellComparator) throws IOException {
  MobStoreEngine engine = new MobStoreEngine();
  engine.createComponents(conf, store, cellComparator);
  return engine;
}
 

@Override
public boolean filterRowKey(Cell firstRowCell) {
  // if stopRowKey is <= buffer, then true, filter row.
  if (filterAllRemaining()) return true;
  int cmp = CellComparator.getInstance().compareRows(firstRowCell, stopRowKey, 0, stopRowKey.length);
  done = reversed ? cmp < 0 : cmp > 0;
  return done;
}
 

/**
 * Create a new {@link IndexKeyValueSkipListSet} based on the passed comparator.
 * @param comparator to use when comparing keyvalues. It is used both to determine sort order as
 *          well as object equality in the map.
 * @return a map that uses the passed comparator
 */
public static IndexKeyValueSkipListSet create(CellComparator comparator) {
  ConcurrentSkipListMap<Cell, Cell> delegate =
      new ConcurrentSkipListMap<Cell, Cell>(comparator);
  IndexKeyValueSkipListSet ret = new IndexKeyValueSkipListSet(delegate);
  return ret;
}
 

public DefaultStoreFileManager(CellComparator cellComparator,
    Comparator<HStoreFile> storeFileComparator, Configuration conf,
    CompactionConfiguration comConf) {
  this.cellComparator = cellComparator;
  this.storeFileComparator = storeFileComparator;
  this.comConf = comConf;
  this.blockingFileCount =
      conf.getInt(HStore.BLOCKING_STOREFILES_KEY, HStore.DEFAULT_BLOCKING_STOREFILE_COUNT);
}
 

protected AbstractMemStore(final Configuration conf, final CellComparator c,
    final RegionServicesForStores regionServices) {
  this.conf = conf;
  this.comparator = c;
  this.regionServices = regionServices;
  resetActive();
  this.snapshot = SegmentFactory.instance().createImmutableSegment(c);
  this.snapshotId = NO_SNAPSHOT_ID;
}
 

/**
 * @param targetCount The maximum count of writers that can be created.
 * @param targetKvs The number of KVs to read from source before starting each new writer.
 * @param left The left boundary of the first writer.
 * @param right The right boundary of the last writer.
 */
public SizeMultiWriter(CellComparator comparator, int targetCount, long targetKvs, byte[] left,
    byte[] right) {
  super(comparator);
  this.targetCount = targetCount;
  this.targetCells = targetKvs;
  this.left = left;
  this.right = right;
  int preallocate = Math.min(this.targetCount, 64);
  this.existingWriters = new ArrayList<>(preallocate);
  this.boundaries = new ArrayList<>(preallocate + 1);
}
 

@Test
public void testWildCardTtlScan() throws IOException {
  long now = System.currentTimeMillis();
  KeyValue [] kvs = new KeyValue[] {
      create("R1", "cf", "a", now-1000, KeyValue.Type.Put, "dont-care"),
      create("R1", "cf", "b", now-10, KeyValue.Type.Put, "dont-care"),
      create("R1", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"),
      create("R1", "cf", "d", now-10000, KeyValue.Type.Put, "dont-care"),
      create("R2", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
      create("R2", "cf", "b", now-10, KeyValue.Type.Put, "dont-care"),
      create("R2", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"),
      create("R2", "cf", "c", now-1000, KeyValue.Type.Put, "dont-care")
  };
  List<KeyValueScanner> scanners = scanFixture(kvs);
  Scan scan = new Scan();
  scan.readVersions(1);
  ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE,
      HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
  try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) {
    List<Cell> results = new ArrayList<>();
    assertEquals(true, scanner.next(results));
    assertEquals(2, results.size());
    assertEquals(kvs[1], results.get(0));
    assertEquals(kvs[2], results.get(1));
    results.clear();

    assertEquals(true, scanner.next(results));
    assertEquals(3, results.size());
    assertEquals(kvs[4], results.get(0));
    assertEquals(kvs[5], results.get(1));
    assertEquals(kvs[6], results.get(2));
    results.clear();

    assertEquals(false, scanner.next(results));
  }
}
 

protected Segment(CellComparator comparator, TimeRangeTracker trt) {
  this.comparator = comparator;
  // Do we need to be thread safe always? What if ImmutableSegment?
  // DITTO for the TimeRangeTracker below.
  this.memStoreSizing = new ThreadSafeMemStoreSizing();
  this.timeRangeTracker = trt;
}
 

/**
 * Verify the data described by <code>simple_tsv</code> matches
 * <code>simple_expected</code>.
 */
protected void doLoadIncrementalHFiles(Path hfiles, TableName tableName)
    throws Exception {

  String[] args = { hfiles.toString(), tableName.getNameAsString() };
  LOG.info(format("Running LoadIncrememntalHFiles with args: %s", Arrays.asList(args)));
  assertEquals("Loading HFiles failed.", 0,
    ToolRunner.run(new BulkLoadHFilesTool(getConf()), args));

  Table table = null;
  Scan scan = new Scan() {{
    setCacheBlocks(false);
    setCaching(1000);
  }};
  try {
    table = util.getConnection().getTable(tableName);
    Iterator<Result> resultsIt = table.getScanner(scan).iterator();
    Iterator<KeyValue> expectedIt = simple_expected.iterator();
    while (resultsIt.hasNext() && expectedIt.hasNext()) {
      Result r = resultsIt.next();
      for (Cell actual : r.rawCells()) {
        assertTrue(
          "Ran out of expected values prematurely!",
          expectedIt.hasNext());
        KeyValue expected = expectedIt.next();
        assertEquals("Scan produced surprising result", 0,
          CellComparator.getInstance().compare(expected, actual));
      }
    }
    assertFalse("Did not consume all expected values.", expectedIt.hasNext());
    assertFalse("Did not consume all scan results.", resultsIt.hasNext());
  } finally {
    if (null != table) table.close();
  }
}
 

@Override
protected void checkScanOrder(Cell prevKV, Cell kv,
    CellComparator comparator) throws IOException {
  // Check that the heap gives us KVs in an increasing order for same row and
  // decreasing order for different rows.
  assert prevKV == null || comparator == null || comparator.compareRows(kv, prevKV) < 0
      || (CellUtil.matchingRows(kv, prevKV) && comparator.compare(kv,
          prevKV) >= 0) : "Key " + prevKV
      + " followed by a " + "error order key " + kv + " in cf " + store
      + " in reversed scan";
}