Java Code Examples for org.apache.hadoop.hbase.HBaseTestingUtility#createRegionAndWAL()

@Before
public void setUp() throws Exception {
  TableDescriptor htd = TableDescriptorBuilder
      .newBuilder(TableName.valueOf("TestQualifierFilter"))
      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILY).build()).build();
  RegionInfo info = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
  this.region = HBaseTestingUtility
      .createRegionAndWAL(info, TEST_UTIL.getDataTestDir(), TEST_UTIL.getConfiguration(), htd);

  // Insert data
  for (byte[] ROW : ROWS) {
    Put p = new Put(ROW);
    p.setDurability(Durability.SKIP_WAL);
    for (byte[] QUALIFIER : QUALIFIERS) {
      p.addColumn(FAMILY, QUALIFIER, VALUE);
    }
    this.region.put(p);
  }

  // Flush
  this.region.flush(true);
}
 

@Before
public void setUp() throws Exception {
  testVals = makeTestVals();

  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(
      TableName.valueOf(this.getClass().getSimpleName()));

  ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor hcd0 =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(FAMILIES[0])
      .setMaxVersions(3);
  tableDescriptor.setColumnFamily(hcd0);
  ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor hcd1 =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(FAMILIES[1])
      .setMaxVersions(3);
  tableDescriptor.setColumnFamily(hcd1);
  RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
  this.region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(),
      TEST_UTIL.getConfiguration(), tableDescriptor);
  addData();
}
 

@Test
public void testReadAndWriteRegionInfoFile() throws IOException, InterruptedException {
  RegionInfo ri = RegionInfoBuilder.FIRST_META_REGIONINFO;
  // Create a region. That'll write the .regioninfo file.
  FSTableDescriptors fsTableDescriptors = new FSTableDescriptors(FS, ROOT_DIR);
  FSTableDescriptors.tryUpdateMetaTableDescriptor(CONF, FS, ROOT_DIR, null);
  HRegion r = HBaseTestingUtility.createRegionAndWAL(ri, ROOT_DIR, CONF,
    fsTableDescriptors.get(TableName.META_TABLE_NAME));
  // Get modtime on the file.
  long modtime = getModTime(r);
  HBaseTestingUtility.closeRegionAndWAL(r);
  Thread.sleep(1001);
  r = HRegion.openHRegion(ROOT_DIR, ri, fsTableDescriptors.get(TableName.META_TABLE_NAME), null,
    CONF);
  // Ensure the file is not written for a second time.
  long modtime2 = getModTime(r);
  assertEquals(modtime, modtime2);
  // Now load the file.
  HRegionFileSystem.loadRegionInfoFileContent(r.getRegionFileSystem().getFileSystem(),
    r.getRegionFileSystem().getRegionDir());
  HBaseTestingUtility.closeRegionAndWAL(r);
}
 

/**
 * Callers must afterward call {@link HBaseTestingUtility#closeRegionAndWAL(HRegion)}
 */
private HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
    String family, BlockCache blockCache) throws IOException {
  TableDescriptorBuilder builder =
      TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName));
  for (int i = 0; i < BLOOM_TYPE.length; i++) {
    BloomType bloomType = BLOOM_TYPE[i];
    builder.setColumnFamily(
        ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family + "_" + bloomType))
            .setBlocksize(1).setBloomFilterType(bloomType).build());
  }
  RegionInfo info = RegionInfoBuilder.newBuilder(TableName.valueOf(tableName)).build();
  Path path = new Path(DIR + callingMethod);
  if (blockCache != null) {
    return HBaseTestingUtility.createRegionAndWAL(info, path, conf, builder.build(), blockCache);
  } else {
    return HBaseTestingUtility.createRegionAndWAL(info, path, conf, builder.build());
  }
}
 

@BeforeClass
public static void setUpBeforeClass() throws Exception {
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(TABLE_NAME);

  for (byte[] family : FAMILIES) {
    ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
      new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(family);
    tableDescriptor.setColumnFamily(familyDescriptor);
  }
  RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
  REGION = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(),
    TEST_UTIL.getConfiguration(), tableDescriptor);
  for(Put put:createPuts(ROWS, FAMILIES, QUALIFIERS, VALUE)){
    REGION.put(put);
  }
}
 

private static HRegion initHRegion(TableName tableName, byte[] startKey, byte[] stopKey,
    String callingMethod, Configuration conf, HBaseTestingUtility testUtil, boolean isReadOnly,
    byte[]... families) throws IOException {
  RegionInfo regionInfo =
      RegionInfoBuilder.newBuilder(tableName).setStartKey(startKey).setEndKey(stopKey).build();
  TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName);
  builder.setReadOnly(isReadOnly).setDurability(Durability.SYNC_WAL);
  for (byte[] family : families) {
    builder.setColumnFamily(
        ColumnFamilyDescriptorBuilder.newBuilder(family).setMaxVersions(Integer.MAX_VALUE)
            .build());
  }
  return HBaseTestingUtility
      .createRegionAndWAL(regionInfo, testUtil.getDataTestDir(callingMethod), conf,
          builder.build(), BlockCacheFactory.createBlockCache(conf));
}
 

private void init(Configuration conf, long mobThreshold) throws Exception {
  this.conf = conf;
  this.mobCellThreshold = mobThreshold;
  HBaseTestingUtility UTIL = new HBaseTestingUtility(conf);

  compactionThreshold = conf.getInt("hbase.hstore.compactionThreshold", 3);
  tableDescriptor = UTIL.createModifyableTableDescriptor(name.getMethodName());
  familyDescriptor =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(COLUMN_FAMILY);
  familyDescriptor.setMobEnabled(true);
  familyDescriptor.setMobThreshold(mobThreshold);
  familyDescriptor.setMaxVersions(1);
  tableDescriptor.modifyColumnFamily(familyDescriptor);

  RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
  region = HBaseTestingUtility.createRegionAndWAL(regionInfo,
    UTIL.getDataTestDir(), conf, tableDescriptor, new MobFileCache(conf));
  fs = FileSystem.get(conf);
}
 

@BeforeClass
public static void setUp() throws Exception {
  CONF = TEST_UTIL.getConfiguration();
  cacheConf = new CacheConfig(CONF);
  fs = TEST_UTIL.getTestFileSystem();
  TableName tableName = TableName.valueOf("test");
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);
  tableDescriptor.setColumnFamily(
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(fam));
  RegionInfo info = RegionInfoBuilder.newBuilder(tableName).build();
  Path path = TEST_UTIL.getDataTestDir("test");
  region = HBaseTestingUtility.createRegionAndWAL(info, path,
    TEST_UTIL.getConfiguration(), tableDescriptor);
}
 

/**
 * testcase for https://issues.apache.org/jira/browse/HBASE-14949.
 */
private void testNameConflictWhenSplit(boolean largeFirst) throws IOException,
    StreamLacksCapabilityException {
  final TableName tableName = TableName.valueOf("testReplayEditsWrittenIntoWAL");
  final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
  final RegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
  final Path basedir = CommonFSUtils.getTableDir(hbaseRootDir, tableName);
  deleteDir(basedir);

  final HTableDescriptor htd = createBasic1FamilyHTD(tableName);
  NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
  for (byte[] fam : htd.getFamiliesKeys()) {
    scopes.put(fam, 0);
  }
  HRegion region = HBaseTestingUtility.createRegionAndWAL(hri, hbaseRootDir, this.conf, htd);
  HBaseTestingUtility.closeRegionAndWAL(region);
  final byte[] family = htd.getColumnFamilies()[0].getName();
  final byte[] rowName = tableName.getName();
  FSWALEntry entry1 = createFSWALEntry(htd, hri, 1L, rowName, family, ee, mvcc, 1, scopes);
  FSWALEntry entry2 = createFSWALEntry(htd, hri, 2L, rowName, family, ee, mvcc, 2, scopes);

  Path largeFile = new Path(logDir, "wal-1");
  Path smallFile = new Path(logDir, "wal-2");
  writerWALFile(largeFile, Arrays.asList(entry1, entry2));
  writerWALFile(smallFile, Arrays.asList(entry2));
  FileStatus first, second;
  if (largeFirst) {
    first = fs.getFileStatus(largeFile);
    second = fs.getFileStatus(smallFile);
  } else {
    first = fs.getFileStatus(smallFile);
    second = fs.getFileStatus(largeFile);
  }
  WALSplitter.splitLogFile(hbaseRootDir, first, fs, conf, null, null, null, wals, null);
  WALSplitter.splitLogFile(hbaseRootDir, second, fs, conf, null, null, null, wals, null);
  WAL wal = createWAL(this.conf, hbaseRootDir, logName);
  region = HRegion.openHRegion(conf, this.fs, hbaseRootDir, hri, htd, wal);
  assertTrue(region.getOpenSeqNum() > mvcc.getWritePoint());
  assertEquals(2, region.get(new Get(rowName)).size());
}
 

private HRegion initHRegion(String callingMethod, Configuration conf) throws IOException {
  int i = 0;
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(TABLENAME);
  for (byte[] family : FAMILIES) {
    ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
      new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(family);
    // even column families are going to have compacted memstore
    if (i % 2 == 0) {
      familyDescriptor.setInMemoryCompaction(MemoryCompactionPolicy
          .valueOf(conf.get(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY)));
    } else {
      familyDescriptor.setInMemoryCompaction(MemoryCompactionPolicy.NONE);
    }
    tableDescriptor.setColumnFamily(familyDescriptor);
    i++;
  }

  RegionInfo info = RegionInfoBuilder.newBuilder(TABLENAME).build();
  Path path = new Path(DIR, callingMethod);
  HRegion region = HBaseTestingUtility.createRegionAndWAL(info, path, conf,
    tableDescriptor, false);
  region.regionServicesForStores = Mockito.spy(region.regionServicesForStores);
  ThreadPoolExecutor pool = (ThreadPoolExecutor) Executors.newFixedThreadPool(1);
  Mockito.when(region.regionServicesForStores.getInMemoryCompactionPool()).thenReturn(pool);
  region.initialize(null);
  return region;
}
 

@Before
public void setUp() throws Exception {
  TableName tableName = TableName.valueOf(getClass().getSimpleName());
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);

  tableDescriptor.setColumnFamily(
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(cfName)
      .setDataBlockEncoding(DataBlockEncoding.FAST_DIFF));
  RegionInfo info = RegionInfoBuilder.newBuilder(tableName).build();
  Path path = testUtil.getDataTestDir(getClass().getSimpleName());
  region = HBaseTestingUtility.createRegionAndWAL(info, path, testUtil.getConfiguration(),
    tableDescriptor);
}
 

private HRegion initHRegion(String callingMethod, Configuration conf) throws IOException {
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(TABLENAME);
  for (byte[] family : FAMILIES) {
    tableDescriptor.setColumnFamily(
      new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(family));
  }
  RegionInfo info = RegionInfoBuilder.newBuilder(TABLENAME).build();
  Path path = new Path(DIR, callingMethod);
  return HBaseTestingUtility.createRegionAndWAL(info, path, conf, tableDescriptor);
}
 

@BeforeClass
public static void setUp() throws IOException {
  Path testDir = UTIL.getDataTestDir();
  RegionInfo hri = RegionInfoBuilder.newBuilder(TESTTABLEDESC.getTableName()).build();
  REGION =
    HBaseTestingUtility.createRegionAndWAL(hri, testDir, UTIL.getConfiguration(), TESTTABLEDESC);
}
 

private void _testBlocksScanned(TableDescriptor td) throws Exception {
  BlockCache blockCache = BlockCacheFactory.createBlockCache(conf);
  RegionInfo regionInfo =
      RegionInfoBuilder.newBuilder(td.getTableName()).setStartKey(START_KEY).setEndKey(END_KEY)
          .build();
  HRegion r = HBaseTestingUtility.createRegionAndWAL(regionInfo, testDir, conf, td, blockCache);
  addContent(r, FAMILY, COL);
  r.flush(true);

  CacheStats stats = blockCache.getStats();
  long before = stats.getHitCount() + stats.getMissCount();
  // Do simple test of getting one row only first.
  Scan scan = new Scan().withStartRow(Bytes.toBytes("aaa")).withStopRow(Bytes.toBytes("aaz"))
      .setReadType(Scan.ReadType.PREAD);
  scan.addColumn(FAMILY, COL);
  scan.readVersions(1);

  InternalScanner s = r.getScanner(scan);
  List<Cell> results = new ArrayList<>();
  while (s.next(results));
  s.close();

  int expectResultSize = 'z' - 'a';
  assertEquals(expectResultSize, results.size());

  int kvPerBlock = (int) Math.ceil(BLOCK_SIZE /
      (double) KeyValueUtil.ensureKeyValue(results.get(0)).getLength());
  assertEquals(2, kvPerBlock);

  long expectDataBlockRead = (long) Math.ceil(expectResultSize / (double) kvPerBlock);
  long expectIndexBlockRead = expectDataBlockRead;

  assertEquals(expectIndexBlockRead + expectDataBlockRead,
      stats.getHitCount() + stats.getMissCount() - before);
}
 

@Test
public void testScannerSelection() throws IOException {
  Configuration conf = TEST_UTIL.getConfiguration();
  conf.setBoolean("hbase.store.delete.expired.storefile", false);
  LruBlockCache cache = (LruBlockCache) BlockCacheFactory.createBlockCache(conf);

  TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLE).setColumnFamily(
      ColumnFamilyDescriptorBuilder.newBuilder(FAMILY_BYTES).setMaxVersions(Integer.MAX_VALUE)
          .setTimeToLive(TTL_SECONDS).build()).build();
  RegionInfo info = RegionInfoBuilder.newBuilder(TABLE).build();
  HRegion region = HBaseTestingUtility
      .createRegionAndWAL(info, TEST_UTIL.getDataTestDir(info.getEncodedName()), conf, td, cache);

  long ts = EnvironmentEdgeManager.currentTime();
  long version = 0; //make sure each new set of Put's have a new ts
  for (int iFile = 0; iFile < totalNumFiles; ++iFile) {
    if (iFile == NUM_EXPIRED_FILES) {
      Threads.sleepWithoutInterrupt(TTL_MS);
      version += TTL_MS;
    }

    for (int iRow = 0; iRow < NUM_ROWS; ++iRow) {
      Put put = new Put(Bytes.toBytes("row" + iRow));
      for (int iCol = 0; iCol < NUM_COLS_PER_ROW; ++iCol) {
        put.addColumn(FAMILY_BYTES, Bytes.toBytes("col" + iCol), ts + version,
                Bytes.toBytes("value" + iFile + "_" + iRow + "_" + iCol));
      }
      region.put(put);
    }
    region.flush(true);
    version++;
  }

  Scan scan = new Scan().readVersions(Integer.MAX_VALUE);
  cache.clearCache();
  InternalScanner scanner = region.getScanner(scan);
  List<Cell> results = new ArrayList<>();
  final int expectedKVsPerRow = numFreshFiles * NUM_COLS_PER_ROW;
  int numReturnedRows = 0;
  LOG.info("Scanning the entire table");
  while (scanner.next(results) || results.size() > 0) {
    assertEquals(expectedKVsPerRow, results.size());
    ++numReturnedRows;
    results.clear();
  }
  assertEquals(NUM_ROWS, numReturnedRows);
  Set<String> accessedFiles = cache.getCachedFileNamesForTest();
  LOG.debug("Files accessed during scan: " + accessedFiles);

  // Exercise both compaction codepaths.
  if (explicitCompaction) {
    HStore store = region.getStore(FAMILY_BYTES);
    store.compactRecentForTestingAssumingDefaultPolicy(totalNumFiles);
  } else {
    region.compact(false);
  }

  HBaseTestingUtility.closeRegionAndWAL(region);
}
 

@Test
public void testMultipleColumnPrefixFilter() throws IOException {
  String family = "Family";
  TableDescriptorBuilder tableDescriptorBuilder =
    TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()));
  ColumnFamilyDescriptor columnFamilyDescriptor =
    ColumnFamilyDescriptorBuilder
      .newBuilder(Bytes.toBytes(family))
      .setMaxVersions(3)
      .build();
  tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
  TableDescriptor tableDescriptor = tableDescriptorBuilder.build();
  // HRegionInfo info = new HRegionInfo(htd, null, null, false);
  RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
  HRegion region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.
      getDataTestDir(), TEST_UTIL.getConfiguration(), tableDescriptor);

  List<String> rows = generateRandomWords(100, "row");
  List<String> columns = generateRandomWords(10000, "column");
  long maxTimestamp = 2;

  List<Cell> kvList = new ArrayList<>();

  Map<String, List<Cell>> prefixMap = new HashMap<>();

  prefixMap.put("p", new ArrayList<>());
  prefixMap.put("q", new ArrayList<>());
  prefixMap.put("s", new ArrayList<>());

  String valueString = "ValueString";

  for (String row: rows) {
    Put p = new Put(Bytes.toBytes(row));
    p.setDurability(Durability.SKIP_WAL);
    for (String column: columns) {
      for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
        KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp,
            valueString);
        p.add(kv);
        kvList.add(kv);
        for (String s: prefixMap.keySet()) {
          if (column.startsWith(s)) {
            prefixMap.get(s).add(kv);
          }
        }
      }
    }
    region.put(p);
  }

  MultipleColumnPrefixFilter filter;
  Scan scan = new Scan();
  scan.readAllVersions();
  byte [][] filter_prefix = new byte [2][];
  filter_prefix[0] = new byte [] {'p'};
  filter_prefix[1] = new byte [] {'q'};

  filter = new MultipleColumnPrefixFilter(filter_prefix);
  scan.setFilter(filter);
  List<Cell> results = new ArrayList<>();
  InternalScanner scanner = region.getScanner(scan);
  while (scanner.next(results))
    ;
  assertEquals(prefixMap.get("p").size() + prefixMap.get("q").size(), results.size());

  HBaseTestingUtility.closeRegionAndWAL(region);
}
 

private void testReplayWorksWithMemoryCompactionPolicy(MemoryCompactionPolicy policy) throws
  IOException {
  Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
  // Set it so we flush every 1M or so.  Thats a lot.
  conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024*1024);
  conf.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, String.valueOf(policy).toLowerCase());
  // The file of recovered edits has a column family of 'meta'.
  final String columnFamily = "meta";
  byte[][] columnFamilyAsByteArray = new byte[][] { Bytes.toBytes(columnFamily) };
  TableDescriptor tableDescriptor = TableDescriptorBuilder
    .newBuilder(TableName.valueOf(testName.getMethodName())).setColumnFamily(
      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(columnFamily)).build())
    .build();
  RegionInfo hri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
  final String encodedRegionName = hri.getEncodedName();
  Path hbaseRootDir = TEST_UTIL.getDataTestDir();
  FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
  Path tableDir = CommonFSUtils.getTableDir(hbaseRootDir, tableDescriptor.getTableName());
  HRegionFileSystem hrfs =
      new HRegionFileSystem(TEST_UTIL.getConfiguration(), fs, tableDir, hri);
  if (fs.exists(hrfs.getRegionDir())) {
    LOG.info("Region directory already exists. Deleting.");
    fs.delete(hrfs.getRegionDir(), true);
  }
  HRegion region = HBaseTestingUtility
      .createRegionAndWAL(hri, hbaseRootDir, conf, tableDescriptor, blockCache);
  assertEquals(encodedRegionName, region.getRegionInfo().getEncodedName());
  List<String> storeFiles = region.getStoreFileList(columnFamilyAsByteArray);
  // There should be no store files.
  assertTrue(storeFiles.isEmpty());
  region.close();
  Path regionDir = FSUtils.getRegionDirFromRootDir(hbaseRootDir, hri);
  Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regionDir);
  // This is a little fragile getting this path to a file of 10M of edits.
  Path recoveredEditsFile = new Path(
    System.getProperty("test.build.classes", "target/test-classes"),
      "0000000000000016310");
  // Copy this file under the region's recovered.edits dir so it is replayed on reopen.
  Path destination = new Path(recoveredEditsDir, recoveredEditsFile.getName());
  fs.copyToLocalFile(recoveredEditsFile, destination);
  assertTrue(fs.exists(destination));
  // Now the file 0000000000000016310 is under recovered.edits, reopen the region to replay.
  region = HRegion.openHRegion(region, null);
  assertEquals(encodedRegionName, region.getRegionInfo().getEncodedName());
  storeFiles = region.getStoreFileList(columnFamilyAsByteArray);
  // Our 0000000000000016310 is 10MB. Most of the edits are for one region. Lets assume that if
  // we flush at 1MB, that there are at least 3 flushed files that are there because of the
  // replay of edits.
  if(policy == MemoryCompactionPolicy.EAGER || policy == MemoryCompactionPolicy.ADAPTIVE) {
    assertTrue("Files count=" + storeFiles.size(), storeFiles.size() >= 1);
  } else {
    assertTrue("Files count=" + storeFiles.size(), storeFiles.size() > 10);
  }
  // Now verify all edits made it into the region.
  int count = verifyAllEditsMadeItIn(fs, conf, recoveredEditsFile, region);
  LOG.info("Checked " + count + " edits made it in");
}
 

@Test
public void testMultipleColumnPrefixFilterWithManyFamilies() throws IOException {
  String family1 = "Family1";
  String family2 = "Family2";
  TableDescriptorBuilder tableDescriptorBuilder =
    TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()));
  ColumnFamilyDescriptor columnFamilyDescriptor =
    ColumnFamilyDescriptorBuilder
      .newBuilder(Bytes.toBytes(family1))
      .setMaxVersions(3)
      .build();
  tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
  columnFamilyDescriptor = ColumnFamilyDescriptorBuilder
    .newBuilder(Bytes.toBytes(family2))
    .setMaxVersions(3)
    .build();
  tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
  TableDescriptor tableDescriptor = tableDescriptorBuilder.build();
  RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
  HRegion region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.
    getDataTestDir(), TEST_UTIL.getConfiguration(), tableDescriptor);

  List<String> rows = generateRandomWords(100, "row");
  List<String> columns = generateRandomWords(10000, "column");
  long maxTimestamp = 3;

  List<Cell> kvList = new ArrayList<>();

  Map<String, List<Cell>> prefixMap = new HashMap<>();

  prefixMap.put("p", new ArrayList<>());
  prefixMap.put("q", new ArrayList<>());
  prefixMap.put("s", new ArrayList<>());

  String valueString = "ValueString";

  for (String row: rows) {
    Put p = new Put(Bytes.toBytes(row));
    p.setDurability(Durability.SKIP_WAL);
    for (String column: columns) {
      for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
        double rand = Math.random();
        Cell kv;
        if (rand < 0.5) {
          kv = KeyValueTestUtil.create(row, family1, column, timestamp, valueString);
        } else {
          kv = KeyValueTestUtil.create(row, family2, column, timestamp, valueString);
        }
        p.add(kv);
        kvList.add(kv);
        for (String s: prefixMap.keySet()) {
          if (column.startsWith(s)) {
            prefixMap.get(s).add(kv);
          }
        }
      }
    }
    region.put(p);
  }

  MultipleColumnPrefixFilter filter;
  Scan scan = new Scan();
  scan.readAllVersions();
  byte [][] filter_prefix = new byte [2][];
  filter_prefix[0] = new byte [] {'p'};
  filter_prefix[1] = new byte [] {'q'};

  filter = new MultipleColumnPrefixFilter(filter_prefix);
  scan.setFilter(filter);
  List<Cell> results = new ArrayList<>();
  InternalScanner scanner = region.getScanner(scan);
  while (scanner.next(results))
    ;
  assertEquals(prefixMap.get("p").size() + prefixMap.get("q").size(), results.size());

  HBaseTestingUtility.closeRegionAndWAL(region);
}
 

@Test
public void testFilterListWithPrefixFilter() throws IOException {
  byte[] family = Bytes.toBytes("f1");
  byte[] qualifier = Bytes.toBytes("q1");
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(
      TableName.valueOf(name.getMethodName()));

  tableDescriptor.setColumnFamily(
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(family));
  RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
  HRegion testRegion = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(),
    TEST_UTIL.getConfiguration(), tableDescriptor);

  for(int i=0; i<5; i++) {
    Put p = new Put(Bytes.toBytes((char)('a'+i) + "row"));
    p.setDurability(Durability.SKIP_WAL);
    p.addColumn(family, qualifier, Bytes.toBytes(String.valueOf(111 + i)));
    testRegion.put(p);
  }
  testRegion.flush(true);

  // rows starting with "b"
  PrefixFilter pf = new PrefixFilter(new byte[] {'b'}) ;
  // rows with value of column 'q1' set to '113'
  SingleColumnValueFilter scvf = new SingleColumnValueFilter(
      family, qualifier, CompareOperator.EQUAL, Bytes.toBytes("113"));
  // combine these two with OR in a FilterList
  FilterList filterList = new FilterList(Operator.MUST_PASS_ONE, pf, scvf);

  Scan s1 = new Scan();
  s1.setFilter(filterList);
  InternalScanner scanner = testRegion.getScanner(s1);
  List<Cell> results = new ArrayList<>();
  int resultCount = 0;
  while (scanner.next(results)) {
    resultCount++;
    byte[] row =  CellUtil.cloneRow(results.get(0));
    LOG.debug("Found row: " + Bytes.toStringBinary(row));
    assertTrue(Bytes.equals(row, Bytes.toBytes("brow"))
        || Bytes.equals(row, Bytes.toBytes("crow")));
    results.clear();
  }
  assertEquals(2, resultCount);
  scanner.close();

  WAL wal = ((HRegion)testRegion).getWAL();
  ((HRegion)testRegion).close();
  wal.close();
}
 

/**
 * Test from client side for scan with maxResultPerCF set
 */
@Test
public void testScanLimitAndOffset() throws Exception {
  //byte [] TABLE = HTestConst.DEFAULT_TABLE_BYTES;
  byte [][] ROWS = HTestConst.makeNAscii(HTestConst.DEFAULT_ROW_BYTES, 2);
  byte [][] FAMILIES = HTestConst.makeNAscii(HTestConst.DEFAULT_CF_BYTES, 3);
  byte [][] QUALIFIERS = HTestConst.makeNAscii(HTestConst.DEFAULT_QUALIFIER_BYTES, 10);

  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(
      TableName.valueOf(HTestConst.DEFAULT_TABLE_BYTES));

  RegionInfo info = RegionInfoBuilder.newBuilder(HTestConst.DEFAULT_TABLE).build();
  for (byte[] family : FAMILIES) {
    ColumnFamilyDescriptor familyDescriptor =
      new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(family);
    tableDescriptor.setColumnFamily(familyDescriptor);
  }
  HRegion region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(),
      TEST_UTIL.getConfiguration(), tableDescriptor);
  try {
    Put put;
    Scan scan;
    Result result;
    boolean toLog = true;

    List<Cell> kvListExp = new ArrayList<>();

    int storeOffset = 1;
    int storeLimit = 3;
    for (int r = 0; r < ROWS.length; r++) {
      put = new Put(ROWS[r]);
      for (int c = 0; c < FAMILIES.length; c++) {
        for (int q = 0; q < QUALIFIERS.length; q++) {
          KeyValue kv = new KeyValue(ROWS[r], FAMILIES[c], QUALIFIERS[q], 1,
              HTestConst.DEFAULT_VALUE_BYTES);
          put.add(kv);
          if (storeOffset <= q && q < storeOffset + storeLimit) {
            kvListExp.add(kv);
          }
        }
      }
      region.put(put);
    }

    scan = new Scan();
    scan.setRowOffsetPerColumnFamily(storeOffset);
    scan.setMaxResultsPerColumnFamily(storeLimit);
    RegionScanner scanner = region.getScanner(scan);
    List<Cell> kvListScan = new ArrayList<>();
    List<Cell> results = new ArrayList<>();
    while (scanner.next(results) || !results.isEmpty()) {
      kvListScan.addAll(results);
      results.clear();
    }
    result = Result.create(kvListScan);
    TestScannersFromClientSide.verifyResult(result, kvListExp, toLog,
        "Testing scan with storeOffset and storeLimit");
  } finally {
    HBaseTestingUtility.closeRegionAndWAL(region);
  }
}