org.apache.hadoop.hbase.wal.WAL Java Examples

private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1});
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
                     new LocalRegionServerServices(conf, ServerName.valueOf(
                       InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 

private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1});
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
      new LocalRegionServerServices(conf, ServerName.valueOf(
          InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 

private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1});
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
                     new LocalRegionServerServices(conf, ServerName.valueOf(
                       InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 

private void checkMinLogRolls(final WAL log, final int minRolls)
    throws Exception {
  final List<Path> paths = new ArrayList<>();
  log.registerWALActionsListener(new WALActionsListener() {
    @Override
    public void postLogRoll(Path oldFile, Path newFile) {
      LOG.debug("postLogRoll: oldFile="+oldFile+" newFile="+newFile);
      paths.add(newFile);
    }
  });

  // Sleep until we should get at least min-LogRoll events
  long wtime = System.currentTimeMillis();
  Thread.sleep((minRolls + 1) * LOG_ROLL_PERIOD);
  // Do some extra sleep in case the machine is slow,
  // and the log-roll is not triggered exactly on LOG_ROLL_PERIOD.
  final int NUM_RETRIES = 1 + 8 * (minRolls - paths.size());
  for (int retry = 0; paths.size() < minRolls && retry < NUM_RETRIES; ++retry) {
    Thread.sleep(LOG_ROLL_PERIOD / 4);
  }
  wtime = System.currentTimeMillis() - wtime;
  LOG.info(String.format("got %d rolls after %dms (%dms each) - expected at least %d rolls",
                         paths.size(), wtime, wtime / paths.size(), minRolls));
  assertFalse(paths.size() < minRolls);
}
 

/**
 * Test blocksize change from HBASE-20520 takes on both asycnfs and old wal provider.
 * Hard to verify more than this given the blocksize is passed down to HDFS on create -- not
 * kept local to the streams themselves.
 */
@Test
public void testBlocksizeDefaultsToTwiceHDFSBlockSize() throws IOException {
  TableName tableName = TableName.valueOf("test");
  final WALFactory walFactory = new WALFactory(TEST_UTIL.getConfiguration(), this.walProvider);
  Configuration conf = TEST_UTIL.getConfiguration();
  WALProvider provider = walFactory.getWALProvider();
  // Get a WAL instance from the provider. Check its blocksize.
  WAL wal = provider.getWAL(null);
  if (wal instanceof AbstractFSWAL) {
    long expectedDefaultBlockSize =
        WALUtil.getWALBlockSize(conf, FileSystem.get(conf), TEST_UTIL.getDataTestDir());
    long blocksize = ((AbstractFSWAL)wal).blocksize;
    assertEquals(expectedDefaultBlockSize, blocksize);
    LOG.info("Found blocksize of {} on {}", blocksize, wal);
  } else {
    fail("Unknown provider " + provider);
  }
}
 

private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1}, null);
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
      new LocalRegionServerServices(conf, ServerName.valueOf(
          InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 

private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1}, null);
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
      new LocalRegionServerServices(conf, ServerName.valueOf(
          InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 

private static HRegion bootstrap(Configuration conf, TableDescriptor td, FileSystem fs,
  Path rootDir, FileSystem walFs, Path walRootDir, WALFactory walFactory,
  MasterRegionWALRoller walRoller, String serverName) throws IOException {
  TableName tn = td.getTableName();
  RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tn).setRegionId(REGION_ID).build();
  Path tmpTableDir = CommonFSUtils.getTableDir(rootDir,
    TableName.valueOf(tn.getNamespaceAsString(), tn.getQualifierAsString() + "-tmp"));
  if (fs.exists(tmpTableDir) && !fs.delete(tmpTableDir, true)) {
    throw new IOException("Can not delete partial created proc region " + tmpTableDir);
  }
  HRegion.createHRegion(conf, regionInfo, fs, tmpTableDir, td).close();
  Path tableDir = CommonFSUtils.getTableDir(rootDir, tn);
  if (!fs.rename(tmpTableDir, tableDir)) {
    throw new IOException("Can not rename " + tmpTableDir + " to " + tableDir);
  }
  WAL wal = createWAL(walFactory, walRoller, serverName, walFs, walRootDir, regionInfo);
  return HRegion.openHRegionFromTableDir(conf, fs, tableDir, regionInfo, td, wal, null, null);
}
 

@Test
public void testReplicateLogEntries_SingleWALEditForMultipleRows() throws IOException {

    byte[] rowKeyA = Bytes.toBytes("A");
    byte[] rowKeyB = Bytes.toBytes("B");
    byte[] data = Bytes.toBytes("data");

    Cell kvA = new KeyValue(rowKeyA, DATA_COLFAM, PAYLOAD_QUALIFIER, data);
    Cell kvB = new KeyValue(rowKeyB, DATA_COLFAM, PAYLOAD_QUALIFIER, data);

    WAL.Entry entry = createHlogEntry(TABLE_NAME, kvA, kvB);

    replicateWALEntry(new WAL.Entry[]{entry});

    SepEvent expectedEventA = SepEvent.create(TABLE_NAME, rowKeyA, Lists.newArrayList(kvA),
            Bytes.toBytes("data"));
    SepEvent expectedEventB = SepEvent.create(TABLE_NAME, rowKeyB, Lists.newArrayList(kvB),
            Bytes.toBytes("data"));

    verify(eventListener).processEvents(Lists.newArrayList(expectedEventA, expectedEventB));
}
 

/**
 * A 'full' WAL transaction involves starting an mvcc transaction followed by an append, an
 * optional sync, and then a call to complete the mvcc transaction. This method does it all. Good
 * for case of adding a single edit or marker to the WAL.
 * <p/>
 * This write is for internal use only. Not for external client consumption.
 * @return WALKeyImpl that was added to the WAL.
 */
private static WALKeyImpl doFullMarkerAppendTransaction(final WAL wal,
  final NavigableMap<byte[], Integer> replicationScope, final RegionInfo hri, final WALEdit edit,
  final MultiVersionConcurrencyControl mvcc,
  final Map<String, byte[]> extendedAttributes, final boolean sync) throws IOException {
  // TODO: Pass in current time to use?
  WALKeyImpl walKey = new WALKeyImpl(hri.getEncodedNameAsBytes(), hri.getTable(),
    System.currentTimeMillis(), mvcc, replicationScope, extendedAttributes);
  long trx = MultiVersionConcurrencyControl.NONE;
  try {
    trx = wal.appendMarker(hri, walKey, edit);
    if (sync) {
      wal.sync(trx);
    }
    // Call complete only here because these are markers only. They are not for clients to read.
    mvcc.complete(walKey.getWriteEntry());
  } catch (IOException ioe) {
    if (walKey.getWriteEntry() != null) {
      mvcc.complete(walKey.getWriteEntry());
    }
    throw ioe;
  }
  return walKey;
}
 

/**
 * Waits until there is only one log(the current writing one) in the replication queue
 * @param numRs number of regionservers
 */
private void waitForLogAdvance(int numRs) throws Exception {
  Waiter.waitFor(CONF1, 10000, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      for (int i = 0; i < numRs; i++) {
        HRegionServer hrs = UTIL1.getHBaseCluster().getRegionServer(i);
        RegionInfo regionInfo =
            UTIL1.getHBaseCluster().getRegions(htable1.getName()).get(0).getRegionInfo();
        WAL wal = hrs.getWAL(regionInfo);
        Path currentFile = ((AbstractFSWAL<?>) wal).getCurrentFileName();
        Replication replicationService = (Replication) UTIL1.getHBaseCluster()
            .getRegionServer(i).getReplicationSourceService();
        for (ReplicationSourceInterface rsi : replicationService.getReplicationManager()
            .getSources()) {
          ReplicationSource source = (ReplicationSource) rsi;
          if (!currentFile.equals(source.getCurrentPath())) {
            return false;
          }
        }
      }
      return true;
    }
  });
}
 

/**
 * Test the case where the secondary region replica is not in reads enabled state because it is
 * waiting for a flush or region open marker from primary region. Replaying CANNOT_FLUSH
 * flush marker entry should restore the reads enabled status in the region and allow the reads
 * to continue.
 */
@Test
public void testReplayingFlushRequestRestoresReadsEnabledState() throws IOException {
  disableReads(secondaryRegion);

  // Test case 1: Test that replaying CANNOT_FLUSH request marker assuming this came from
  // triggered flush restores readsEnabled
  primaryRegion.flushcache(true, true, FlushLifeCycleTracker.DUMMY);
  reader = createWALReaderForPrimary();
  while (true) {
    WAL.Entry entry = reader.next();
    if (entry == null) {
      break;
    }
    FlushDescriptor flush = WALEdit.getFlushDescriptor(entry.getEdit().getCells().get(0));
    if (flush != null) {
      secondaryRegion.replayWALFlushMarker(flush, entry.getKey().getSequenceId());
    }
  }

  // now reads should be enabled
  secondaryRegion.get(new Get(Bytes.toBytes(0)));
}
 

protected final void checkOrder(int expectedEntries) throws IOException {
  try (WAL.Reader reader =
    WALFactory.createReader(UTIL.getTestFileSystem(), logPath, UTIL.getConfiguration())) {
    long seqId = -1L;
    int count = 0;
    for (Entry entry;;) {
      entry = reader.next();
      if (entry == null) {
        break;
      }
      assertTrue(
        "Sequence id go backwards from " + seqId + " to " + entry.getKey().getSequenceId(),
        entry.getKey().getSequenceId() >= seqId);
      seqId = entry.getKey().getSequenceId();
      count++;
    }
    assertEquals(expectedEntries, count);
  }
}
 

private void addWALEdits(final TableName tableName, final RegionInfo hri, final byte[] rowName,
    final byte[] family, final int count, EnvironmentEdge ee, final WAL wal,
    final NavigableMap<byte[], Integer> scopes, final MultiVersionConcurrencyControl mvcc)
    throws IOException {
  String familyStr = Bytes.toString(family);
  long txid = -1;
  for (int j = 0; j < count; j++) {
    byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
    byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
    WALEdit edit = new WALEdit();
    edit.add(new KeyValue(rowName, family, qualifierBytes, ee.currentTime(), columnBytes));
    // uses WALKeyImpl instead of HLogKey on purpose. will only work for tests where we don't care
    // about legacy coprocessors
    txid = wal.appendData(hri,
      new WALKeyImpl(hri.getEncodedNameAsBytes(), tableName, ee.currentTime(), mvcc), edit);
  }
  if (-1 != txid) {
    wal.sync(txid);
  }
}
 

private long getMaxSeqId(HRegionServer rs, RegionInfo region) throws IOException {
  Path walFile = ((AbstractFSWAL<?>) rs.getWAL(null)).getCurrentFileName();
  long maxSeqId = -1L;
  try (WAL.Reader reader =
    WALFactory.createReader(UTIL.getTestFileSystem(), walFile, UTIL.getConfiguration())) {
    for (;;) {
      WAL.Entry entry = reader.next();
      if (entry == null) {
        break;
      }
      if (Bytes.equals(region.getEncodedNameAsBytes(), entry.getKey().getEncodedRegionName())) {
        maxSeqId = Math.max(maxSeqId, entry.getKey().getSequenceId());
      }
    }
  }
  return maxSeqId;
}
 

HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
    byte[]... families) throws IOException {
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(TableName.valueOf(tableName));
  for (byte[] family : families) {
    tableDescriptor.setColumnFamily(
      new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(family));
  }
  ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null);
  RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
  Path path = new Path(DIR + callingMethod);
  WAL wal = HBaseTestingUtility.createWal(conf, path, info);
  HRegion r = HRegion.createHRegion(info, path, conf, tableDescriptor, wal);
  // this following piece is a hack. currently a coprocessorHost
  // is secretly loaded at OpenRegionHandler. we don't really
  // start a region server here, so just manually create cphost
  // and set it to region.
  RegionCoprocessorHost host = new RegionCoprocessorHost(r, null, conf);
  r.setCoprocessorHost(host);
  return r;
}
 

/**
 * Add a bunch of dummy data and roll the logs every two insert. We
 * should end up with 10 rolled files (plus the roll called in
 * the constructor). Also test adding a listener while it's running.
 */
@Test
public void testActionListener() throws Exception {
  DummyWALActionsListener observer = new DummyWALActionsListener();
  final WALFactory wals = new WALFactory(conf, "testActionListener");
  wals.getWALProvider().addWALActionsListener(observer);
  DummyWALActionsListener laterobserver = new DummyWALActionsListener();
  RegionInfo hri = RegionInfoBuilder.newBuilder(TableName.valueOf(SOME_BYTES))
      .setStartKey(SOME_BYTES).setEndKey(SOME_BYTES).build();
  final WAL wal = wals.getWAL(hri);
  MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
  for (int i = 0; i < 20; i++) {
    byte[] b = Bytes.toBytes(i + "");
    KeyValue kv = new KeyValue(b, b, b);
    WALEdit edit = new WALEdit();
    edit.add(kv);
    NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    scopes.put(b, 0);
    long txid = wal.appendData(hri,
      new WALKeyImpl(hri.getEncodedNameAsBytes(), TableName.valueOf(b), 0, mvcc, scopes), edit);
    wal.sync(txid);
    if (i == 10) {
      wal.registerWALActionsListener(laterobserver);
    }
    if (i % 2 == 0) {
      wal.rollWriter();
    }
  }

  wal.close();

  assertEquals(11, observer.preLogRollCounter);
  assertEquals(11, observer.postLogRollCounter);
  assertEquals(5, laterobserver.preLogRollCounter);
  assertEquals(5, laterobserver.postLogRollCounter);
  assertEquals(1, observer.closedCount);
}
 

/**
 * Constructor
 * @param log the write ahead log
 * @param conf the configuration
 */
public WALCoprocessorHost(final WAL log, final Configuration conf) {
  // We don't want to require an Abortable passed down through (FS)HLog, so
  // this means that a failure to load of a WAL coprocessor won't abort the
  // server. This isn't ideal, and means that security components that
  // utilize a WALObserver will have to check the observer initialization
  // state manually. However, WALObservers will eventually go away so it
  // should be an acceptable state of affairs.
  super(null);
  this.wal = log;
  // load system default cp's from configuration.
  loadSystemCoprocessors(conf, WAL_COPROCESSOR_CONF_KEY);
}
 

private WAL createWAL(final Configuration c, WALFactory walFactory) throws IOException {
  WAL wal = walFactory.getWAL(null);

  // Set down maximum recovery so we dfsclient doesn't linger retrying something
  // long gone.
  HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
  return wal;
}
 

/**
 * @return A region on which you must call {@link HBaseTestingUtility#closeRegionAndWAL(HRegion)}
 *         when done.
 */
public static HRegion initHRegion(TableName tableName, byte[] startKey, byte[] stopKey, WAL wal)
  throws IOException {
  ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null);
  return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, false, Durability.USE_DEFAULT,
    wal, COLUMN_FAMILY_BYTES);
}
 

/**
 * Tests a case where we replay only a flush start marker, then the region is closed. This region
 * should not block indefinitely
 */
@Test
public void testOnlyReplayingFlushStartDoesNotHoldUpRegionClose() throws IOException {
  // load some data to primary and flush
  int start = 0;
  LOG.info("-- Writing some data to primary from " +  start + " to " + (start+100));
  putData(primaryRegion, Durability.SYNC_WAL, start, 100, cq, families);
  LOG.info("-- Flushing primary, creating 3 files for 3 stores");
  primaryRegion.flush(true);

  // now replay the edits and the flush marker
  reader = createWALReaderForPrimary();

  LOG.info("-- Replaying edits and flush events in secondary");
  while (true) {
    WAL.Entry entry = reader.next();
    if (entry == null) {
      break;
    }
    FlushDescriptor flushDesc
      = WALEdit.getFlushDescriptor(entry.getEdit().getCells().get(0));
    if (flushDesc != null) {
      if (flushDesc.getAction() == FlushAction.START_FLUSH) {
        LOG.info("-- Replaying flush start in secondary");
        secondaryRegion.replayWALFlushStartMarker(flushDesc);
      } else if (flushDesc.getAction() == FlushAction.COMMIT_FLUSH) {
        LOG.info("-- NOT Replaying flush commit in secondary");
      }
    } else {
      replayEdit(secondaryRegion, entry);
    }
  }

  assertTrue(rss.getRegionServerAccounting().getGlobalMemStoreDataSize() > 0);
  // now close the region which should not cause hold because of un-committed flush
  secondaryRegion.close();

  // verify that the memstore size is back to what it was
  assertEquals(0, rss.getRegionServerAccounting().getGlobalMemStoreDataSize());
}
 

@Test
public void testSystemCatalogWALEntryFilter() throws Exception {

  //now create WAL.Entry objects that refer to cells in those view rows in System.Catalog

  Get tenantGet = getGet(catalogTable, TENANT_BYTES, TENANT_VIEW_NAME);
  Get nonTenantGet = getGet(catalogTable, DEFAULT_TENANT_BYTES, NONTENANT_VIEW_NAME);

  WAL.Entry nonTenantEntry = getEntry(systemCatalogTableName, nonTenantGet);
  WAL.Entry tenantEntry = getEntry(systemCatalogTableName, tenantGet);

  //verify that the tenant view WAL.Entry passes the filter and the non-tenant view does not
  SystemCatalogWALEntryFilter filter = new SystemCatalogWALEntryFilter();
  Assert.assertNull(filter.filter(nonTenantEntry));
  WAL.Entry filteredTenantEntry = filter.filter(tenantEntry);
  Assert.assertNotNull("Tenant view was filtered when it shouldn't be!", filteredTenantEntry);
  Assert.assertEquals(tenantEntry.getEdit().size(),
      filter.filter(tenantEntry).getEdit().size());

  //now check that a WAL.Entry with cells from both a tenant and a non-tenant
  //catalog row only allow the tenant cells through
  WALEdit comboEdit = new WALEdit();
  comboEdit.getCells().addAll(nonTenantEntry.getEdit().getCells());
  comboEdit.getCells().addAll(tenantEntry.getEdit().getCells());
  WAL.Entry comboEntry = new WAL.Entry(walKey, comboEdit);

  Assert.assertEquals(tenantEntry.getEdit().size() + nonTenantEntry.getEdit().size()
      , comboEntry.getEdit().size());
  Assert.assertEquals(tenantEntry.getEdit().size(),
      filter.filter(comboEntry).getEdit().size());
}
 

protected void compactingSetUp() throws Exception {
  super.internalSetUp();
  Configuration conf = new Configuration();
  conf.setBoolean(MemStoreLAB.USEMSLAB_KEY, true);
  conf.setFloat(MemStoreLAB.CHUNK_POOL_MAXSIZE_KEY, 0.2f);
  conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 1000);
  HBaseTestingUtility hbaseUtility = new HBaseTestingUtility(conf);
  ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(FAMILY);
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(TableName.valueOf("foobar"));

  tableDescriptor.setColumnFamily(familyDescriptor);
  RegionInfo info = RegionInfoBuilder.newBuilder(TableName.valueOf("foobar")).build();
  WAL wal = HBaseTestingUtility.createWal(conf, hbaseUtility.getDataTestDir(), info);
  this.region = HRegion.createHRegion(info, hbaseUtility.getDataTestDir(), conf,
    tableDescriptor, wal, true);
  this.regionServicesForStores = Mockito.spy(region.getRegionServicesForStores());
  ThreadPoolExecutor pool = (ThreadPoolExecutor) Executors.newFixedThreadPool(1);
  Mockito.when(regionServicesForStores.getInMemoryCompactionPool()).thenReturn(pool);
  this.store = new HStore(region, familyDescriptor, conf, false);

  long globalMemStoreLimit = (long) (ManagementFactory.getMemoryMXBean().getHeapMemoryUsage()
      .getMax() * MemorySizeUtil.getGlobalMemStoreHeapPercent(conf, false));
  chunkCreator = ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false,
      globalMemStoreLimit, 0.4f, MemStoreLAB.POOL_INITIAL_SIZE_DEFAULT,
          null);
  assertTrue(chunkCreator != null);
}
 

/**
 * Create an unmanaged WAL. Be sure to close it when you're through.
 */
public static WAL createWal(final Configuration conf, final Path rootDir, final RegionInfo hri)
    throws IOException {
  // The WAL subsystem will use the default rootDir rather than the passed in rootDir
  // unless I pass along via the conf.
  Configuration confForWAL = new Configuration(conf);
  confForWAL.set(HConstants.HBASE_DIR, rootDir.toString());
  return new WALFactory(confForWAL, "hregion-" + RandomStringUtils.randomNumeric(8)).getWAL(hri);
}
 

@Override
public void getWALPositions(RpcController controller,
                               SpliceMessage.GetWALPositionsRequest request,
                               RpcCallback<SpliceMessage.GetWALPositionsResponse> done) {

    SpliceMessage.GetWALPositionsResponse.Builder responseBuilder =
            SpliceMessage.GetWALPositionsResponse.newBuilder();

    try {
        List<WAL> wals = regionServerServices.getWALs();
        for (WAL wal : wals) {
            AbstractFSWAL abstractFSWAL = (AbstractFSWAL) wal;
            Path walName = abstractFSWAL.getCurrentFileName();
            OptionalLong size = wal.getLogFileSizeIfBeingWritten(walName);
            responseBuilder.addResult(
                    SpliceMessage.GetWALPositionsResponse.Result
                            .newBuilder()
                            .setPosition(size.isPresent() ? size.getAsLong() : 0)
                            .setWALName(walName.getName())
                            .build()
            );
        }

        SpliceMessage.GetWALPositionsResponse response = responseBuilder.build();
        done.run(response);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 

@Test
public void testWALMonotonicallyIncreasingSeqId() throws Exception {
  List<Thread> putThreads = new ArrayList<>();
  for (int i = 0; i < 1; i++) {
    putThreads.add(new PutThread(region));
  }
  IncThread incThread = new IncThread(region);
  for (int i = 0; i < 1; i++) {
    putThreads.get(i).start();
  }
  incThread.start();
  incThread.join();

  Path logPath = ((AbstractFSWAL<?>) region.getWAL()).getCurrentFileName();
  region.getWAL().rollWriter();
  Thread.sleep(10);
  Path hbaseDir = new Path(walConf.get(HConstants.HBASE_DIR));
  Path oldWalsDir = new Path(hbaseDir, HConstants.HREGION_OLDLOGDIR_NAME);
  try (WAL.Reader reader = createReader(logPath, oldWalsDir)) {
    long currentMaxSeqid = 0;
    for (WAL.Entry e; (e = reader.next()) != null;) {
      if (!WALEdit.isMetaEditFamily(e.getEdit().getCells().get(0))) {
        long currentSeqid = e.getKey().getSequenceId();
        if (currentSeqid > currentMaxSeqid) {
          currentMaxSeqid = currentSeqid;
        } else {
          fail("Current max Seqid is " + currentMaxSeqid +
            ", but the next seqid in wal is smaller:" + currentSeqid);
        }
      }
    }
  }
}
 

/**
 * Sanity check that we can move logs around while we are reading
 * from them. Should this test fail, ReplicationSource would have a hard
 * time reading logs that are being archived.
 */
@Test
public void testLogMoving() throws Exception{
  Path logPath = new Path(logDir, "log");
  if (!FS.exists(logDir)) FS.mkdirs(logDir);
  if (!FS.exists(oldLogDir)) FS.mkdirs(oldLogDir);
  WALProvider.Writer writer = WALFactory.createWALWriter(FS, logPath,
      TEST_UTIL.getConfiguration());
  for(int i = 0; i < 3; i++) {
    byte[] b = Bytes.toBytes(Integer.toString(i));
    KeyValue kv = new KeyValue(b,b,b);
    WALEdit edit = new WALEdit();
    edit.add(kv);
    WALKeyImpl key = new WALKeyImpl(b, TableName.valueOf(b), 0, 0,
        HConstants.DEFAULT_CLUSTER_ID);
    writer.append(new WAL.Entry(key, edit));
    writer.sync(false);
  }
  writer.close();

  WAL.Reader reader = WALFactory.createReader(FS, logPath, TEST_UTIL.getConfiguration());
  WAL.Entry entry = reader.next();
  assertNotNull(entry);

  Path oldLogPath = new Path(oldLogDir, "log");
  FS.rename(logPath, oldLogPath);

  entry = reader.next();
  assertNotNull(entry);

  entry = reader.next();
  entry = reader.next();

  assertNull(entry);
  reader.close();
}
 

@After
public void teardown() throws IOException {
  if (region != null) {
    CacheConfig cacheConfig = region.getStores().get(0).getCacheConfig();
    region.close();
    WAL wal = region.getWAL();
    if (wal != null) {
      wal.close();
    }
    cacheConfig.getBlockCache().ifPresent(BlockCache::shutdown);
    region = null;
  }
}
 

private WAL.Entry createHlogEntry(byte[] tableName, long writeTime, Cell... keyValues) {
    WAL.Entry entry = mock(WAL.Entry.class, Mockito.RETURNS_DEEP_STUBS);
    when(entry.getEdit().getCells()).thenReturn(Lists.newArrayList(keyValues));
    when(entry.getKey().getTablename()).thenReturn(TableName.valueOf(tableName));
    when(entry.getKey().getWriteTime()).thenReturn(writeTime);
    when(entry.getKey().getEncodedRegionName()).thenReturn(encodedRegionName);
    when(entry.getKey().getClusterIds()).thenReturn(clusterUUIDs);
    return entry;
}
 

protected void addEdits(WAL log, RegionInfo hri, TableDescriptor htd, int times,
    MultiVersionConcurrencyControl mvcc, NavigableMap<byte[], Integer> scopes, String cf)
    throws IOException {
  final byte[] row = Bytes.toBytes(cf);
  for (int i = 0; i < times; i++) {
    long timestamp = System.currentTimeMillis();
    WALEdit cols = new WALEdit();
    cols.add(new KeyValue(row, row, row, timestamp, row));
    WALKeyImpl key = new WALKeyImpl(hri.getEncodedNameAsBytes(), htd.getTableName(),
        SequenceId.NO_SEQUENCE_ID, timestamp, WALKey.EMPTY_UUIDS, HConstants.NO_NONCE,
        HConstants.NO_NONCE, mvcc, scopes);
    log.appendData(hri, key, cols);
  }
  log.sync();
}