Java Code Examples for org.apache.hadoop.hbase.wal.WAL#Reader

protected final void checkOrder(int expectedEntries) throws IOException {
  try (WAL.Reader reader =
    WALFactory.createReader(UTIL.getTestFileSystem(), logPath, UTIL.getConfiguration())) {
    long seqId = -1L;
    int count = 0;
    for (Entry entry;;) {
      entry = reader.next();
      if (entry == null) {
        break;
      }
      assertTrue(
        "Sequence id go backwards from " + seqId + " to " + entry.getKey().getSequenceId(),
        entry.getKey().getSequenceId() >= seqId);
      seqId = entry.getKey().getSequenceId();
      count++;
    }
    assertEquals(expectedEntries, count);
  }
}
 

private long getMaxSeqId(HRegionServer rs, RegionInfo region) throws IOException {
  Path walFile = ((AbstractFSWAL<?>) rs.getWAL(null)).getCurrentFileName();
  long maxSeqId = -1L;
  try (WAL.Reader reader =
    WALFactory.createReader(UTIL.getTestFileSystem(), walFile, UTIL.getConfiguration())) {
    for (;;) {
      WAL.Entry entry = reader.next();
      if (entry == null) {
        break;
      }
      if (Bytes.equals(region.getEncodedNameAsBytes(), entry.getKey().getEncodedRegionName())) {
        maxSeqId = Math.max(maxSeqId, entry.getKey().getSequenceId());
      }
    }
  }
  return maxSeqId;
}
 

private static void transformFile(Path input, Path output)
    throws IOException {
  Configuration conf = HBaseConfiguration.create();

  FileSystem inFS = input.getFileSystem(conf);
  FileSystem outFS = output.getFileSystem(conf);

  WAL.Reader in = WALFactory.createReaderIgnoreCustomClass(inFS, input, conf);
  WALProvider.Writer out = null;

  try {
    if (!(in instanceof ReaderBase)) {
      System.err.println("Cannot proceed, invalid reader type: " + in.getClass().getName());
      return;
    }
    boolean compress = ((ReaderBase)in).hasCompression();
    conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, !compress);
    out = WALFactory.createWALWriter(outFS, output, conf);

    WAL.Entry e = null;
    while ((e = in.next()) != null) out.append(e);
  } finally {
    in.close();
    if (out != null) {
      out.close();
      out = null;
    }
  }
}
 

/**
 * Sanity check that we can move logs around while we are reading
 * from them. Should this test fail, ReplicationSource would have a hard
 * time reading logs that are being archived.
 */
@Test
public void testLogMoving() throws Exception{
  Path logPath = new Path(logDir, "log");
  if (!FS.exists(logDir)) FS.mkdirs(logDir);
  if (!FS.exists(oldLogDir)) FS.mkdirs(oldLogDir);
  WALProvider.Writer writer = WALFactory.createWALWriter(FS, logPath,
      TEST_UTIL.getConfiguration());
  for(int i = 0; i < 3; i++) {
    byte[] b = Bytes.toBytes(Integer.toString(i));
    KeyValue kv = new KeyValue(b,b,b);
    WALEdit edit = new WALEdit();
    edit.add(kv);
    WALKeyImpl key = new WALKeyImpl(b, TableName.valueOf(b), 0, 0,
        HConstants.DEFAULT_CLUSTER_ID);
    writer.append(new WAL.Entry(key, edit));
    writer.sync(false);
  }
  writer.close();

  WAL.Reader reader = WALFactory.createReader(FS, logPath, TEST_UTIL.getConfiguration());
  WAL.Entry entry = reader.next();
  assertNotNull(entry);

  Path oldLogPath = new Path(oldLogDir, "log");
  FS.rename(logPath, oldLogPath);

  entry = reader.next();
  assertNotNull(entry);

  entry = reader.next();
  entry = reader.next();

  assertNull(entry);
  reader.close();
}
 

private int countWAL(Path log, FileSystem fs, Configuration conf) throws IOException {
  int count = 0;
  try (WAL.Reader in = WALFactory.createReader(fs, log, conf)) {
    WAL.Entry e;
    while ((e = in.next()) != null) {
      if (!WALEdit.isMetaEditFamily(e.getEdit().getCells().get(0))) {
        count++;
      }
    }
  }
  return count;
}
 

private void verifyWALCount(WALFactory wals, WAL log, int expected) throws Exception {
  Path walPath = AbstractFSWALProvider.getCurrentFileName(log);
  WAL.Reader reader = wals.createReader(FS, walPath);
  int count = 0;
  WAL.Entry entry = new WAL.Entry();
  while (reader.next(entry) != null) {
    count++;
  }
  reader.close();
  assertEquals(expected, count);
}
 

private WAL.Reader createReader(Path logPath, Path oldWalsDir) throws IOException {
  try {
    return wals.createReader(fileSystem, logPath);
  } catch (IOException e) {
    return wals.createReader(fileSystem, new Path(oldWalsDir, logPath.getName()));
  }
}
 

@Test
public void testWALMonotonicallyIncreasingSeqId() throws Exception {
  List<Thread> putThreads = new ArrayList<>();
  for (int i = 0; i < 1; i++) {
    putThreads.add(new PutThread(region));
  }
  IncThread incThread = new IncThread(region);
  for (int i = 0; i < 1; i++) {
    putThreads.get(i).start();
  }
  incThread.start();
  incThread.join();

  Path logPath = ((AbstractFSWAL<?>) region.getWAL()).getCurrentFileName();
  region.getWAL().rollWriter();
  Thread.sleep(10);
  Path hbaseDir = new Path(walConf.get(HConstants.HBASE_DIR));
  Path oldWalsDir = new Path(hbaseDir, HConstants.HREGION_OLDLOGDIR_NAME);
  try (WAL.Reader reader = createReader(logPath, oldWalsDir)) {
    long currentMaxSeqid = 0;
    for (WAL.Entry e; (e = reader.next()) != null;) {
      if (!WALEdit.isMetaEditFamily(e.getEdit().getCells().get(0))) {
        long currentSeqid = e.getKey().getSequenceId();
        if (currentSeqid > currentMaxSeqid) {
          currentMaxSeqid = currentSeqid;
        } else {
          fail("Current max Seqid is " + currentMaxSeqid +
            ", but the next seqid in wal is smaller:" + currentSeqid);
        }
      }
    }
  }
}
 

@Override
protected int doWork() throws Exception {
  Path path = new Path(file);
  FileSystem fs = path.getFileSystem(conf);
  try (WAL.Reader reader = WALFactory.createReader(fs, path, conf)) {
    for (;;) {
      WAL.Entry entry = reader.next();
      if (entry == null) {
        return 0;
      }
      WALKey key = entry.getKey();
      WALEdit edit = entry.getEdit();
      long sequenceId = key.getSequenceId();
      long writeTime = key.getWriteTime();
      out.println(
        String.format(KEY_TMPL, sequenceId, FORMATTER.format(Instant.ofEpochMilli(writeTime))));
      for (Cell cell : edit.getCells()) {
        Map<String, Object> op = WALPrettyPrinter.toStringMap(cell);
        if (!Bytes.equals(PROC_FAMILY, 0, PROC_FAMILY.length, cell.getFamilyArray(),
          cell.getFamilyOffset(), cell.getFamilyLength())) {
          // We could have cells other than procedure edits, for example, a flush marker
          WALPrettyPrinter.printCell(out, op, false);
          continue;
        }
        long procId = Bytes.toLong(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
        out.println("pid=" + procId + ", type=" + op.get("type") + ", column=" +
          op.get("family") + ":" + op.get("qualifier"));
        if (cell.getType() == Cell.Type.Put) {
          if (cell.getValueLength() > 0) {
            // should be a normal put
            Procedure<?> proc =
              ProcedureUtil.convertToProcedure(ProcedureProtos.Procedure.parser()
                .parseFrom(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
            out.println("\t" + proc.toStringDetails());
          } else {
            // should be a 'delete' put
            out.println("\tmark deleted");
          }
        }
        out.println("cell total size sum: " + cell.heapSize());
      }
      out.println("edit heap size: " + edit.heapSize());
      out.println("position: " + reader.getPosition());
    }
  }
}
 

@Test
public void testRegionSplit() throws Exception {
  TableName tableName = createTable();
  try (Table table = UTIL.getConnection().getTable(tableName)) {
    for (int i = 0; i < 100; i++) {
      table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i)));
    }
  }
  UTIL.flush(tableName);
  RegionInfo region = UTIL.getAdmin().getRegions(tableName).get(0);
  UTIL.getAdmin().splitRegionAsync(region.getEncodedNameAsBytes(), Bytes.toBytes(50)).get(30,
    TimeUnit.SECONDS);
  UTIL.waitUntilNoRegionsInTransition(30000);
  List<RegionInfo> regions = UTIL.getAdmin().getRegions(tableName);
  assertEquals(2, regions.size());
  try (Table table = UTIL.getConnection().getTable(tableName)) {
    for (int i = 0; i < 100; i++) {
      table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i)));
    }
  }
  enablePeerAndWaitUntilReplicationDone(200);
  Map<String, Long> regionsToSeqId = new HashMap<>();
  regionsToSeqId.put(region.getEncodedName(), -1L);
  regions.stream().map(RegionInfo::getEncodedName).forEach(n -> regionsToSeqId.put(n, -1L));
  try (WAL.Reader reader =
    WALFactory.createReader(UTIL.getTestFileSystem(), logPath, UTIL.getConfiguration())) {
    int count = 0;
    for (Entry entry;;) {
      entry = reader.next();
      if (entry == null) {
        break;
      }
      String encodedName = Bytes.toString(entry.getKey().getEncodedRegionName());
      Long seqId = regionsToSeqId.get(encodedName);
      assertNotNull(
        "Unexcepted entry " + entry + ", expected regions " + region + ", or " + regions, seqId);
      assertTrue("Sequence id go backwards from " + seqId + " to " +
        entry.getKey().getSequenceId() + " for " + encodedName,
        entry.getKey().getSequenceId() >= seqId.longValue());
      if (count < 100) {
        assertEquals(encodedName + " is pushed before parent " + region.getEncodedName(),
          region.getEncodedName(), encodedName);
      } else {
        assertNotEquals(region.getEncodedName(), encodedName);
      }
      count++;
    }
    assertEquals(200, count);
  }
}
 

@Test
public void testRegionMerge() throws Exception {
  byte[] splitKey = Bytes.toBytes(50);
  TableName tableName = TableName.valueOf(name.getMethodName());
  UTIL.getAdmin().createTable(
    TableDescriptorBuilder.newBuilder(tableName)
      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(CF)
        .setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build())
      .build(),
    new byte[][] { splitKey });
  UTIL.waitTableAvailable(tableName);
  try (Table table = UTIL.getConnection().getTable(tableName)) {
    for (int i = 0; i < 100; i++) {
      table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i)));
    }
  }
  List<RegionInfo> regions = UTIL.getAdmin().getRegions(tableName);
  UTIL.getAdmin()
    .mergeRegionsAsync(
      regions.stream().map(RegionInfo::getEncodedNameAsBytes).toArray(byte[][]::new), false)
    .get(30, TimeUnit.SECONDS);
  UTIL.waitUntilNoRegionsInTransition(30000);
  List<RegionInfo> regionsAfterMerge = UTIL.getAdmin().getRegions(tableName);
  assertEquals(1, regionsAfterMerge.size());
  try (Table table = UTIL.getConnection().getTable(tableName)) {
    for (int i = 0; i < 100; i++) {
      table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i)));
    }
  }
  enablePeerAndWaitUntilReplicationDone(200);
  Map<String, Long> regionsToSeqId = new HashMap<>();
  RegionInfo region = regionsAfterMerge.get(0);
  regionsToSeqId.put(region.getEncodedName(), -1L);
  regions.stream().map(RegionInfo::getEncodedName).forEach(n -> regionsToSeqId.put(n, -1L));
  try (WAL.Reader reader =
    WALFactory.createReader(UTIL.getTestFileSystem(), logPath, UTIL.getConfiguration())) {
    int count = 0;
    for (Entry entry;;) {
      entry = reader.next();
      if (entry == null) {
        break;
      }
      String encodedName = Bytes.toString(entry.getKey().getEncodedRegionName());
      Long seqId = regionsToSeqId.get(encodedName);
      assertNotNull(
        "Unexcepted entry " + entry + ", expected regions " + region + ", or " + regions, seqId);
      assertTrue("Sequence id go backwards from " + seqId + " to " +
        entry.getKey().getSequenceId() + " for " + encodedName,
        entry.getKey().getSequenceId() >= seqId.longValue());
      if (count < 100) {
        assertNotEquals(
          encodedName + " is pushed before parents " +
            regions.stream().map(RegionInfo::getEncodedName).collect(Collectors.joining(" and ")),
          region.getEncodedName(), encodedName);
      } else {
        assertEquals(region.getEncodedName(), encodedName);
      }
      count++;
    }
    assertEquals(200, count);
  }
}
 

@Test
public void testRace() throws Exception {
  UTIL.waitFor(30000, new ExplainingPredicate<Exception>() {

    @Override
    public boolean evaluate() throws Exception {
      for (RegionServerThread t : UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
        ReplicationSource source =
          (ReplicationSource) ((Replication) t.getRegionServer().getReplicationSourceService())
            .getReplicationManager().getSource(PEER_ID);
        if (source == null || source.getReplicationEndpoint() == null) {
          return false;
        }
      }
      return true;
    }

    @Override
    public String explainFailure() throws Exception {
      return "Replication source has not been initialized yet";
    }
  });
  UTIL.getAdmin().createTable(
    TableDescriptorBuilder.newBuilder(TABLE_NAME).setColumnFamily(ColumnFamilyDescriptorBuilder
      .newBuilder(CF).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build()).build());
  UTIL.waitTableAvailable(TABLE_NAME);
  try (Table table = UTIL.getConnection().getTable(TABLE_NAME)) {
    table.put(new Put(Bytes.toBytes(1)).addColumn(CF, CQ, Bytes.toBytes(1)));
  }
  NULL_UUID = false;
  UTIL.waitFor(30000, new ExplainingPredicate<Exception>() {

    @Override
    public boolean evaluate() throws Exception {
      try (WAL.Reader reader = WALFactory.createReader(FS, LOG_PATH, UTIL.getConfiguration())) {
        return reader.next() != null;
      } catch (IOException e) {
        return false;
      }
    }

    @Override
    public String explainFailure() throws Exception {
      return "Replication has not catched up";
    }
  });
  try (WAL.Reader reader = WALFactory.createReader(FS, LOG_PATH, UTIL.getConfiguration())) {
    Cell cell = reader.next().getEdit().getCells().get(0);
    assertEquals(1, Bytes.toInt(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()));
    assertArrayEquals(CF, CellUtil.cloneFamily(cell));
    assertArrayEquals(CQ, CellUtil.cloneQualifier(cell));
    assertEquals(1,
      Bytes.toInt(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
  }
}
 

/**
 * @param fs
 * @param conf
 * @param edits
 * @param region
 * @return Return how many edits seen.
 * @throws IOException
 */
private int verifyAllEditsMadeItIn(final FileSystem fs, final Configuration conf,
    final Path edits, final HRegion region) throws IOException {
  int count = 0;
  // Read all cells from recover edits
  List<Cell> walCells = new ArrayList<>();
  try (WAL.Reader reader = WALFactory.createReader(fs, edits, conf)) {
    WAL.Entry entry;
    while ((entry = reader.next()) != null) {
      WALKey key = entry.getKey();
      WALEdit val = entry.getEdit();
      count++;
      // Check this edit is for this region.
      if (!Bytes.equals(key.getEncodedRegionName(),
          region.getRegionInfo().getEncodedNameAsBytes())) {
        continue;
      }
      Cell previous = null;
      for (Cell cell : val.getCells()) {
        if (WALEdit.isMetaEditFamily(cell)) {
          continue;
        }
        if (previous != null && CellComparatorImpl.COMPARATOR.compareRows(previous, cell) == 0) {
          continue;
        }
        previous = cell;
        walCells.add(cell);
      }
    }
  }

  // Read all cells from region
  List<Cell> regionCells = new ArrayList<>();
  try (RegionScanner scanner = region.getScanner(new Scan())) {
    List<Cell> tmpCells;
    do {
      tmpCells = new ArrayList<>();
      scanner.nextRaw(tmpCells);
      regionCells.addAll(tmpCells);
    } while (!tmpCells.isEmpty());
  }

  Collections.sort(walCells, CellComparatorImpl.COMPARATOR);
  int found = 0;
  for (int i = 0, j = 0; i < walCells.size() && j < regionCells.size(); ) {
    int compareResult = PrivateCellUtil
        .compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, walCells.get(i),
            regionCells.get(j));
    if (compareResult == 0) {
      i++;
      j++;
      found++;
    } else if (compareResult > 0) {
      j++;
    } else {
      i++;
    }
  }
  assertEquals("Only found " + found + " cells in region, but there are " + walCells.size() +
      " cells in recover edits", found, walCells.size());
  return count;
}
 

WAL.Reader createWALReaderForPrimary() throws FileNotFoundException, IOException {
  return WALFactory.createReader(TEST_UTIL.getTestFileSystem(),
    AbstractFSWALProvider.getCurrentFileName(walPrimary),
    TEST_UTIL.getConfiguration());
}