Java Code Examples for org.apache.hadoop.hbase.zookeeper.ZKUtil#setData()

The following examples show how to use org.apache.hadoop.hbase.zookeeper.ZKUtil#setData() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestSplitLogManager.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testWorkerCrash() throws Exception {
  slm = new SplitLogManager(master, conf);
  TaskBatch batch = new TaskBatch();

  String tasknode = submitTaskAndWait(batch, "foo/1");
  final ServerName worker1 = ServerName.valueOf("worker1,1,1");

  SplitLogTask slt = new SplitLogTask.Owned(worker1);
  ZKUtil.setData(zkw, tasknode, slt.toByteArray());
  if (tot_mgr_heartbeat.sum() == 0) {
    waitForCounter(tot_mgr_heartbeat, 0, 1, to/2);
  }

  // Not yet resubmitted.
  Assert.assertEquals(0, tot_mgr_resubmit.sum());

  // This server becomes dead
  Mockito.when(sm.isServerOnline(worker1)).thenReturn(false);

  Thread.sleep(1300); // The timeout checker is done every 1000 ms (hardcoded).

  // It has been resubmitted
  Assert.assertEquals(1, tot_mgr_resubmit.sum());
}
 
Example 2
Source File: MirroringTableStateManager.java    From hbase with Apache License 2.0 6 votes vote down vote up
private void updateZooKeeper(TableState tableState) throws IOException {
  if (tableState == null) {
    return;
  }
  String znode = ZNodePaths.joinZNode(this.master.getZooKeeper().getZNodePaths().tableZNode,
    tableState.getTableName().getNameAsString());
  try {
    // Make sure znode exists.
    if (ZKUtil.checkExists(this.master.getZooKeeper(), znode) == -1) {
      ZKUtil.createAndFailSilent(this.master.getZooKeeper(), znode);
    }
    // Now set newState
    ZooKeeperProtos.DeprecatedTableState.Builder builder =
      ZooKeeperProtos.DeprecatedTableState.newBuilder();
    builder.setState(
      ZooKeeperProtos.DeprecatedTableState.State.valueOf(tableState.getState().toString()));
    byte[] data = ProtobufUtil.prependPBMagic(builder.build().toByteArray());
    ZKUtil.setData(this.master.getZooKeeper(), znode, data);
  } catch (KeeperException e) {
    // Only hbase1 clients suffer if this fails.
    LOG.warn("Failed setting table state to zookeeper mirrored for hbase-1.x clients", e);
  }
}
 
Example 3
Source File: TestSplitLogManager.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testTaskResigned() throws Exception {
  LOG.info("TestTaskResigned - resubmit task node once in RESIGNED state");
  assertEquals(0, tot_mgr_resubmit.sum());
  slm = new SplitLogManager(master, conf);
  assertEquals(0, tot_mgr_resubmit.sum());
  TaskBatch batch = new TaskBatch();
  String tasknode = submitTaskAndWait(batch, "foo/1");
  assertEquals(0, tot_mgr_resubmit.sum());
  final ServerName worker1 = ServerName.valueOf("worker1,1,1");
  assertEquals(0, tot_mgr_resubmit.sum());
  SplitLogTask slt = new SplitLogTask.Resigned(worker1);
  assertEquals(0, tot_mgr_resubmit.sum());
  ZKUtil.setData(zkw, tasknode, slt.toByteArray());
  ZKUtil.checkExists(zkw, tasknode);
  // Could be small race here.
  if (tot_mgr_resubmit.sum() == 0) {
    waitForCounter(tot_mgr_resubmit, 0, 1, to/2);
  }
  assertEquals(1, tot_mgr_resubmit.sum());

  byte[] taskstate = ZKUtil.getData(zkw, tasknode);
  slt = SplitLogTask.parseFrom(taskstate);
  assertTrue(slt.isUnassigned(master.getServerName()));
}
 
Example 4
Source File: TestSplitLogManager.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testTaskErr() throws Exception {
  LOG.info("TestTaskErr - cleanup task node once in ERR state");

  conf.setInt("hbase.splitlog.max.resubmit", 0);
  slm = new SplitLogManager(master, conf);
  TaskBatch batch = new TaskBatch();

  String tasknode = submitTaskAndWait(batch, "foo/1");
  final ServerName worker1 = ServerName.valueOf("worker1,1,1");
  SplitLogTask slt = new SplitLogTask.Err(worker1);
  ZKUtil.setData(zkw, tasknode, slt.toByteArray());

  synchronized (batch) {
    while (batch.installed != batch.error) {
      batch.wait();
    }
  }
  waitForCounter(tot_mgr_task_deleted, 0, 1, to/2);
  assertTrue(ZKUtil.checkExists(zkw, tasknode) == -1);
  conf.setInt("hbase.splitlog.max.resubmit", ZKSplitLogManagerCoordination.DEFAULT_MAX_RESUBMIT);
}
 
Example 5
Source File: TestSplitLogManager.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testTaskDone() throws Exception {
  LOG.info("TestTaskDone - cleanup task node once in DONE state");

  slm = new SplitLogManager(master, conf);
  TaskBatch batch = new TaskBatch();
  String tasknode = submitTaskAndWait(batch, "foo/1");
  final ServerName worker1 = ServerName.valueOf("worker1,1,1");
  SplitLogTask slt = new SplitLogTask.Done(worker1);
  ZKUtil.setData(zkw, tasknode, slt.toByteArray());
  synchronized (batch) {
    while (batch.installed != batch.done) {
      batch.wait();
    }
  }
  waitForCounter(tot_mgr_task_deleted, 0, 1, to/2);
  assertTrue(ZKUtil.checkExists(zkw, tasknode) == -1);
}
 
Example 6
Source File: ZKReplicationPeerStorage.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public void updatePeerConfig(String peerId, ReplicationPeerConfig peerConfig)
    throws ReplicationException {
  try {
    ZKUtil.setData(this.zookeeper, getPeerNode(peerId),
      ReplicationPeerConfigUtil.toByteArray(peerConfig));
  } catch (KeeperException e) {
    throw new ReplicationException(
      "There was a problem trying to save changes to the " + "replication peer " + peerId, e);
  }
}
 
Example 7
Source File: TestSplitLogWorker.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testPreemptTask() throws Exception {
  LOG.info("testPreemptTask");
  SplitLogCounters.resetCounters();
  final ServerName SRV = ServerName.valueOf("tpt_svr,1,1");
  final String PATH = ZKSplitLog.getEncodedNodeName(zkw, "tpt_task");
  RegionServerServices mockedRS = getRegionServer(SRV);
  SplitLogWorker slw =
      new SplitLogWorker(ds, TEST_UTIL.getConfiguration(), mockedRS, neverEndingTask);
  slw.start();
  try {
    Thread.yield(); // let the worker start
    Thread.sleep(1000);
    waitForCounter(SplitLogCounters.tot_wkr_task_grabing, 0, 1, WAIT_TIME);

    // this time create a task node after starting the splitLogWorker
    zkw.getRecoverableZooKeeper().create(PATH,
      new SplitLogTask.Unassigned(MANAGER).toByteArray(),
      Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);

    waitForCounter(SplitLogCounters.tot_wkr_task_acquired, 0, 1, WAIT_TIME);
    assertEquals(1, slw.getTaskReadySeq());
    byte [] bytes = ZKUtil.getData(zkw, PATH);
    SplitLogTask slt = SplitLogTask.parseFrom(bytes);
    assertTrue(slt.isOwned(SRV));
    slt = new SplitLogTask.Owned(MANAGER);
    ZKUtil.setData(zkw, PATH, slt.toByteArray());
    waitForCounter(SplitLogCounters.tot_wkr_preempt_task, 0, 1, WAIT_TIME);
  } finally {
    stopSplitLogWorker(slw);
  }
}
 
Example 8
Source File: TestSplitLogManager.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testDeadWorker() throws Exception {
  LOG.info("testDeadWorker");

  conf.setLong("hbase.splitlog.max.resubmit", 0);
  slm = new SplitLogManager(master, conf);
  TaskBatch batch = new TaskBatch();

  String tasknode = submitTaskAndWait(batch, "foo/1");
  int version = ZKUtil.checkExists(zkw, tasknode);
  final ServerName worker1 = ServerName.valueOf("worker1,1,1");
  SplitLogTask slt = new SplitLogTask.Owned(worker1);
  ZKUtil.setData(zkw, tasknode, slt.toByteArray());
  if (tot_mgr_heartbeat.sum() == 0) {
    waitForCounter(tot_mgr_heartbeat, 0, 1, to/2);
  }
  slm.handleDeadWorker(worker1);
  if (tot_mgr_resubmit.sum() == 0) {
    waitForCounter(tot_mgr_resubmit, 0, 1, to+to/2);
  }
  if (tot_mgr_resubmit_dead_server_task.sum() == 0) {
    waitForCounter(tot_mgr_resubmit_dead_server_task, 0, 1, to + to/2);
  }

  int version1 = ZKUtil.checkExists(zkw, tasknode);
  assertTrue(version1 > version);
  byte[] taskstate = ZKUtil.getData(zkw, tasknode);
  slt = SplitLogTask.parseFrom(taskstate);
  assertTrue(slt.isUnassigned(master.getServerName()));
  return;
}
 
Example 9
Source File: TestSplitLogManager.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testUnassignedTimeout() throws Exception {
  LOG.info("TestUnassignedTimeout - iff all tasks are unassigned then" +
      " resubmit");

  // create an orphan task in OWNED state
  String tasknode1 = ZKSplitLog.getEncodedNodeName(zkw, "orphan/1");
  final ServerName worker1 = ServerName.valueOf("worker1,1,1");
  SplitLogTask slt = new SplitLogTask.Owned(worker1);
  zkw.getRecoverableZooKeeper().create(tasknode1, slt.toByteArray(), Ids.OPEN_ACL_UNSAFE,
      CreateMode.PERSISTENT);

  slm = new SplitLogManager(master, conf);
  waitForCounter(tot_mgr_orphan_task_acquired, 0, 1, to/2);

  // submit another task which will stay in unassigned mode
  TaskBatch batch = new TaskBatch();
  submitTaskAndWait(batch, "foo/1");

  // keep updating the orphan owned node every to/2 seconds
  for (int i = 0; i < (3 * to)/100; i++) {
    Thread.sleep(100);
    final ServerName worker2 = ServerName.valueOf("worker1,1,1");
    slt = new SplitLogTask.Owned(worker2);
    ZKUtil.setData(zkw, tasknode1, slt.toByteArray());
  }

  // since we have stopped heartbeating the owned node therefore it should
  // get resubmitted
  LOG.info("waiting for manager to resubmit the orphan task");
  waitForCounter(tot_mgr_resubmit, 0, 1, to + to/2);

  // now all the nodes are unassigned. manager should post another rescan
  waitForCounter(tot_mgr_resubmit_unassigned, 0, 1, 2 * to + to/2);
}
 
Example 10
Source File: TestSplitLogManager.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testRescanCleanup() throws Exception {
  LOG.info("TestRescanCleanup - ensure RESCAN nodes are cleaned up");

  slm = new SplitLogManager(master, conf);
  TaskBatch batch = new TaskBatch();

  String tasknode = submitTaskAndWait(batch, "foo/1");
  int version = ZKUtil.checkExists(zkw, tasknode);
  final ServerName worker1 = ServerName.valueOf("worker1,1,1");
  SplitLogTask slt = new SplitLogTask.Owned(worker1);
  ZKUtil.setData(zkw, tasknode, slt.toByteArray());
  waitForCounter(tot_mgr_heartbeat, 0, 1, to/2);
  waitForCounter(new Expr() {
    @Override
    public long eval() {
      return (tot_mgr_resubmit.sum() + tot_mgr_resubmit_failed.sum());
    }
  }, 0, 1, 5*60000); // wait long enough
  Assert.assertEquals("Could not run test. Lost ZK connection?",
    0, tot_mgr_resubmit_failed.sum());
  int version1 = ZKUtil.checkExists(zkw, tasknode);
  assertTrue(version1 > version);
  byte[] taskstate = ZKUtil.getData(zkw, tasknode);
  slt = SplitLogTask.parseFrom(taskstate);
  assertTrue(slt.isUnassigned(master.getServerName()));

  waitForCounter(tot_mgr_rescan_deleted, 0, 1, to/2);
}
 
Example 11
Source File: TestSplitLogManager.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testMultipleResubmits() throws Exception {
  LOG.info("TestMultipleResbmits - no indefinite resubmissions");
  conf.setInt("hbase.splitlog.max.resubmit", 2);
  slm = new SplitLogManager(master, conf);
  TaskBatch batch = new TaskBatch();

  String tasknode = submitTaskAndWait(batch, "foo/1");
  int version = ZKUtil.checkExists(zkw, tasknode);
  final ServerName worker1 = ServerName.valueOf("worker1,1,1");
  final ServerName worker2 = ServerName.valueOf("worker2,1,1");
  final ServerName worker3 = ServerName.valueOf("worker3,1,1");
  SplitLogTask slt = new SplitLogTask.Owned(worker1);
  ZKUtil.setData(zkw, tasknode, slt.toByteArray());
  waitForCounter(tot_mgr_heartbeat, 0, 1, to/2);
  waitForCounter(tot_mgr_resubmit, 0, 1, to + to/2);
  int version1 = ZKUtil.checkExists(zkw, tasknode);
  assertTrue(version1 > version);
  slt = new SplitLogTask.Owned(worker2);
  ZKUtil.setData(zkw, tasknode, slt.toByteArray());
  waitForCounter(tot_mgr_heartbeat, 1, 2, to/2);
  waitForCounter(tot_mgr_resubmit, 1, 2, to + to/2);
  int version2 = ZKUtil.checkExists(zkw, tasknode);
  assertTrue(version2 > version1);
  slt = new SplitLogTask.Owned(worker3);
  ZKUtil.setData(zkw, tasknode, slt.toByteArray());
  waitForCounter(tot_mgr_heartbeat, 2, 3, to/2);
  waitForCounter(tot_mgr_resubmit_threshold_reached, 0, 1, to + to/2);
  Thread.sleep(to + to/2);
  assertEquals(2L, tot_mgr_resubmit.sum() - tot_mgr_resubmit_force.sum());
}
 
Example 12
Source File: SplitOrMergeTracker.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Set the switch on/off
 * @param enabled switch enabled or not?
 * @throws KeeperException keepException will be thrown out
 */
public void setSwitchEnabled(boolean enabled) throws KeeperException {
  byte [] upData = toByteArray(enabled);
  try {
    ZKUtil.setData(watcher, node, upData);
  } catch(KeeperException.NoNodeException nne) {
    ZKUtil.createAndWatch(watcher, node, upData);
  }
  super.nodeDataChanged(node);
}
 
Example 13
Source File: TestZKReplicationQueueStorage.java    From hbase with Apache License 2.0 5 votes vote down vote up
private ZKReplicationQueueStorage createWithUnstableVersion() throws IOException {
  return new ZKReplicationQueueStorage(UTIL.getZooKeeperWatcher(), UTIL.getConfiguration()) {

    private int called = 0;
    private int getLastSeqIdOpIndex = 0;

    @Override
    protected int getQueuesZNodeCversion() throws KeeperException {
      if (called < 4) {
        called++;
      }
      return called;
    }

    @Override
    protected Pair<Long, Integer> getLastSequenceIdWithVersion(String encodedRegionName,
        String peerId) throws KeeperException {
      Pair<Long, Integer> oldPair = super.getLastSequenceIdWithVersion(encodedRegionName, peerId);
      if (getLastSeqIdOpIndex < 100) {
        // Let the ZNode version increase.
        String path = getSerialReplicationRegionPeerNode(encodedRegionName, peerId);
        ZKUtil.createWithParents(zookeeper, path);
        ZKUtil.setData(zookeeper, path, ZKUtil.positionToByteArray(100L));
      }
      getLastSeqIdOpIndex++;
      return oldPair;
    }
  };
}
 
Example 14
Source File: ZKReplicationPeerStorage.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public void setPeerState(String peerId, boolean enabled) throws ReplicationException {
  byte[] stateBytes = enabled ? ENABLED_ZNODE_BYTES : DISABLED_ZNODE_BYTES;
  try {
    ZKUtil.setData(zookeeper, getPeerStateNode(peerId), stateBytes);
  } catch (KeeperException e) {
    throw new ReplicationException("Unable to change state of the peer with id=" + peerId, e);
  }
}
 
Example 15
Source File: TestReplicationSourceManager.java    From hbase with Apache License 2.0 4 votes vote down vote up
protected static void setupZkAndReplication() throws Exception {
  // The implementing class should set up the conf
  assertNotNull(conf);
  zkw = new ZKWatcher(conf, "test", null);
  ZKUtil.createWithParents(zkw, "/hbase/replication");
  ZKUtil.createWithParents(zkw, "/hbase/replication/peers/1");
  ZKUtil.setData(zkw, "/hbase/replication/peers/1",
      Bytes.toBytes(conf.get(HConstants.ZOOKEEPER_QUORUM) + ":"
          + conf.get(HConstants.ZOOKEEPER_CLIENT_PORT) + ":/1"));
  ZKUtil.createWithParents(zkw, "/hbase/replication/peers/1/peer-state");
  ZKUtil.setData(zkw, "/hbase/replication/peers/1/peer-state",
    ZKReplicationPeerStorage.ENABLED_ZNODE_BYTES);
  ZKUtil.createWithParents(zkw, "/hbase/replication/peers/1/sync-rep-state");
  ZKUtil.setData(zkw, "/hbase/replication/peers/1/sync-rep-state",
    ZKReplicationPeerStorage.NONE_STATE_ZNODE_BYTES);
  ZKUtil.createWithParents(zkw, "/hbase/replication/peers/1/new-sync-rep-state");
  ZKUtil.setData(zkw, "/hbase/replication/peers/1/new-sync-rep-state",
    ZKReplicationPeerStorage.NONE_STATE_ZNODE_BYTES);
  ZKUtil.createWithParents(zkw, "/hbase/replication/state");
  ZKUtil.setData(zkw, "/hbase/replication/state", ZKReplicationPeerStorage.ENABLED_ZNODE_BYTES);

  ZKClusterId.setClusterId(zkw, new ClusterId());
  CommonFSUtils.setRootDir(utility.getConfiguration(), utility.getDataTestDir());
  fs = FileSystem.get(conf);
  oldLogDir = utility.getDataTestDir(HConstants.HREGION_OLDLOGDIR_NAME);
  logDir = utility.getDataTestDir(HConstants.HREGION_LOGDIR_NAME);
  remoteLogDir = utility.getDataTestDir(ReplicationUtils.REMOTE_WAL_DIR_NAME);
  replication = new Replication();
  replication.initialize(new DummyServer(), fs, logDir, oldLogDir, null);
  managerOfCluster = getManagerFromCluster();
  if (managerOfCluster != null) {
    // After replication procedure, we need to add peer by hand (other than by receiving
    // notification from zk)
    managerOfCluster.addPeer(slaveId);
  }

  manager = replication.getReplicationManager();
  manager.addSource(slaveId);
  if (managerOfCluster != null) {
    waitPeer(slaveId, managerOfCluster, true);
  }
  waitPeer(slaveId, manager, true);

  htd = TableDescriptorBuilder.newBuilder(test)
    .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(f1)
      .setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build())
    .setColumnFamily(ColumnFamilyDescriptorBuilder.of(f2)).build();

  scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
  for(byte[] fam : htd.getColumnFamilyNames()) {
    scopes.put(fam, 0);
  }
  hri = RegionInfoBuilder.newBuilder(htd.getTableName()).setStartKey(r1).setEndKey(r2).build();
}
 
Example 16
Source File: TestSplitLogWorker.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Test
public void testMultipleTasks() throws Exception {
  LOG.info("testMultipleTasks");
  SplitLogCounters.resetCounters();
  final ServerName SRV = ServerName.valueOf("tmt_svr,1,1");
  final String PATH1 = ZKSplitLog.getEncodedNodeName(zkw, "tmt_task");
  RegionServerServices mockedRS = getRegionServer(SRV);
  SplitLogWorker slw =
      new SplitLogWorker(ds, TEST_UTIL.getConfiguration(), mockedRS, neverEndingTask);
  slw.start();
  try {
    Thread.yield(); // let the worker start
    Thread.sleep(100);
    waitForCounter(SplitLogCounters.tot_wkr_task_grabing, 0, 1, WAIT_TIME);

    SplitLogTask unassignedManager =
      new SplitLogTask.Unassigned(MANAGER);
    zkw.getRecoverableZooKeeper().create(PATH1, unassignedManager.toByteArray(),
      Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);

    waitForCounter(SplitLogCounters.tot_wkr_task_acquired, 0, 1, WAIT_TIME);
    // now the worker is busy doing the above task

    // create another task
    final String PATH2 = ZKSplitLog.getEncodedNodeName(zkw, "tmt_task_2");
    zkw.getRecoverableZooKeeper().create(PATH2, unassignedManager.toByteArray(),
      Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);

    // preempt the first task, have it owned by another worker
    final ServerName anotherWorker = ServerName.valueOf("another-worker,1,1");
    SplitLogTask slt = new SplitLogTask.Owned(anotherWorker);
    ZKUtil.setData(zkw, PATH1, slt.toByteArray());
    waitForCounter(SplitLogCounters.tot_wkr_preempt_task, 0, 1, WAIT_TIME);

    waitForCounter(SplitLogCounters.tot_wkr_task_acquired, 1, 2, WAIT_TIME);
    assertEquals(2, slw.getTaskReadySeq());
    byte [] bytes = ZKUtil.getData(zkw, PATH2);
    slt = SplitLogTask.parseFrom(bytes);
    assertTrue(slt.isOwned(SRV));
  } finally {
    stopSplitLogWorker(slw);
  }
}
 
Example 17
Source File: TestSplitLogWorker.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Test
public void testRescan() throws Exception {
  LOG.info("testRescan");
  SplitLogCounters.resetCounters();
  final ServerName SRV = ServerName.valueOf("svr,1,1");
  RegionServerServices mockedRS = getRegionServer(SRV);
  slw = new SplitLogWorker(ds, TEST_UTIL.getConfiguration(), mockedRS, neverEndingTask);
  slw.start();
  Thread.yield(); // let the worker start
  Thread.sleep(100);

  String task = ZKSplitLog.getEncodedNodeName(zkw, "task");
  SplitLogTask slt = new SplitLogTask.Unassigned(MANAGER);
  zkw.getRecoverableZooKeeper().create(task,slt.toByteArray(), Ids.OPEN_ACL_UNSAFE,
    CreateMode.PERSISTENT);

  waitForCounter(SplitLogCounters.tot_wkr_task_acquired, 0, 1, WAIT_TIME);
  // now the worker is busy doing the above task

  // preempt the task, have it owned by another worker
  ZKUtil.setData(zkw, task, slt.toByteArray());
  waitForCounter(SplitLogCounters.tot_wkr_preempt_task, 0, 1, WAIT_TIME);

  // create a RESCAN node
  String rescan = ZKSplitLog.getEncodedNodeName(zkw, "RESCAN");
  rescan = zkw.getRecoverableZooKeeper().create(rescan, slt.toByteArray(), Ids.OPEN_ACL_UNSAFE,
    CreateMode.PERSISTENT_SEQUENTIAL);

  waitForCounter(SplitLogCounters.tot_wkr_task_acquired, 1, 2, WAIT_TIME);
  // RESCAN node might not have been processed if the worker became busy
  // with the above task. preempt the task again so that now the RESCAN
  // node is processed
  ZKUtil.setData(zkw, task, slt.toByteArray());
  waitForCounter(SplitLogCounters.tot_wkr_preempt_task, 1, 2, WAIT_TIME);
  waitForCounter(SplitLogCounters.tot_wkr_task_acquired_rescan, 0, 1, WAIT_TIME);

  List<String> nodes = ZKUtil.listChildrenNoWatch(zkw, zkw.getZNodePaths().splitLogZNode);
  LOG.debug(Objects.toString(nodes));
  int num = 0;
  for (String node : nodes) {
    num++;
    if (node.startsWith("RESCAN")) {
      String name = ZKSplitLog.getEncodedNodeName(zkw, node);
      String fn = ZKSplitLog.getFileName(name);
      byte [] data = ZKUtil.getData(zkw,
              ZNodePaths.joinZNode(zkw.getZNodePaths().splitLogZNode, fn));
      slt = SplitLogTask.parseFrom(data);
      assertTrue(slt.toString(), slt.isDone(SRV));
    }
  }
  assertEquals(2, num);
}