Java Code Examples for org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil#toRegionInfo()

/**
 * Parses an RegionInfo instance from the passed in stream.
 * Presumes the RegionInfo was serialized to the stream with
 * {@link #toDelimitedByteArray(RegionInfo)}.
 * @return An instance of RegionInfo.
 */
static RegionInfo parseFrom(final DataInputStream in) throws IOException {
  // I need to be able to move back in the stream if this is not a pb
  // serialization so I can do the Writable decoding instead.
  int pblen = ProtobufUtil.lengthOfPBMagic();
  byte [] pbuf = new byte[pblen];
  if (in.markSupported()) { //read it with mark()
    in.mark(pblen);
  }

  //assumption: if Writable serialization, it should be longer than pblen.
  int read = in.read(pbuf);
  if (read != pblen) throw new IOException("read=" + read + ", wanted=" + pblen);
  if (ProtobufUtil.isPBMagicPrefix(pbuf)) {
    return ProtobufUtil.toRegionInfo(HBaseProtos.RegionInfo.parseDelimitedFrom(in));
  } else {
    throw new IOException("PB encoded RegionInfo expected");
  }
}
 

/**
 * @param bytes A pb RegionInfo serialized with a pb magic prefix.
 * @param offset starting point in the byte array
 * @param len length to read on the byte array
 * @return A deserialized {@link RegionInfo}
 */
@InterfaceAudience.Private
static RegionInfo parseFrom(final byte [] bytes, int offset, int len)
throws DeserializationException {
  if (ProtobufUtil.isPBMagicPrefix(bytes, offset, len)) {
    int pblen = ProtobufUtil.lengthOfPBMagic();
    try {
      HBaseProtos.RegionInfo.Builder builder = HBaseProtos.RegionInfo.newBuilder();
      ProtobufUtil.mergeFrom(builder, bytes, pblen + offset, len - pblen);
      HBaseProtos.RegionInfo ri = builder.build();
      return ProtobufUtil.toRegionInfo(ri);
    } catch (IOException e) {
      throw new DeserializationException(e);
    }
  } else {
    throw new DeserializationException("PB encoded RegionInfo expected");
  }
}
 

public static List<RegionInfo> getRegionInfosFromManifest(SnapshotManifest manifest) {
  List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
  if (regionManifests == null) {
    throw new IllegalArgumentException("Snapshot seems empty");
  }

  List<RegionInfo> regionInfos = Lists.newArrayListWithCapacity(regionManifests.size());

  for (SnapshotRegionManifest regionManifest : regionManifests) {
    RegionInfo hri = ProtobufUtil.toRegionInfo(regionManifest.getRegionInfo());
    if (hri.isOffline() && (hri.isSplit() || hri.isSplitParent())) {
      continue;
    }
    regionInfos.add(hri);
  }
  return regionInfos;
}
 

@Override
protected void deserializeStateData(ProcedureStateSerializer serializer)
    throws IOException {
  final LockProcedureData state = serializer.deserialize(LockProcedureData.class);
  type = LockType.valueOf(state.getLockType().name());
  description = state.getDescription();
  if (state.getRegionInfoCount() > 0) {
    regionInfos = new RegionInfo[state.getRegionInfoCount()];
    for (int i = 0; i < state.getRegionInfoCount(); ++i) {
      regionInfos[i] = ProtobufUtil.toRegionInfo(state.getRegionInfo(i));
    }
  } else if (state.hasNamespace()) {
    namespace = state.getNamespace();
  } else if (state.hasTableName()) {
    tableName = ProtobufUtil.toTableName(state.getTableName());
  }
  recoveredMasterLock = state.getIsMasterLock();
  this.lock = setupLock();
}
 

@Override
protected RegionOpeningState execOpenRegion(ServerName server, RegionOpenInfo openReq)
    throws IOException {
  RegionInfo hri = ProtobufUtil.toRegionInfo(openReq.getRegion());
  long previousOpenSeqNum =
    am.getRegionStates().getOrCreateRegionStateNode(hri).getOpenSeqNum();
  sendTransitionReport(server, openReq.getRegion(), TransitionCode.OPENED,
    previousOpenSeqNum + 2);
  // Concurrency?
  // Now update the state of our cluster in regionsToRegionServers.
  SortedSet<byte[]> regions = regionsToRegionServers.get(server);
  if (regions == null) {
    regions = new ConcurrentSkipListSet<byte[]>(Bytes.BYTES_COMPARATOR);
    regionsToRegionServers.put(server, regions);
  }
  if (regions.contains(hri.getRegionName())) {
    throw new UnsupportedOperationException(hri.getRegionNameAsString());
  }
  regions.add(hri.getRegionName());
  return RegionOpeningState.OPENED;
}
 

@Test
public void testConvert() {
  final TableName tableName =
    TableName.valueOf("ns1:" + name.getTableName().getQualifierAsString());
  byte[] startKey = Bytes.toBytes("startKey");
  byte[] endKey = Bytes.toBytes("endKey");
  boolean split = false;
  long regionId = System.currentTimeMillis();
  int replicaId = 42;

  RegionInfo ri = RegionInfoBuilder.newBuilder(tableName).setStartKey(startKey).setEndKey(endKey)
    .setSplit(split).setRegionId(regionId).setReplicaId(replicaId).build();

  // convert two times, compare
  RegionInfo convertedRi = ProtobufUtil.toRegionInfo(ProtobufUtil.toRegionInfo(ri));

  assertEquals(ri, convertedRi);

  // test convert RegionInfo without replicaId
  HBaseProtos.RegionInfo info = HBaseProtos.RegionInfo.newBuilder()
    .setTableName(HBaseProtos.TableName.newBuilder()
      .setQualifier(UnsafeByteOperations.unsafeWrap(tableName.getQualifier()))
      .setNamespace(UnsafeByteOperations.unsafeWrap(tableName.getNamespace())).build())
    .setStartKey(UnsafeByteOperations.unsafeWrap(startKey))
    .setEndKey(UnsafeByteOperations.unsafeWrap(endKey)).setSplit(split).setRegionId(regionId)
    .build();

  convertedRi = ProtobufUtil.toRegionInfo(info);
  RegionInfo expectedRi = RegionInfoBuilder.newBuilder(tableName).setStartKey(startKey)
    .setEndKey(endKey).setSplit(split).setRegionId(regionId).setReplicaId(0).build();

  assertEquals(expectedRi, convertedRi);
}
 

@Test
public void testConversion() {
  TableName tn = TableName.valueOf("table1");

  RegionInfo hri1 = RegionInfoBuilder.newBuilder(tn)
      .setStartKey(Bytes.toBytes("a"))
      .setEndKey(Bytes.toBytes("b"))
      .build();
  RegionInfo hri2 = RegionInfoBuilder.newBuilder(tn)
      .setStartKey(Bytes.toBytes("b"))
      .setEndKey(Bytes.toBytes("c"))
      .build();
  RegionInfo hri3 = RegionInfoBuilder.newBuilder(tn)
      .setStartKey(Bytes.toBytes("c"))
      .setEndKey(Bytes.toBytes("d"))
      .build();
  RegionSizeStore store = RegionSizeStoreFactory.getInstance().createStore();
  store.put(hri1, 1024L * 1024L);
  store.put(hri2, 1024L * 1024L * 8L);
  store.put(hri3, 1024L * 1024L * 32L);

  // Call the real method to convert the map into a protobuf
  HRegionServer rs = mock(HRegionServer.class);
  doCallRealMethod().when(rs).buildRegionSpaceUseReportRequest(any(RegionSizeStore.class));
  doCallRealMethod().when(rs).convertRegionSize(any(), anyLong());

  RegionSpaceUseReportRequest requests = rs.buildRegionSpaceUseReportRequest(store);
  assertEquals(store.size(), requests.getSpaceUseCount());
  for (RegionSpaceUse spaceUse : requests.getSpaceUseList()) {
    RegionInfo hri = ProtobufUtil.toRegionInfo(spaceUse.getRegionInfo());
    RegionSize expectedSize = store.remove(hri);
    assertNotNull("Could not find size for HRI: " + hri, expectedSize);
    assertEquals(expectedSize.getSize(), spaceUse.getRegionSize());
  }
  assertTrue("Should not have any space use entries left: " + store, store.isEmpty());
}
 

@Test
public void testRegionInTransition() throws IOException {
  // Check ReportRegionInTransition
  HBaseProtos.RegionInfo meta_ri =
    ProtobufUtil.toRegionInfo(RegionInfoBuilder.FIRST_META_REGIONINFO);
  HBaseProtos.RegionInfo normal_ri =
    ProtobufUtil.toRegionInfo(RegionInfoBuilder.newBuilder(TableName.valueOf("test:table"))
      .setStartKey(Bytes.toBytes("a")).setEndKey(Bytes.toBytes("b")).build());


  RegionServerStatusProtos.RegionStateTransition metaTransition = RegionServerStatusProtos
      .RegionStateTransition.newBuilder()
      .addRegionInfo(meta_ri)
      .setTransitionCode(RegionServerStatusProtos.RegionStateTransition.TransitionCode.CLOSED)
      .build();

  RegionServerStatusProtos.RegionStateTransition normalTransition = RegionServerStatusProtos
      .RegionStateTransition.newBuilder()
      .addRegionInfo(normal_ri)
      .setTransitionCode(RegionServerStatusProtos.RegionStateTransition.TransitionCode.CLOSED)
      .build();

  RegionServerStatusProtos.ReportRegionStateTransitionRequest metaTransitionRequest =
      RegionServerStatusProtos.ReportRegionStateTransitionRequest.newBuilder()
          .setServer(ProtobufUtil.toServerName(ServerName.valueOf("locahost:60020", 100)))
          .addTransition(normalTransition)
          .addTransition(metaTransition).build();

  RegionServerStatusProtos.ReportRegionStateTransitionRequest normalTransitionRequest =
      RegionServerStatusProtos.ReportRegionStateTransitionRequest.newBuilder()
          .setServer(ProtobufUtil.toServerName(ServerName.valueOf("locahost:60020", 100)))
          .addTransition(normalTransition).build();

  final String reportFuncName = "ReportRegionStateTransition";
  checkMethod(conf, reportFuncName, HConstants.META_QOS, qosFunction,
      metaTransitionRequest);
  checkMethod(conf, reportFuncName, HConstants.HIGH_QOS, qosFunction, normalTransitionRequest);
}
 

@Override
protected RegionOpeningState execOpenRegion(final ServerName server, RegionOpenInfo openReq)
    throws IOException {
  RegionInfo hri = ProtobufUtil.toRegionInfo(openReq.getRegion());
  long previousOpenSeqNum =
    am.getRegionStates().getOrCreateRegionStateNode(hri).getOpenSeqNum();
  switch (rand.nextInt(3)) {
    case 0:
      LOG.info("Return OPENED response");
      sendTransitionReport(server, openReq.getRegion(), TransitionCode.OPENED,
        previousOpenSeqNum + 2);
      return OpenRegionResponse.RegionOpeningState.OPENED;
    case 1:
      LOG.info("Return transition report that FAILED_OPEN/FAILED_OPENING response");
      sendTransitionReport(server, openReq.getRegion(), TransitionCode.FAILED_OPEN, -1);
      return OpenRegionResponse.RegionOpeningState.FAILED_OPENING;
    default:
      // fall out
  }
  // The procedure on master will just hang forever because nothing comes back
  // from the RS in this case.
  LOG.info("Return null as response; means proc stuck so we send in a crash report after" +
    " a few seconds...");
  executor.schedule(new Runnable() {
    @Override
    public void run() {
      LOG.info("Delayed CRASHING of " + server);
      doCrash(server);
    }
  }, 5, TimeUnit.SECONDS);
  return null;
}
 

/**
 * Do get of RegionInfo from Master using encoded region name.
 */
private void testGetWithRegionName(ServerName sn, RegionInfo inputRI,
    byte [] regionName) throws IOException {
  RegionInfo ri = ProtobufUtil.toRegionInfo(FutureUtils.get(
    TEST_UTIL.getAsyncConnection().getRegionServerAdmin(sn).getRegionInfo(
      ProtobufUtil.getGetRegionInfoRequest(regionName))).getRegionInfo());
  assertEquals(inputRI, ri);
}
 

private void executeOpenRegionProcedures(OpenRegionRequest request,
    Map<TableName, TableDescriptor> tdCache) {
  long masterSystemTime = request.hasMasterSystemTime() ? request.getMasterSystemTime() : -1;
  for (RegionOpenInfo regionOpenInfo : request.getOpenInfoList()) {
    RegionInfo regionInfo = ProtobufUtil.toRegionInfo(regionOpenInfo.getRegion());
    TableDescriptor tableDesc = tdCache.get(regionInfo.getTable());
    if (tableDesc == null) {
      try {
        tableDesc = regionServer.getTableDescriptors().get(regionInfo.getTable());
      } catch (IOException e) {
        // Here we do not fail the whole method since we also need deal with other
        // procedures, and we can not ignore this one, so we still schedule a
        // AssignRegionHandler and it will report back to master if we still can not get the
        // TableDescriptor.
        LOG.warn("Failed to get TableDescriptor of {}, will try again in the handler",
          regionInfo.getTable(), e);
      }
    }
    if (regionOpenInfo.getFavoredNodesCount() > 0) {
      regionServer.updateRegionFavoredNodesMapping(regionInfo.getEncodedName(),
        regionOpenInfo.getFavoredNodesList());
    }
    long procId = regionOpenInfo.getOpenProcId();
    if (regionServer.submitRegionProcedure(procId)) {
      regionServer.executorService.submit(AssignRegionHandler
          .create(regionServer, regionInfo, procId, tableDesc,
              masterSystemTime));
    }
  }
}
 

@Override
public UpdateFavoredNodesResponse updateFavoredNodes(RpcController controller,
    UpdateFavoredNodesRequest request) throws ServiceException {
  List<UpdateFavoredNodesRequest.RegionUpdateInfo> openInfoList = request.getUpdateInfoList();
  UpdateFavoredNodesResponse.Builder respBuilder = UpdateFavoredNodesResponse.newBuilder();
  for (UpdateFavoredNodesRequest.RegionUpdateInfo regionUpdateInfo : openInfoList) {
    RegionInfo hri = ProtobufUtil.toRegionInfo(regionUpdateInfo.getRegion());
    if (regionUpdateInfo.getFavoredNodesCount() > 0) {
      regionServer.updateRegionFavoredNodesMapping(hri.getEncodedName(),
        regionUpdateInfo.getFavoredNodesList());
    }
  }
  respBuilder.setResponse(openInfoList.size());
  return respBuilder.build();
}
 

@Override
protected void deserializeStateData(ProcedureStateSerializer serializer) throws IOException {
  RegionRemoteProcedureBaseStateData data =
    serializer.deserialize(RegionRemoteProcedureBaseStateData.class);
  region = ProtobufUtil.toRegionInfo(data.getRegion());
  targetServer = ProtobufUtil.toServerName(data.getTargetServer());
  state = data.getState();
  if (data.hasTransitionCode()) {
    transitionCode = data.getTransitionCode();
    seqId = data.getSeqId();
  }
}
 

/**
 * Iterate over the snapshot store files in the specified region
 *
 * @param manifest snapshot manifest to inspect
 * @param visitor callback object to get the store files
 * @throws IOException if an error occurred while scanning the directory
 */
static void visitRegionStoreFiles(final SnapshotRegionManifest manifest,
    final StoreFileVisitor visitor) throws IOException {
  RegionInfo regionInfo = ProtobufUtil.toRegionInfo(manifest.getRegionInfo());
  for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) {
    String familyName = familyFiles.getFamilyName().toStringUtf8();
    for (SnapshotRegionManifest.StoreFile storeFile: familyFiles.getStoreFilesList()) {
      visitor.storeFile(regionInfo, familyName, storeFile);
    }
  }
}
 

/**
 * Verify that the regionInfo is valid
 * @param region the region to check
 * @param manifest snapshot manifest to inspect
 */
private void verifyRegionInfo(final RegionInfo region,
    final SnapshotRegionManifest manifest) throws IOException {
  RegionInfo manifestRegionInfo = ProtobufUtil.toRegionInfo(manifest.getRegionInfo());
  if (RegionInfo.COMPARATOR.compare(region, manifestRegionInfo) != 0) {
    String msg = "Manifest region info " + manifestRegionInfo +
                 "doesn't match expected region:" + region;
    throw new CorruptedSnapshotException(msg, ProtobufUtil.createSnapshotDesc(snapshot));
  }
}
 

/**
 * Contacts a region server and waits up to timeout ms
 * to close the region.  This bypasses the active hmaster.
 * Pass -1 as timeout if you do not want to wait on result.
 */
public static void closeRegionSilentlyAndWait(AsyncClusterConnection connection,
    ServerName server, RegionInfo region, long timeout) throws IOException, InterruptedException {
  AsyncRegionServerAdmin admin = connection.getRegionServerAdmin(server);
  try {
    FutureUtils.get(
      admin.closeRegion(ProtobufUtil.buildCloseRegionRequest(server, region.getRegionName())));
  } catch (IOException e) {
    LOG.warn("Exception when closing region: " + region.getRegionNameAsString(), e);
  }
  if (timeout < 0) {
    return;
  }
  long expiration = timeout + System.currentTimeMillis();
  while (System.currentTimeMillis() < expiration) {
    try {
      RegionInfo rsRegion = ProtobufUtil.toRegionInfo(FutureUtils
        .get(
          admin.getRegionInfo(RequestConverter.buildGetRegionInfoRequest(region.getRegionName())))
        .getRegionInfo());
      if (rsRegion == null) {
        return;
      }
    } catch (IOException ioe) {
      if (ioe instanceof NotServingRegionException) {
        // no need to retry again
        return;
      }
      LOG.warn("Exception when retrieving regioninfo from: " + region.getRegionNameAsString(),
        ioe);
    }
    Thread.sleep(1000);
  }
  throw new IOException("Region " + region + " failed to close within" + " timeout " + timeout);
}
 

@Override
protected void deserializeStateData(ProcedureStateSerializer serializer)
    throws IOException {
  super.deserializeStateData(serializer);
  final GCMultipleMergedRegionsStateData msg =
      serializer.deserialize(GCMultipleMergedRegionsStateData.class);
  this.parents = msg.getParentsList().stream().map(ProtobufUtil::toRegionInfo).
      collect(Collectors.toList());
  this.mergedChild = ProtobufUtil.toRegionInfo(msg.getMergedChild());
}
 

@Override
protected void deserializeStateData(ProcedureStateSerializer serializer)
    throws IOException {
  super.deserializeStateData(serializer);
  final MasterProcedureProtos.GCMergedRegionsStateData msg =
      serializer.deserialize(MasterProcedureProtos.GCMergedRegionsStateData.class);
  this.father = ProtobufUtil.toRegionInfo(msg.getParentA());
  this.mother = ProtobufUtil.toRegionInfo(msg.getParentB());
  this.mergedChild = ProtobufUtil.toRegionInfo(msg.getMergedChild());
}
 

@Override
protected void deserializeStateData(ProcedureStateSerializer serializer)
    throws IOException {
  super.deserializeStateData(serializer);
  this.hri = ProtobufUtil.toRegionInfo(serializer.deserialize(HBaseProtos.RegionInfo.class));
}
 

/**
 * Convert a protobuf HBaseProtos.RegionState to a RegionState
 *
 * @return the RegionState
 */
public static RegionState convert(ClusterStatusProtos.RegionState proto) {
  return new RegionState(ProtobufUtil.toRegionInfo(proto.getRegionInfo()),
    State.convert(proto.getState()), proto.getStamp(), null);
}