Java Code Examples for org.apache.hadoop.hbase.client.Admin#getClusterStatus()

/**
 * As of Apache NiFi 1.5.0, due to changes made to
 * {@link SecurityUtil#loginKerberos(Configuration, String, String)}, which is used by this
 * class to authenticate a principal with Kerberos, HBase controller services no longer
 * attempt relogins explicitly.  For more information, please read the documentation for
 * {@link SecurityUtil#loginKerberos(Configuration, String, String)}.
 * <p/>
 * In previous versions of NiFi, a {@link org.apache.nifi.hadoop.KerberosTicketRenewer} was started
 * when the HBase controller service was enabled.  The use of a separate thread to explicitly relogin could cause
 * race conditions with the implicit relogin attempts made by hadoop/HBase code on a thread that references the same
 * {@link UserGroupInformation} instance.  One of these threads could leave the
 * {@link javax.security.auth.Subject} in {@link UserGroupInformation} to be cleared or in an unexpected state
 * while the other thread is attempting to use the {@link javax.security.auth.Subject}, resulting in failed
 * authentication attempts that would leave the HBase controller service in an unrecoverable state.
 *
 * @see SecurityUtil#loginKerberos(Configuration, String, String)
 */
@OnEnabled
public void onEnabled(final ConfigurationContext context) throws InitializationException, IOException, InterruptedException {
    this.connection = createConnection(context);

    // connection check
    if (this.connection != null) {
        final Admin admin = this.connection.getAdmin();
        if (admin != null) {
            admin.listTableNames();

            final ClusterStatus clusterStatus = admin.getClusterStatus();
            if (clusterStatus != null) {
                final ServerName master = clusterStatus.getMaster();
                if (master != null) {
                    masterAddress = master.getHostAndPort();
                } else {
                    masterAddress = null;
                }
            }
        }
    }
}
 

/**
 * As of Apache NiFi 1.5.0, due to changes made to
 * {@link SecurityUtil#loginKerberos(Configuration, String, String)}, which is used by this
 * class to authenticate a principal with Kerberos, HBase controller services no longer
 * attempt relogins explicitly.  For more information, please read the documentation for
 * {@link SecurityUtil#loginKerberos(Configuration, String, String)}.
 * <p/>
 * In previous versions of NiFi, a {@link org.apache.nifi.hadoop.KerberosTicketRenewer} was started
 * when the HBase controller service was enabled.  The use of a separate thread to explicitly relogin could cause
 * race conditions with the implicit relogin attempts made by hadoop/HBase code on a thread that references the same
 * {@link UserGroupInformation} instance.  One of these threads could leave the
 * {@link javax.security.auth.Subject} in {@link UserGroupInformation} to be cleared or in an unexpected state
 * while the other thread is attempting to use the {@link javax.security.auth.Subject}, resulting in failed
 * authentication attempts that would leave the HBase controller service in an unrecoverable state.
 *
 * @see SecurityUtil#loginKerberos(Configuration, String, String)
 */
@OnEnabled
public void onEnabled(final ConfigurationContext context) throws InitializationException, IOException, InterruptedException {
    this.connection = createConnection(context);

    // connection check
    if (this.connection != null) {
        final Admin admin = this.connection.getAdmin();
        if (admin != null) {
            admin.listTableNames();

            final ClusterStatus clusterStatus = admin.getClusterStatus();
            if (clusterStatus != null) {
                final ServerName master = clusterStatus.getMaster();
                if (master != null) {
                    masterAddress = master.getHostAndPort();
                } else {
                    masterAddress = null;
                }
            }
        }
    }
}
 

private void init(RegionLocator regionLocator, Admin admin)
    throws IOException {
  if (!enabled(admin.getConfiguration())) {
    LOG.info("Region size calculation disabled.");
    return;
  }

  LOG.info("Calculating region sizes for table \"" + regionLocator.getName() + "\".");

  //get regions for table
  List<HRegionLocation> tableRegionInfos = regionLocator.getAllRegionLocations();
  Set<byte[]> tableRegions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
  for (HRegionLocation regionInfo : tableRegionInfos) {
    tableRegions.add(regionInfo.getRegionInfo().getRegionName());
  }

  ClusterStatus clusterStatus = admin.getClusterStatus();
  Collection<ServerName> servers = clusterStatus.getServers();
  final long megaByte = 1024L * 1024L;

  //iterate all cluster regions, filter regions from our table and compute their size
  for (ServerName serverName: servers) {
    ServerLoad serverLoad = clusterStatus.getLoad(serverName);

    for (RegionLoad regionLoad: serverLoad.getRegionsLoad().values()) {
      byte[] regionId = regionLoad.getName();

      if (tableRegions.contains(regionId)) {

        long regionSizeBytes = (regionLoad.getStorefileSizeMB() + regionLoad.getMemStoreSizeMB()) * megaByte;
        sizeMap.put(regionId, regionSizeBytes);

        if (LOG.isDebugEnabled()) {
          LOG.debug("Region " + regionLoad.getNameAsString() + " has size " + regionSizeBytes);
        }
      }
    }
  }
  LOG.debug("Region sizes calculated");
}
 

/**
 * Computes size of each region for table and given column families.
 * */
public HBaseRegionSizeCalculator(String tableName, Connection hbaseConnection) throws IOException {

    Table table = null;
    Admin admin = null;
    try {
        table = hbaseConnection.getTable(TableName.valueOf(tableName));
        admin = hbaseConnection.getAdmin();

        if (!enabled(table.getConfiguration())) {
            logger.info("Region size calculation disabled.");
            return;
        }

        logger.info("Calculating region sizes for table \"" + table.getName() + "\".");

        // Get regions for table.
        RegionLocator regionLocator = hbaseConnection.getRegionLocator(table.getName());
        List<HRegionLocation> regionLocationList = regionLocator.getAllRegionLocations();
        Set<byte[]> tableRegions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);

        for (HRegionLocation hRegionLocation : regionLocationList) {
            tableRegions.add(hRegionLocation.getRegionInfo().getRegionName());
        }

        ClusterStatus clusterStatus = admin.getClusterStatus();
        Collection<ServerName> servers = clusterStatus.getServers();
        final long megaByte = 1024L * 1024L;

        // Iterate all cluster regions, filter regions from our table and
        // compute their size.
        for (ServerName serverName : servers) {
            ServerLoad serverLoad = clusterStatus.getLoad(serverName);

            for (RegionLoad regionLoad : serverLoad.getRegionsLoad().values()) {
                byte[] regionId = regionLoad.getName();

                if (tableRegions.contains(regionId)) {

                    long regionSizeBytes = regionLoad.getStorefileSizeMB() * megaByte;
                    sizeMap.put(regionId, regionSizeBytes);
                    countMap.put(regionId, new Pair<>(regionLoad.getStores(), regionLoad.getStorefiles()));

                    if (regionSizeBytes == 0L) {
                        logger.info("Region " + regionLoad.getNameAsString() + " has size " + regionSizeBytes);
                    }
                }
            }
        }
    } finally {
        IOUtils.closeQuietly(admin);
    }

}
 

/**
 * Computes size of each region for table and given column families.
 * */
public HBaseRegionSizeCalculator(String tableName, Connection hbaseConnection) throws IOException {

    Table table = null;
    Admin admin = null;
    try {
        table = hbaseConnection.getTable(TableName.valueOf(tableName));
        admin = hbaseConnection.getAdmin();

        if (!enabled(table.getConfiguration())) {
            logger.info("Region size calculation disabled.");
            return;
        }

        logger.info("Calculating region sizes for table \"" + table.getName() + "\".");

        // Get regions for table.
        RegionLocator regionLocator = hbaseConnection.getRegionLocator(table.getName());
        List<HRegionLocation> regionLocationList = regionLocator.getAllRegionLocations();
        Set<byte[]> tableRegions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);

        for (HRegionLocation hRegionLocation : regionLocationList) {
            tableRegions.add(hRegionLocation.getRegionInfo().getRegionName());
        }

        ClusterStatus clusterStatus = admin.getClusterStatus();
        Collection<ServerName> servers = clusterStatus.getServers();
        final long megaByte = 1024L * 1024L;

        // Iterate all cluster regions, filter regions from our table and
        // compute their size.
        for (ServerName serverName : servers) {
            ServerLoad serverLoad = clusterStatus.getLoad(serverName);

            for (RegionLoad regionLoad : serverLoad.getRegionsLoad().values()) {
                byte[] regionId = regionLoad.getName();

                if (tableRegions.contains(regionId)) {

                    long regionSizeBytes = regionLoad.getStorefileSizeMB() * megaByte;
                    sizeMap.put(regionId, regionSizeBytes);
                    countMap.put(regionId, new Pair<>(regionLoad.getStores(), regionLoad.getStorefiles()));

                    if (regionSizeBytes == 0L) {
                        logger.info("Region " + regionLoad.getNameAsString() + " has size " + regionSizeBytes);
                    }
                }
            }
        }
    } finally {
        IOUtils.closeQuietly(admin);
    }

}