org.apache.hadoop.hbase.client.HConnection Java Examples

public Tailer(Configuration conf, Mongo mongo, HConnection hbase, String tailerName) {
  _conf = conf;
  _mongo = mongo;
  _hbase = hbase;
  _knownTables = new HashMap<String, HTable>();
  _translator = ConfigUtil.getTranslator(_conf);

  _stateTable = createStateTable();

  if (tailerName == null) {
    List<ServerAddress> addresses = _mongo.getAllAddress();
    tailerName = StringUtils.join(addresses, ",");
  }

  _tailerID = tailerName.getBytes();

  _skipUpdates = ConfigUtil.getSkipUpdates(_conf);
  _skipDeletes = ConfigUtil.getSkipDeletes(_conf);
  _bufferWrites = ConfigUtil.getBufferWrites(_conf);
}
 

private static void populateUserProfileData(HConnection connection) throws Exception {
  UserProfile up1 = new UserProfile();
  up1.userId = "101";
  up1.lastUpdatedTimeStamp = System.currentTimeMillis() - 1000;
  up1.historicAvg90PercentSingleDaySpend = 90.0;
  up1.historicAvgSingleDaySpend = 50.0;
  up1.todayMaxSpend = 0.0;
  up1.todayNumOfPurchases = 0l;
  HBaseUtils.populateUserProfile(connection, up1);

  up1.userId = "102";
  up1.lastUpdatedTimeStamp = System.currentTimeMillis() - 1000;
  up1.historicAvg90PercentSingleDaySpend = 90.0;
  up1.historicAvgSingleDaySpend = 50.0;
  up1.todayMaxSpend = 0.0;
  up1.todayNumOfPurchases = 0l;
  HBaseUtils.populateUserProfile(connection, up1);

  up1.userId = "103";
  up1.lastUpdatedTimeStamp = System.currentTimeMillis() - 1000;
  up1.historicAvg90PercentSingleDaySpend = 90.0;
  up1.historicAvgSingleDaySpend = 50.0;
  up1.todayMaxSpend = 0.0;
  up1.todayNumOfPurchases = 0l;
  HBaseUtils.populateUserProfile(connection, up1);
}
 

@Override
public boolean initializeAdapter() {

	// Initialize HBase Table
	Configuration conf = null;
	conf = HBaseConfiguration.create();
	conf.set("hbase.zookeeper.quorum", _quorum);
	conf.set("hbase.zookeeper.property.clientPort", _port);

	try {
		LOGGER.debug("=======Connecting to HBASE===========");
		LOGGER.debug("=======ZOOKEEPER = "
				+ conf.get("hbase.zookeeper.quorum"));
		HConnection connection = HConnectionManager.createConnection(conf);
		table = connection.getTable(_tableName);
		return true;
	} catch (IOException e) {
		// TODO Auto-generated catch block
		LOGGER.debug("=======Unable to Connect to HBASE===========");
		e.printStackTrace();
	}

	return false;
}
 

public static void testGridTable(double hitRatio, double indexRatio) throws IOException {
    System.out.println("Testing grid table scanning, hit ratio " + hitRatio + ", index ratio " + indexRatio);
    String hbaseUrl = "hbase"; // use hbase-site.xml on classpath

    HConnection conn = HBaseConnection.get(hbaseUrl);
    createHTableIfNeeded(conn, TEST_TABLE);
    prepareData(conn);

    Hits hits = new Hits(N_ROWS, hitRatio, indexRatio);

    for (int i = 0; i < ROUND; i++) {
        System.out.println("==================================== ROUND " + (i + 1) + " ========================================");
        testRowScanWithIndex(conn, hits.getHitsForRowScanWithIndex());
        testRowScanNoIndexFullScan(conn, hits.getHitsForRowScanNoIndex());
        testRowScanNoIndexSkipScan(conn, hits.getHitsForRowScanNoIndex());
        testColumnScan(conn, hits.getHitsForColumnScan());
    }

}
 

@Override
public boolean initializeAdapter() {

	// Initialize HBase Table
	Configuration conf = null;
	conf = HBaseConfiguration.create();
	conf.set("hbase.zookeeper.quorum", _quorum);
	conf.set("hbase.zookeeper.property.clientPort", _port);

	try {
		LOGGER.debug("=======Connecting to HBASE===========");
		LOGGER.debug("=======ZOOKEEPER = "
				+ conf.get("hbase.zookeeper.quorum"));
		HConnection connection = HConnectionManager.createConnection(conf);
		table = connection.getTable(_tableName);
		return true;
	} catch (IOException e) {
		// TODO Auto-generated catch block
		LOGGER.debug("=======Unable to Connect to HBASE===========");
		e.printStackTrace();
	}

	return false;
}
 

private static void fullScan(HConnection conn, boolean[] hits, Stats stats) throws IOException {
    HTableInterface table = conn.getTable(TEST_TABLE);
    try {
        stats.markStart();

        Scan scan = new Scan();
        scan.addFamily(CF);
        ResultScanner scanner = table.getScanner(scan);
        int i = 0;
        for (Result r : scanner) {
            if (hits[i])
                stats.consume(r);
            dot(i, N_ROWS);
            i++;
        }

        stats.markEnd();
    } finally {
        IOUtils.closeQuietly(table);
    }
}
 

public SerializedHBaseTupleIterator(HConnection conn, List<HBaseKeyRange> segmentKeyRanges, CubeInstance cube, Collection<TblColRef> dimensions, TupleFilter filter, Collection<TblColRef> groupBy, Collection<RowValueDecoder> rowValueDecoders, StorageContext context) {

        this.context = context;
        int limit = context.getLimit();
        this.partialResultLimit = Math.max(limit, PARTIAL_DEFAULT_LIMIT);

        this.segmentIteratorList = new ArrayList<CubeSegmentTupleIterator>(segmentKeyRanges.size());
        Map<CubeSegment, List<HBaseKeyRange>> rangesMap = makeRangesMap(segmentKeyRanges);
        for (Map.Entry<CubeSegment, List<HBaseKeyRange>> entry : rangesMap.entrySet()) {
            CubeSegmentTupleIterator segIter = new CubeSegmentTupleIterator(entry.getKey(), entry.getValue(), conn, dimensions, filter, groupBy, rowValueDecoders, context);
            this.segmentIteratorList.add(segIter);
        }

        this.segmentIteratorIterator = this.segmentIteratorList.iterator();
        if (this.segmentIteratorIterator.hasNext()) {
            this.segmentIterator = this.segmentIteratorIterator.next();
        } else {
            this.segmentIterator = ITupleIterator.EMPTY_TUPLE_ITERATOR;
        }
    }
 

public void setup() throws IOException {

        KylinConfig.destoryInstance();
        System.setProperty(KylinConfig.KYLIN_CONF, AbstractKylinTestCase.SANDBOX_TEST_DATA);

        kylinConfig = KylinConfig.getInstanceFromEnv();
        cli = kylinConfig.getCliCommandExecutor();

        String metadataUrl = kylinConfig.getMetadataUrl();
        // split TABLE@HBASE_URL
        int cut = metadataUrl.indexOf('@');
        tableNameBase = metadataUrl.substring(0, cut);
        String hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);

        HConnection conn = HBaseConnection.get(hbaseUrl);
        try {
            hbase = new HBaseAdmin(conn);
            config = hbase.getConfiguration();
            //allTables = hbase.listTables();
        } catch (IOException e) {
            e.printStackTrace();
            throw e;
        }

        uploadTarballToRemote();
    }
 

public CubeSegmentTupleIterator(CubeSegment cubeSeg, Collection<HBaseKeyRange> keyRanges, HConnection conn, Collection<TblColRef> dimensions, TupleFilter filter, Collection<TblColRef> groupBy, Collection<RowValueDecoder> rowValueDecoders, StorageContext context) {
    this.cube = cubeSeg.getCubeInstance();
    this.cubeSeg = cubeSeg;
    this.dimensions = dimensions;
    this.filter = filter;
    this.groupBy = groupBy;
    this.rowValueDecoders = rowValueDecoders;
    this.context = context;
    this.tableName = cubeSeg.getStorageLocationIdentifier();
    this.rowKeyDecoder = new RowKeyDecoder(this.cubeSeg);
    this.scanCount = 0;

    try {
        this.table = conn.getTable(tableName);
    } catch (Throwable t) {
        throw new StorageException("Error when open connection to table " + tableName, t);
    }
    this.rangeIterator = keyRanges.iterator();
    scanNextRange();
}
 

public static HConnection get(String url) {
    // find configuration
    Configuration conf = ConfigCache.get(url);
    if (conf == null) {
        conf = HadoopUtil.newHBaseConfiguration(url);
        ConfigCache.put(url, conf);
    }

    HConnection connection = ConnPool.get(url);
    try {
        // I don't use DCL since recreate a connection is not a big issue.
        if (connection == null) {
            connection = HConnectionManager.createConnection(conf);
            ConnPool.put(url, connection);
        }
    } catch (Throwable t) {
        throw new StorageException("Error when open connection " + url, t);
    }

    return connection;
}
 

public synchronized static void waitForConnection(long timeout, TimeUnit timeoutUnit) {
    long before = System.currentTimeMillis();
    long after;
    long timeoutMS = TimeUnit.MILLISECONDS.convert(timeout, timeoutUnit);
    do {
        try {
            HConnection hc = HConnectionManager.createConnection(HBaseConfiguration.create());
            hc.close();
            after = System.currentTimeMillis();
            log.info("HBase server to started after about {} ms", after - before);
            return;
        } catch (IOException e) {
            log.info("Exception caught while waiting for the HBase server to start", e);
        }
        after = System.currentTimeMillis();
    } while (timeoutMS > after - before);
    after = System.currentTimeMillis();
    log.warn("HBase server did not start in {} ms", after - before);
}
 

public HConnection openConn() {
	if (null == hconn) {
		try {
			this.hconn = HConnectionManager.createConnection(config);
		} catch (Exception ex) {
			ex.printStackTrace();
		}
	}
	return hconn;
}
 

public synchronized HConnection getConn() {
	if (null == conn) {
		try {
			this.conn = HConnectionManager.createConnection(config);
		} catch (Exception ex) {
			LOGGER.error("create conn err:", ex);
		}
	}
	return conn;
}
 

private static void testColumnScan(HConnection conn, List<Pair<Integer, Integer>> colScans) throws IOException {
    Stats stats = new Stats("COLUMN_SCAN");

    HTableInterface table = conn.getTable(TEST_TABLE);
    try {
        stats.markStart();

        int nLogicCols = colScans.size();
        int nLogicRows = colScans.get(0).getSecond() - colScans.get(0).getFirst();
        
        Scan[] scans = new Scan[nLogicCols];
        ResultScanner[] scanners = new ResultScanner[nLogicCols];
        for (int i = 0; i < nLogicCols; i++) {
            scans[i] = new Scan();
            scans[i].addFamily(CF);
            scanners[i] = table.getScanner(scans[i]);
        }
        for (int i = 0; i < nLogicRows; i++) {
            for (int c = 0; c < nLogicCols; c++) {
                Result r = scanners[c].next();
                stats.consume(r);
            }
            dot(i, nLogicRows);
        }
        
        stats.markEnd();
    } finally {
        IOUtils.closeQuietly(table);
    }
}
 

public EmptyRegionChecker(HConnection connection, String tableName,
                          HRegionInfo regionInfo, Set<HRegionInfo> emptyRegions) {
    this.connection = connection;
    this.tableName = tableName;
    this.regionInfo = regionInfo;
    this.emptyRegions = emptyRegions;
}
 

public synchronized HConnection getConn() {
	if (null == conn) {
		try {
			this.conn = HConnectionManager.createConnection(this.config);
		} catch (Exception ex) {
			LOGGER.error("create conn err:", ex);
		}
	}
	return conn;
}
 

public EmptyRegionChecker(HConnection connection, String tableName,
                          HRegionInfo regionInfo, Set<HRegionInfo> emptyRegions) {
    this.connection = connection;
    this.tableName = tableName;
    this.regionInfo = regionInfo;
    this.emptyRegions = emptyRegions;
}
 

public EmptyRegionChecker(HConnection connection, String tableName,
                          HRegionInfo regionInfo, Set<HRegionInfo> emptyRegions) {
    this.connection = connection;
    this.tableName = tableName;
    this.regionInfo = regionInfo;
    this.emptyRegions = emptyRegions;
}
 

private static void populateValidationRules(HConnection connection) throws Exception {
  HashSet<String> banndedVandors = new HashSet<String>();
  banndedVandors.add("badVendor");

  ValidationRules rules = new ValidationRules(banndedVandors, 2.0);

  HBaseUtils.populateValidationRules(connection, rules);
}
 

public EmptyRegionChecker(HConnection connection, String tableName,
                          HRegionInfo regionInfo, Set<HRegionInfo> emptyRegions) {
    this.connection = connection;
    this.tableName = tableName;
    this.regionInfo = regionInfo;
    this.emptyRegions = emptyRegions;
}
 

public static void populateUserProfile(HConnection connection, UserProfile userProfile) throws Exception {
  HTableInterface table = connection.getTable(HBaseTableMetaModel.profileCacheTableName);

  try {
    Put put = new Put(convertKeyToRowKey(HBaseTableMetaModel.profileCacheTableName, userProfile.userId));
    put.add(HBaseTableMetaModel.profileCacheColumnFamily, HBaseTableMetaModel.profileCacheJsonColumn, Bytes.toBytes(userProfile.getJSONObject().toString()));
    put.add(HBaseTableMetaModel.profileCacheColumnFamily, HBaseTableMetaModel.profileCacheTsColumn, Bytes.toBytes(System.currentTimeMillis()));
    table.put(put);
  } finally {
    table.close();
  }
}
 

public static void populateValidationRules(HConnection connection, ValidationRules rules) throws Exception {
  HTableInterface table = connection.getTable(HBaseTableMetaModel.profileCacheTableName);

  try {
    Put put = new Put(HBaseTableMetaModel.validationRulesRowKey);
    put.add(HBaseTableMetaModel.profileCacheColumnFamily, HBaseTableMetaModel.validationRulesRowKey, Bytes.toBytes(rules.getJSONObject().toString()));
    table.put(put);
  } finally {
    table.close();
  }
}
 

public  HConnection getHConnection() {
    HConnection con = null;
    try {
        if (null == con || con.isClosed()) {
            con = HConnectionManager.createConnection(config);
        }
    } catch (IOException e) {
        logger.error("Cannot get HBase connection.", e.getCause());
    }
    return con;
}
 

public boolean initializeAdapter() {
	Configuration conf = null;
	conf = HBaseConfiguration.create();
	conf.set("hbase.zookeeper.quorum", _quorum);
	conf.set("hbase.zookeeper.property.clientPort", _port);
	conf.set("zookeeper.session.timeout", "20");
	conf.set("hbase.rpc.timeout", "20");
	conf.set("zookeeper.recovery.retry", "1");
	conf.set("zookeeper.recovery.retry.intervalmill", "1");

	try {

		LOG.trace("[OpenSOC] Connecting to HBase");
		LOG.trace("[OpenSOC] ZOOKEEPER = "
				+ conf.get("hbase.zookeeper.quorum"));

		LOG.trace("[OpenSOC] CONNECTING TO HBASE WITH: " + conf);

		HConnection connection = HConnectionManager.createConnection(conf);

		LOG.trace("[OpenSOC] CONNECTED TO HBASE");

		table = connection.getTable(_table_name);

		LOG.trace("--------CONNECTED TO TABLE: " + table);

		JSONObject tester = enrich("cisco.com");

		if (tester.keySet().size() == 0)
			throw new IOException(
					"Either HBASE is misconfigured or whois table is missing");

		return true;
	} catch (IOException e) {
		e.printStackTrace();
	}

	return false;

}
 

public HBaseClientKVIterator(HConnection hconn, String tableName, byte[] family, byte[] qualifier) throws IOException {
    this.family = family;
    this.qualifier = qualifier;

    this.table = hconn.getTable(tableName);
    this.scanner = table.getScanner(family, qualifier);
    this.iterator = scanner.iterator();
}
 

private static void createHTableIfNeeded(HConnection conn, String tableName) throws IOException {
    HBaseAdmin hbase = new HBaseAdmin(conn);

    try {
        boolean tableExist = false;
        try {
            hbase.getTableDescriptor(TableName.valueOf(tableName));
            tableExist = true;
        } catch (TableNotFoundException e) {
        }

        if (tableExist) {
            System.out.println("HTable '" + tableName + "' already exists");
            return;
        }

        System.out.println("Creating HTable '" + tableName + "'");

        HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));

        HColumnDescriptor fd = new HColumnDescriptor(CF);
        fd.setBlocksize(CELL_SIZE);
        desc.addFamily(fd);
        hbase.createTable(desc);

        System.out.println("HTable '" + tableName + "' created");
    } finally {
        hbase.close();
    }
}
 

private void setup() throws IOException {
    long currentTIME = System.currentTimeMillis();
    exportFolder = "/tmp/hbase-export/" + currentTIME + "/";
    backupArchive = "/tmp/kylin_" + currentTIME + ".tar.gz";

    KylinConfig.destoryInstance();
    System.setProperty(KylinConfig.KYLIN_CONF, AbstractKylinTestCase.SANDBOX_TEST_DATA);

    kylinConfig = KylinConfig.getInstanceFromEnv();
    cli = kylinConfig.getCliCommandExecutor();

    String metadataUrl = kylinConfig.getMetadataUrl();
    // split TABLE@HBASE_URL
    int cut = metadataUrl.indexOf('@');
    tableNameBase = metadataUrl.substring(0, cut);
    String hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);

    HConnection conn = HBaseConnection.get(hbaseUrl);
    try {
        hbase = new HBaseAdmin(conn);
        config = hbase.getConfiguration();
        allTables = hbase.listTables();
    } catch (IOException e) {
        e.printStackTrace();
        throw e;
    }
}
 

@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest) {
    String tableName = seg.getStorageLocationIdentifier();

    //HConnection is cached, so need not be closed
    HConnection conn = HBaseConnection.get(context.getConnUrl());
    try {
        return new EndpointTupleIterator(seg, sqlDigest.filter, sqlDigest.groupbyColumns, new ArrayList<>(sqlDigest.aggregations), context, conn);
    } catch (Throwable e) {
        e.printStackTrace();
        throw new IllegalStateException("Error when connecting to II htable " + tableName, e);
    }
}
 

@Override
public void run() {
    for (HConnection conn : ConnPool.values()) {
        try {
            conn.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}
 

public static void createHTableIfNeeded(HConnection conn, String tableName, String... families) throws IOException {
    HBaseAdmin hbase = new HBaseAdmin(conn);

    try {
        boolean tableExist = false;
        try {
            hbase.getTableDescriptor(TableName.valueOf(tableName));
            tableExist = true;
        } catch (TableNotFoundException e) {
        }

        if (tableExist) {
            logger.debug("HTable '" + tableName + "' already exists");
            return;
        }

        logger.debug("Creating HTable '" + tableName + "'");

        HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));

        if (null != families && families.length > 0) {
            for (String family : families) {
                HColumnDescriptor fd = new HColumnDescriptor(family);
                fd.setInMemory(true); // metadata tables are best in memory
                desc.addFamily(fd);
            }
        }
        hbase.createTable(desc);

        logger.debug("HTable '" + tableName + "' created");
    } finally {
        hbase.close();
    }
}