Java Code Examples for org.apache.hadoop.hbase.client.Table#getConfiguration()

private void verifyTableCount(Partition partition, int count) throws Exception {
    Table htable = ((ClientPartition) partition).unwrapDelegate();
    int i = 0;
    List<Cell> newCells = new ArrayList<>();
    Scan scan = new Scan();
    SplitRegionScanner srs = new SplitRegionScanner(scan,
            htable,
            clock,partition, driver.getConfiguration(), htable.getConfiguration());
    while (srs.next(newCells)) {
        i++;
        newCells.clear();
    }
    srs.close();
    htable.close();
    Assert.assertEquals("Did not return all rows ", count, i);
}
 

private void verifyTableSample(Partition partition, int count) throws Exception {
    Table htable = ((ClientPartition) partition).unwrapDelegate();
    int i = 0;
    List<Cell> newCells = new ArrayList<>();
    Scan scan = new Scan();
    scan.setFilter(new SamplingFilter(0.2));
    SplitRegionScanner srs = new SplitRegionScanner(scan,
            htable,
            clock,partition, driver.getConfiguration(), htable.getConfiguration());
    while (srs.next(newCells)) {
        i++;
        newCells.clear();
    }
    srs.close();
    htable.close();

    Assert.assertTrue("Returned more rows than expected: " + i, i < (count * 0.35));
    Assert.assertTrue("Returned less rows than expected: " + i, i > (count * 0.05));
}
 

@Test
public void simpleScan() throws SQLException, IOException, InterruptedException {
       Partition partition = driver.getTableFactory()
               .getTable(sqlUtil.getConglomID(spliceTableWatcherA.toString()));
       Table htable = ((ClientPartition) partition).unwrapDelegate();
       List<Partition> partitions = partition.subPartitions();
       int i = 0;
       List<Cell> newCells = new ArrayList<>();
       simpleScan = new TreeSet();
       for (Partition subPartition: partitions){
           Scan scan = new Scan(subPartition.getStartKey(),subPartition.getEndKey());
           SplitRegionScanner srs = new SplitRegionScanner(scan,
                   htable,
                   clock,subPartition, driver.getConfiguration(), htable.getConfiguration());
           while (srs.next(newCells)) {
               i++;
               simpleScan.add(CellUtils.toHex(CellUtil.cloneRow(newCells.get(0))));
               newCells.clear();
           }
           srs.close();
       }
       htable.close();
       Assert.assertEquals("Did not return all rows ",ITERATIONS,i);
}
 

@Test
public void simpleScanPostFlush() throws SQLException, IOException, InterruptedException {
    Partition partition = driver.getTableFactory()
            .getTable(sqlUtil.getConglomID(spliceTableWatcherB.toString()));
    Table htable = ((ClientPartition) partition).unwrapDelegate();
    List<Partition> partitions = partition.subPartitions();
    int i = 0;
    List<Cell> newCells = new ArrayList<>();
    partition.flush();
    for (Partition subPartition: partitions){
        Scan scan = new Scan(subPartition.getStartKey(),subPartition.getEndKey());
        SplitRegionScanner srs = new SplitRegionScanner(scan,
                htable,
                clock,subPartition, driver.getConfiguration(), htable.getConfiguration());
        while (srs.next(newCells)) {
            i++;
            newCells.clear();
        }
        srs.close();
    }
    htable.close();
    Assert.assertEquals("Did not return all rows ",ITERATIONS,i);


}
 

@Test
public void simpleScanPostSplit() throws SQLException, IOException, InterruptedException {
    String tableName = sqlUtil.getConglomID(spliceTableWatcherC.toString());
    Partition partition = driver.getTableFactory()
            .getTable(tableName);
    Table htable = ((ClientPartition) partition).unwrapDelegate();
    int i = 0;
    driver.getTableFactory().getAdmin().splitTable(tableName);
    List<Cell> newCells = new ArrayList<>();
    Scan scan = new Scan();
    SplitRegionScanner srs = new SplitRegionScanner(scan,
            htable,
            clock,partition, driver.getConfiguration(), htable.getConfiguration());
    while (srs.next(newCells)) {
        i++;
        newCells.clear();
    }
    srs.close();
    htable.close();
    Assert.assertEquals("Did not return all rows ",ITERATIONS,i);
}
 

@Test
public void sampledScanWithConcurrentFlushAndSplitTest() throws Exception {

    String tableName = sqlUtil.getConglomID(spliceTableWatcherK.toString());
    Partition partition = driver.getTableFactory()
            .getTable(tableName);
    Table htable = ((ClientPartition) partition).unwrapDelegate();
    List<Partition> partitions = partition.subPartitions();
    int i = 0;
    List<Cell> newCells = new ArrayList<>();
    for (Partition subPartition: partitions){
        Scan scan = new Scan(subPartition.getStartKey(),subPartition.getEndKey());
        scan.setFilter(new SamplingFilter(0.2)); // enable sampling for this scan
        SplitRegionScanner srs = new SplitRegionScanner(scan,
                htable,
                clock,subPartition, driver.getConfiguration(), htable.getConfiguration());
        while (srs.next(newCells)) {
            i++;
            if (i==(ITERATIONS*0.2)/2) {
                partition.flush();
                driver.getTableFactory().getAdmin().splitTable(tableName);
            }
            newCells.clear();
        }
        srs.close();
    }
    htable.close();

    Assert.assertTrue("Returned more rows than expected: " + i, i < (ITERATIONS * 0.35));
    Assert.assertTrue("Returned less rows than expected: " + i, i > (ITERATIONS * 0.05));
}
 

/**
 * @param htable the {@link org.apache.hadoop.hbase.HTableDescriptor} to scan.
 */
public void setHTable(Table htable) {
  Configuration conf = htable.getConfiguration();
  logScannerActivity = conf.getBoolean(
    ConnectionConfiguration.LOG_SCANNER_ACTIVITY, false);
  logPerRowCount = conf.getInt(LOG_PER_ROW_COUNT, 100);
  this.htable = htable;
}
 

@Test
public void moveRegionDuringScan() throws SQLException, IOException, InterruptedException {
    String tableNameStr = sqlUtil.getConglomID(spliceTableWatcherI.toString());
    Partition partition = driver.getTableFactory().getTable(tableNameStr);
    Table htable = ((ClientPartition) partition).unwrapDelegate();

    int i = 0;
    try (Admin admin = connection.getAdmin()) {
        HRegionLocation regionLocation = getRegionLocation(tableNameStr, admin);
        Collection<ServerName> allServers = getAllServers(admin);
        ServerName newServer = getNotIn(allServers, regionLocation.getServerName());

        List<Cell> newCells = new ArrayList<>();
        Scan scan = new Scan();
        SplitRegionScanner srs = new SplitRegionScanner(scan, htable, clock, partition, driver.getConfiguration(), htable.getConfiguration());
        while (srs.next(newCells)) {
            if (i++ == ITERATIONS / 2) {
                // JL - TODO Now that we are blocking moves, this hangs.
               // admin.move(regionLocation.getRegionInfo().getEncodedNameAsBytes(), Bytes.toBytes(newServer.getServerName()));
            }
            newCells.clear();
        }
        srs.close();
        htable.close();
    }
    Assert.assertEquals("Did not return all rows ", ITERATIONS, i);
}
 

@Test
public void testSplitRegionScannerReinit() throws Exception {
    String tableName = sqlUtil.getConglomID(spliceTableWatcherH.toString());
    Partition partition = driver.getTableFactory()
            .getTable(tableName);
    partition.subPartitions(); // Grabs the latest partitions in cache... / Key to force a split issue
    partition.flush();
    driver.getTableFactory().getAdmin().splitTable(tableName);
    Thread.sleep(2000);
    driver.getTableFactory().getAdmin().splitTable(tableName);
    Thread.sleep(2000);
    Table htable = ((ClientPartition) partition).unwrapDelegate();
    int i = 0;
    driver.getTableFactory().getAdmin().splitTable(tableName);
    List<Cell> newCells = new ArrayList<>();
    Scan scan = new Scan();
    SplitRegionScanner srs = new SplitRegionScanner(scan,
            htable,
            clock,partition, driver.getConfiguration(), htable.getConfiguration());
    while (srs.next(newCells)) {
        i++;
        if (i==ITERATIONS/2) {
            driver.getTableFactory().getAdmin().splitTable(tableName);
        }
        Assert.assertTrue("Empty Cells Should Not Be Returned",newCells!=null&&!newCells.isEmpty());
        newCells.clear();
    }
    srs.close();
    htable.close();
    Assert.assertEquals("Did not return all rows ",ITERATIONS,i);
}
 

@Test
public void multipleSplits() throws Exception {
    spliceClassWatcher.executeUpdate(String.format("insert into %s select col1+" + ITERATIONS+", col2 from %s"
            ,SCHEMA + ".G",SCHEMA + ".A"));
    String tableName = sqlUtil.getConglomID(spliceTableWatcherG.toString());
    Partition partition = driver.getTableFactory()
            .getTable(tableName);
    partition.flush();
    driver.getTableFactory().getAdmin().splitTable(tableName);
    Thread.sleep(2000);
    driver.getTableFactory().getAdmin().splitTable(tableName);
    Table htable = ((ClientPartition) partition).unwrapDelegate();
    int i = 0;
    driver.getTableFactory().getAdmin().splitTable(tableName);
    List<Cell> newCells = new ArrayList<>();
        Scan scan = new Scan();
        SplitRegionScanner srs = new SplitRegionScanner(scan,
                htable,
                clock,partition, driver.getConfiguration(), htable.getConfiguration());
        while (srs.next(newCells)) {
            i++;
            if (i==ITERATIONS/2) {
                driver.getTableFactory().getAdmin().splitTable(tableName);
            }
            Assert.assertTrue("Empty Cells Should Not Be Returned",newCells!=null&&!newCells.isEmpty());
            newCells.clear();
        }
        srs.close();
    htable.close();
    Assert.assertEquals("Did not return all rows ",2*ITERATIONS,i);
}
 

@Test
public void simpleMergeWithConcurrentFlushAndSplitTest() throws Exception {
    // Currently fails with:
    // org.apache.hadoop.hbase.DoNotRetryIOException: org.apache.hadoop.hbase.DoNotRetryIOException
    // at com.splicemachine.hbase.MemstoreAwareObserver.preStoreScannerOpen(BaseMemstoreAwareObserver.java:159)
    // at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1291)

    String tableName = sqlUtil.getConglomID(spliceTableWatcherJ.toString());
    Partition partition = driver.getTableFactory()
                                .getTable(tableName);
    Table htable = ((ClientPartition) partition).unwrapDelegate();
    List<Partition> partitions = partition.subPartitions();
    int i = 0;
    List<Cell> newCells = new ArrayList<>();
    for (Partition subPartition: partitions){
        Scan scan = new Scan(subPartition.getStartKey(),subPartition.getEndKey());
        SplitRegionScanner srs = new SplitRegionScanner(scan,
                                                        htable,
                                                        clock,subPartition, driver.getConfiguration(), htable.getConfiguration());
        while (srs.next(newCells)) {
            i++;
            if (i==ITERATIONS/2) {
                partition.flush();
                driver.getTableFactory().getAdmin().splitTable(tableName);
            }
            newCells.clear();
        }
        srs.close();
    }
    htable.close();
    Assert.assertEquals("Did not return all rows ",ITERATIONS,i);
}
 

@Test
public void simpleMergeWithConcurrentFlushTest() throws Exception {
    spliceClassWatcher.executeUpdate(String.format("insert into %s select col1+" + ITERATIONS+", col2 from %s"
            ,SCHEMA + ".F",SCHEMA + ".A"));
    String tableName = sqlUtil.getConglomID(spliceTableWatcherF.toString());
    Partition partition = driver.getTableFactory()
            .getTable(tableName);
    Table htable = ((ClientPartition) partition).unwrapDelegate();
    List<Partition> partitions = partition.subPartitions();
    int i = 0;
    driver.getTableFactory().getAdmin().splitTable(tableName);
    List<Cell> newCells = new ArrayList<>();
    for (Partition subPartition: partitions){
        Scan scan = new Scan(subPartition.getStartKey(),subPartition.getEndKey());
        SplitRegionScanner srs = new SplitRegionScanner(scan,
                htable,
                clock,subPartition, driver.getConfiguration(), htable.getConfiguration());
        while (srs.next(newCells)) {
            i++;
            if (i==ITERATIONS/2)
                partition.flush();
            newCells.clear();
        }
        srs.close();
    }
    htable.close();
    Assert.assertEquals("Did not return all rows ",2*ITERATIONS,i);
}
 

@Test
public void simpleMergeWithConcurrentSplitTest() throws Exception {
    spliceClassWatcher.executeUpdate(String.format("insert into %s select col1+" + ITERATIONS+", col2 from %s"
            ,SCHEMA + ".E",SCHEMA + ".A"));
    String tableName = sqlUtil.getConglomID(spliceTableWatcherE.toString());
    Partition partition = driver.getTableFactory()
            .getTable(tableName);
    Table htable = ((ClientPartition) partition).unwrapDelegate();
    List<Partition> partitions = partition.subPartitions();
    int i = 0;
    driver.getTableFactory().getAdmin().splitTable(tableName);
    List<Cell> newCells = new ArrayList<>();
    for (Partition subPartition: partitions){
        Scan scan = new Scan(subPartition.getStartKey(),subPartition.getEndKey());
        SplitRegionScanner srs = new SplitRegionScanner(scan,
                htable,
                clock,subPartition, driver.getConfiguration(), htable.getConfiguration());
        while (srs.next(newCells)) {
            i++;
            if (i==ITERATIONS/2) {
                spliceClassWatcher.executeUpdate(
                        String.format("call syscs_util.syscs_perform_major_compaction_on_table('%s', '%s')",
                                SCHEMA, "E"));
                driver.getTableFactory().getAdmin().splitTable(tableName);
            }
            newCells.clear();
        }
        srs.close();
    }
    htable.close();
    Assert.assertEquals("Did not return all rows ",2*ITERATIONS,i);
}
 

@Test
public void simpleMergeTest() throws Exception {
    // Test Records at end
    spliceClassWatcher.executeUpdate(String.format("insert into %s select col1+" + ITERATIONS+", col2 from %s"
            ,SCHEMA + ".D",SCHEMA + ".A"));
    spliceClassWatcher.executeUpdate(String.format("insert into %s values (-1,'foo')" // Test A Record before
            ,SCHEMA + ".D"));

    String tableName = sqlUtil.getConglomID(spliceTableWatcherD.toString());
    Partition partition = driver.getTableFactory()
            .getTable(tableName);
    Table htable = ((ClientPartition) partition).unwrapDelegate();
    List<Partition> partitions = partition.subPartitions();
    int i = 0;
    driver.getTableFactory().getAdmin().splitTable(tableName);
    List<Cell> newCells = new ArrayList<>();
    for (Partition subPartition: partitions){
        Scan scan = new Scan(subPartition.getStartKey(),subPartition.getEndKey());
        SplitRegionScanner srs = new SplitRegionScanner(scan,
                htable,
                clock,subPartition, driver.getConfiguration(), htable.getConfiguration());
        while (srs.next(newCells)) {
            i++;
            newCells.clear();
        }
        srs.close();
    }
    htable.close();
    Assert.assertEquals("Did not return all rows ",2*ITERATIONS+1,i);
}
 

@Test
@Ignore
public void validateAccurateRecordsWithRegionFlush() throws SQLException, IOException, InterruptedException{
    int i=0;
    TableName tableName=TableName.valueOf(sqlUtil.getConglomID(SCHEMA_NAME+".A"));
    try (Admin admin = connection.getAdmin()) {
        Table table = connection.getTable(tableName);
        Scan scan = new Scan();
        scan.setCaching(50);
        scan.setBatch(50);
        scan.setMaxVersions();
        scan.setAttribute(MRConstants.SPLICE_SCAN_MEMSTORE_ONLY, HConstants.EMPTY_BYTE_ARRAY);

        try (SkeletonClientSideRegionScanner clientSideRegionScanner =
                   new HBaseClientSideRegionScanner(table,
                         table.getConfiguration(), FSUtils.getCurrentFileSystem(table.getConfiguration()),
                         FSUtils.getRootDir(table.getConfiguration()),
                         table.getTableDescriptor(),
                         connection.getRegionLocator(tableName).getRegionLocation(scan.getStartRow()).getRegionInfo(),
                         scan,
                         connection.getRegionLocator(tableName).getRegionLocation(scan.getStartRow()).getHostnamePort())) {
            List results = new ArrayList();
            while (clientSideRegionScanner.nextRaw(results)) {
                i++;
                if (i == 100)
                    admin.flush(tableName);
                results.clear();
            }
        }
        Assert.assertEquals("Results Returned Are Not Accurate", 500, i);
    }
}
 

@Test
@Ignore
public void validateAccurateRecordsWithStoreFileAndMemstore() throws SQLException, IOException, InterruptedException{
    int i=0;
    TableName tableName=TableName.valueOf(sqlUtil.getConglomID(SCHEMA_NAME+".A"));
    try(Admin admin = connection.getAdmin()) {
        Table table = connection.getTable(tableName);
        Scan scan=new Scan();
        scan.setCaching(50);
        scan.setBatch(50);
        scan.setMaxVersions();
        scan.setAttribute(MRConstants.SPLICE_SCAN_MEMSTORE_ONLY,HConstants.EMPTY_BYTE_ARRAY);
        try(SkeletonClientSideRegionScanner clientSideRegionScanner=
                    new HBaseClientSideRegionScanner(table,
                          table.getConfiguration(), FSUtils.getCurrentFileSystem(table.getConfiguration()),
                          FSUtils.getRootDir(table.getConfiguration()),
                          table.getTableDescriptor(),
                          connection.getRegionLocator(tableName).getRegionLocation(scan.getStartRow()).getRegionInfo(),
                          scan,
                          connection.getRegionLocator(tableName).getRegionLocation(scan.getStartRow()).getHostnamePort())){
            List results=new ArrayList();
            while(clientSideRegionScanner.nextRaw(results)){
                i++;
                results.clear();
            }
        }
        Assert.assertEquals("Results Returned Are Not Accurate",500,i);
    }
}
 

/**
 * Sets the HBase table.
 *
 * @param htable  The {@link org.apache.hadoop.hbase.HTableDescriptor} to scan.
 */
public void setHTable(Table htable) {
  Configuration conf = htable.getConfiguration();
  logScannerActivity = conf.getBoolean(
    ConnectionConfiguration.LOG_SCANNER_ACTIVITY, false);
  logPerRowCount = conf.getInt(LOG_PER_ROW_COUNT, 100);
  this.htable = htable;
}