org.apache.phoenix.hbase.index.write.IndexWriterUtils Java Examples

@BeforeClass
public static synchronized void doSetup() throws Exception {
    Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(10);
    serverProps.put("hbase.coprocessor.region.classes", IndexTableFailingRegionObserver.class.getName());
    serverProps.put(Indexer.RecoveryFailurePolicyKeyForTesting, ReleaseLatchOnFailurePolicy.class.getName());
    serverProps.put(IndexWriterUtils.INDEX_WRITER_RPC_RETRIES_NUMBER, "2");
    serverProps.put(HConstants.HBASE_RPC_TIMEOUT_KEY, "10000");
    serverProps.put(IndexWriterUtils.INDEX_WRITER_RPC_PAUSE, "5000");
    serverProps.put("data.tx.snapshot.dir", "/tmp");
    serverProps.put("hbase.balancer.period", String.valueOf(Integer.MAX_VALUE));
    serverProps.put(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_ATTRIB, Boolean.FALSE.toString());
    Map<String, String> clientProps = Maps.newHashMapWithExpectedSize(2);
    clientProps.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.FALSE.toString());
    clientProps.put(QueryServices.INDEX_REGION_OBSERVER_ENABLED_ATTRIB, Boolean.FALSE.toString());
    NUM_SLAVES_BASE = 2;
    setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), new ReadOnlyProps(clientProps.entrySet().iterator()));
}
 

protected static Map<String,String> getServerProps(){
    Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(10);
    serverProps.put("hbase.coprocessor.region.classes", FailingRegionObserver.class.getName());
    serverProps.put(HConstants.HBASE_RPC_TIMEOUT_KEY, "10000");
    serverProps.put(IndexWriterUtils.INDEX_WRITER_RPC_PAUSE, "5000");
    serverProps.put("data.tx.snapshot.dir", "/tmp");
    serverProps.put("hbase.balancer.period", String.valueOf(Integer.MAX_VALUE));
    // need to override rpc retries otherwise test doesn't pass
    serverProps.put(QueryServices.INDEX_REBUILD_RPC_RETRIES_COUNTER, Long.toString(numRpcRetries));
    serverProps.put(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_FORWARD_TIME_ATTRIB, Long.toString(forwardOverlapMs));
    serverProps.put(QueryServices.INDEX_REBUILD_DISABLE_TIMESTAMP_THRESHOLD, Long.toString(disableTimestampThresholdMs));
    /*
     * Effectively disable running the index rebuild task by having an infinite delay
     * because we want to control it's execution ourselves
     */
    serverProps.put(QueryServices.INDEX_REBUILD_TASK_INITIAL_DELAY, Long.toString(Long.MAX_VALUE));
    return serverProps;
}
 

@BeforeClass
public static void doSetup() throws Exception {
  Map<String, String> props = getDefaultProps();
  props.put(IntegrationTestingUtility.IS_DISTRIBUTED_CLUSTER, "false");
  props.put(QueryServices.QUEUE_SIZE_ATTRIB, Integer.toString(5000));
  props.put(IndexWriterUtils.HTABLE_THREAD_KEY, Integer.toString(100));
  // Make a small batch size to test multiple calls to reserve sequences
  props.put(QueryServices.SEQUENCE_CACHE_SIZE_ATTRIB, Long.toString(BATCH_SIZE));
  // Must update config before starting server
  setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
}
 

@Override
public void setup(IndexWriter parent, RegionCoprocessorEnvironment env, String name) {
    this.env = env;
    Configuration conf = env.getConfiguration();
    setup(IndexWriterUtils.getDefaultDelegateHTableFactory(env),
            ThreadPoolManager.getExecutor(
                    new ThreadPoolBuilder(name, conf).setMaxThread(NUM_CONCURRENT_INDEX_WRITER_THREADS_CONF_KEY,
                            DEFAULT_CONCURRENT_INDEX_WRITER_THREADS).setCoreTimeout(
                            INDEX_WRITER_KEEP_ALIVE_TIME_CONF_KEY), env), env.getRegionServerServices(), parent,
            CachingHTableFactory.getCacheSize(conf));
}
 

public GlobalIndexRegionScanner(RegionScanner innerScanner, final Region region, final Scan scan,
        final RegionCoprocessorEnvironment env) throws IOException {
    super(innerScanner);
    final Configuration config = env.getConfiguration();
    if (scan.getAttribute(BaseScannerRegionObserver.INDEX_REBUILD_PAGING) != null) {
        byte[] pageSizeFromScan =
                scan.getAttribute(BaseScannerRegionObserver.INDEX_REBUILD_PAGE_ROWS);
        if (pageSizeFromScan != null) {
            pageSizeInRows = Bytes.toLong(pageSizeFromScan);
        } else {
            pageSizeInRows =
                    config.getLong(INDEX_REBUILD_PAGE_SIZE_IN_ROWS,
                        QueryServicesOptions.DEFAULT_INDEX_REBUILD_PAGE_SIZE_IN_ROWS);
        }
    }
    maxBatchSize = config.getInt(MUTATE_BATCH_SIZE_ATTRIB, QueryServicesOptions.DEFAULT_MUTATE_BATCH_SIZE);
    indexMetaData = scan.getAttribute(PhoenixIndexCodec.INDEX_PROTO_MD);
    if (indexMetaData == null) {
        indexMetaData = scan.getAttribute(PhoenixIndexCodec.INDEX_MD);
    }
    List<IndexMaintainer> maintainers = IndexMaintainer.deserialize(indexMetaData, true);
    indexMaintainer = maintainers.get(0);
    this.scan = scan;
    this.innerScanner = innerScanner;
    this.region = region;
    // Create the following objects only for rebuilds by IndexTool
    hTableFactory = IndexWriterUtils.getDefaultDelegateHTableFactory(env);
    indexHTable = hTableFactory.getTable(new ImmutableBytesPtr(indexMaintainer.getIndexTableName()));
    indexTableTTL = indexHTable.getTableDescriptor().getColumnFamilies()[0].getTimeToLive();
    pool = new WaitForCompletionTaskRunner(ThreadPoolManager.getExecutor(
            new ThreadPoolBuilder("IndexVerify",
                    env.getConfiguration()).setMaxThread(NUM_CONCURRENT_INDEX_VERIFY_THREADS_CONF_KEY,
                    DEFAULT_CONCURRENT_INDEX_VERIFY_THREADS).setCoreTimeout(
                    INDEX_WRITER_KEEP_ALIVE_TIME_CONF_KEY), env));
    rowCountPerTask = config.getInt(INDEX_VERIFY_ROW_COUNTS_PER_TASK_CONF_KEY,
            DEFAULT_INDEX_VERIFY_ROW_COUNTS_PER_TASK);
}
 

private static void setHTableThreads(Configuration conf) {
    // set the number of threads allowed per table.
    int htableThreads =
            conf.getInt(IndexWriterUtils.INDEX_WRITER_PER_TABLE_THREADS_CONF_KEY,
                IndexWriterUtils.DEFAULT_NUM_PER_TABLE_THREADS);
    IndexManagementUtil.setIfNotSet(conf, IndexWriterUtils.HTABLE_THREAD_KEY, htableThreads);
}