org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest Java Examples

@Override
public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs, InternalScanner s,
    CompactionRequest request)
    throws IOException {
  // Get the latest tx snapshot state for the compaction
  TransactionVisibilityState snapshot = cache.getLatestState();

  // Record tx state before the compaction
  if (compactionState != null) {
    compactionState.record(request, snapshot);
  }

  // silently close the passed scanner as we're returning a brand-new one
  try (InternalScanner temp = s) { }
  // Also make sure to use the same snapshot for the compaction
  return createStoreScanner(c.getEnvironment(), "compaction", snapshot, store, scanners, scanType, earliestPutTs);
}
 

@Override
public InternalScanner preCompact(
    final ObserverContext<RegionCoprocessorEnvironment> e,
    final Store store,
    final InternalScanner scanner,
    final ScanType scanType,
    final CompactionRequest request) throws IOException {
  if (opStore == null) {
    return super.preCompact(e, store, scanner, scanType, request);
  }
  return super.preCompact(
      e,
      store,
      wrapScannerWithOps(
          e.getEnvironment().getRegionInfo().getTable(),
          scanner,
          null,
          ServerOpScope.MAJOR_COMPACTION,
          INTERNAL_SCANNER_FACTORY),
      scanType,
      request);
}
 

@Override
public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs, InternalScanner s,
    CompactionRequest request)
    throws IOException {
  // Get the latest tx snapshot state for the compaction
  TransactionVisibilityState snapshot = cache.getLatestState();

  // Record tx state before the compaction
  if (compactionState != null) {
    compactionState.record(request, snapshot);
  }

  // silently close the passed scanner as we're returning a brand-new one
  try (InternalScanner temp = s) { }
  // Also make sure to use the same snapshot for the compaction
  return createStoreScanner(c.getEnvironment(), "compaction", snapshot, store, scanners, scanType, earliestPutTs);
}
 

@Override
public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs, InternalScanner s,
    CompactionRequest request)
    throws IOException {
  // Get the latest tx snapshot state for the compaction
  TransactionVisibilityState snapshot = cache.getLatestState();

  // Record tx state before the compaction
  if (compactionState != null) {
    compactionState.record(request, snapshot);
  }

  // silently close the passed scanner as we're returning a brand-new one
  try (InternalScanner temp = s) { }
  // Also make sure to use the same snapshot for the compaction
  return createStoreScanner(c.getEnvironment(), "compaction", snapshot, store, scanners, scanType, earliestPutTs);
}
 

@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
                                  InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker,
                                  CompactionRequest request) throws IOException {
    try {
        BlockingProbe.blockPreCompact();
        if (!(request instanceof SpliceCompactionRequest)) {
            SpliceLogUtils.error(LOG,"Compaction request must be a SpliceCompactionRequest");
            throw new DoNotRetryIOException();
        }
        SpliceCompactionRequest scr = (SpliceCompactionRequest) request;
        // memstoreAware is injected into the request, where the blocking logic lives, and where compaction
        // count will be incremented and decremented.
        scr.setMemstoreAware(memstoreAware);
        HRegion region = (HRegion) c.getEnvironment().getRegion();
        scr.setRegion(region);
        return scanner == null ? DummyScanner.INSTANCE : scanner;
    } catch (Throwable t) {
        throw CoprocessorUtils.getIOException(t);
    }
}
 

@Override
public InternalScanner preCompact(
    org.apache.hadoop.hbase.coprocessor.ObserverContext<RegionCoprocessorEnvironment> c,
    Store store, InternalScanner scanner, ScanType scanType,
    org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker tracker,
    CompactionRequest request) throws IOException {
  // Get the latest tx snapshot state for the compaction
  TransactionVisibilityState snapshot = cache.getLatestState();
  // Record tx state before the compaction
  if (compactionState != null) {
    compactionState.record(request, snapshot);
  }
  // Also make sure to use the same snapshot for the compaction
  InternalScanner s =
      createStoreScanner(c.getEnvironment(), "compaction", snapshot, scanner, scanType);
  if (s != null) {
    return s;
  }
  return scanner;
}
 

@Override
public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs, InternalScanner s,
    CompactionRequest request)
    throws IOException {
  // Get the latest tx snapshot state for the compaction
  TransactionVisibilityState snapshot = cache.getLatestState();

  // Record tx state before the compaction
  if (compactionState != null) {
    compactionState.record(request, snapshot);
  }

  // silently close the passed scanner as we're returning a brand-new one
  try (InternalScanner temp = s) { }
  // Also make sure to use the same snapshot for the compaction
  return createStoreScanner(c.getEnvironment(), "compaction", snapshot, store, scanners, scanType, earliestPutTs);
}
 

@Override
public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs, InternalScanner s,
    CompactionRequest request)
    throws IOException {
  // Get the latest tx snapshot state for the compaction
  TransactionVisibilityState snapshot = cache.getLatestState();

  // Record tx state before the compaction
  if (compactionState != null) {
    compactionState.record(request, snapshot);
  }

  // silently close the passed scanner as we're returning a brand-new one
  try (InternalScanner temp = s) { }
  // Also make sure to use the same snapshot for the compaction
  return createStoreScanner(c.getEnvironment(), "compaction", snapshot, store, scanners, scanType, earliestPutTs);
}
 

@Override
public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs, InternalScanner s,
    CompactionRequest request)
    throws IOException {
  // Get the latest tx snapshot state for the compaction
  TransactionVisibilityState snapshot = cache.getLatestState();

  // Record tx state before the compaction
  if (compactionState != null) {
    compactionState.record(request, snapshot);
  }

  // silently close the passed scanner as we're returning a brand-new one
  try (InternalScanner temp = s) { }
  // Also make sure to use the same snapshot for the compaction
  return createStoreScanner(c.getEnvironment(), "compaction", snapshot, store, scanners, scanType, earliestPutTs);
}
 

@Override
public void postCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store, StoreFile resultFile,
						CompactionLifeCycleTracker tracker, CompactionRequest request) throws IOException {
	if(LOG.isDebugEnabled()) {
		LOG.debug("==> RangerAuthorizationCoprocessor.postCompact()");
	}

	try {
		activatePluginClassLoader();
		implRegionObserver.postCompact(c, store, resultFile, tracker, request);
	} finally {
		deactivatePluginClassLoader();
	}

	if(LOG.isDebugEnabled()) {
		LOG.debug("<== RangerAuthorizationCoprocessor.postCompact()");
	}
}
 

@Override
public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs, InternalScanner s,
    CompactionRequest request)
    throws IOException {
  // Get the latest tx snapshot state for the compaction
  TransactionVisibilityState snapshot = cache.getLatestState();

  // Record tx state before the compaction
  if (compactionState != null) {
    compactionState.record(request, snapshot);
  }

  // silently close the passed scanner as we're returning a brand-new one
  try (InternalScanner temp = s) { }
  // Also make sure to use the same snapshot for the compaction
  return createStoreScanner(c.getEnvironment(), "compaction", snapshot, store, scanners, scanType, earliestPutTs);
}
 

@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store,	InternalScanner scanner,
								  ScanType scanType, CompactionLifeCycleTracker tracker, CompactionRequest request) throws IOException {

	final InternalScanner ret;

	if(LOG.isDebugEnabled()) {
		LOG.debug("==> RangerAuthorizationCoprocessor.preCompact()");
	}

	try {
		activatePluginClassLoader();
		ret = implRegionObserver.preCompact(c, store, scanner, scanType, tracker, request);
	} finally {
		deactivatePluginClassLoader();
	}

	if(LOG.isDebugEnabled()) {
		LOG.debug("<== RangerAuthorizationCoprocessor.preCompact()");
	}

	return ret;
}
 

@Override
public void postCompactSelection(ObserverContext<RegionCoprocessorEnvironment> c, Store store, List<? extends StoreFile> selected, CompactionLifeCycleTracker tracker, CompactionRequest request) {
	if(LOG.isDebugEnabled()) {
		LOG.debug("==> RangerAuthorizationCoprocessor.postCompactSelection()");
	}

	try {
		activatePluginClassLoader();
		implRegionObserver.postCompactSelection(c, store, selected, tracker, request);
	} finally {
		deactivatePluginClassLoader();
	}

	if(LOG.isDebugEnabled()) {
		LOG.debug("<== RangerAuthorizationCoprocessor.postCompactSelection()");
	}
}
 

@Override
public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c,
        Store store, List<? extends KeyValueScanner> scanners, ScanType scanType,
        long earliestPutTs, InternalScanner s, CompactionRequest request) throws IOException {
    InternalScanner internalScanner = super.preCompactScannerOpen(c, store, scanners, scanType, earliestPutTs, s, request);
    Collection<StoreFile> files = request.getFiles();
    storeFilesCount = 0;
    compactedFilesCount = 0;
    for(StoreFile file:files) {
        if(!file.isReference()) {
            return internalScanner;
        }
    }
    storeFilesCount = files.size();
    return internalScanner;
}
 

@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e,
    Store store, InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker,
    CompactionRequest request) {
  preCompactCalled = true;
  return scanner;
}
 

@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store,
    InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker,
    CompactionRequest request)
    throws IOException {
  try {
    Thread.sleep(sleepDuration);
  } catch (InterruptedException ie) {
    IOException ioe = new InterruptedIOException();
    ioe.initCause(ie);
    throw ioe;
  }
  return scanner;
}
 

@Override
public void postCompactSelection(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    List<? extends StoreFile> selected, CompactionLifeCycleTracker tracker,
    CompactionRequest request) {
  if (TRACKER != null) {
    assertSame(tracker, TRACKER);
  }
}
 

@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker,
    CompactionRequest request) throws IOException {
  if (TRACKER != null) {
    assertSame(tracker, TRACKER);
  }
  return scanner;
}
 

@Override
public void postCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    StoreFile resultFile, CompactionLifeCycleTracker tracker, CompactionRequest request)
    throws IOException {
  if (TRACKER != null) {
    assertSame(tracker, TRACKER);
  }
}
 

@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
        InternalScanner s, ScanType scanType, CompactionLifeCycleTracker tracker,
        CompactionRequest request) throws IOException {

    if (!IndexUtil.isLocalIndexStore(store)) { return s; }
    if (!store.hasReferences()) {
        InternalScanner repairScanner = null;
        if (request.isMajor() && (!RepairUtil.isLocalIndexStoreFilesConsistent(c.getEnvironment(), store))) {
            LOGGER.info("we have found inconsistent data for local index for region:"
                    + c.getEnvironment().getRegion().getRegionInfo());
            if (c.getEnvironment().getConfiguration().getBoolean(LOCAL_INDEX_AUTOMATIC_REPAIR, true)) {
                LOGGER.info("Starting automatic repair of local Index for region:"
                        + c.getEnvironment().getRegion().getRegionInfo());
                repairScanner = getRepairScanner(c.getEnvironment(), store);
            }
        }
        if (repairScanner != null) {
            if (s!=null) {
                s.close();
            }
            return repairScanner;
        } else {
            return s;
        }
    }
    return s;
}
 

@Override
public void postCompactSelection(
    ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    List<? extends StoreFile> selected, CompactionLifeCycleTracker tracker,
    CompactionRequest request) {
  if (selected != null) {
    filesCompactedCounter.increment(selected.size());
  }
}
 

@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker,
    CompactionRequest request) throws IOException {
  requirePermission(c, "compact", getTableName(c.getEnvironment()),
      null, null, Action.ADMIN, Action.CREATE);
  return scanner;
}
 

/**
 * Called prior to opening store scanner for compaction.
 */
public ScanInfo preCompactScannerOpen(HStore store, ScanType scanType,
    CompactionLifeCycleTracker tracker, CompactionRequest request, User user) throws IOException {
  if (coprocEnvironments.isEmpty()) {
    return store.getScanInfo();
  }
  CustomizedScanInfoBuilder builder = new CustomizedScanInfoBuilder(store.getScanInfo());
  execOperation(new RegionObserverOperationWithoutResult(user) {
    @Override
    public void call(RegionObserver observer) throws IOException {
      observer.preCompactScannerOpen(this, store, scanType, builder, tracker, request);
    }
  });
  return builder.build();
}
 

/**
 * Called after the {@link HStoreFile}s to be compacted have been selected from the available
 * candidates.
 * @param store The store where compaction is being requested
 * @param selected The store files selected to compact
 * @param tracker used to track the life cycle of a compaction
 * @param request the compaction request
 * @param user the user
 */
public void postCompactSelection(final HStore store, final List<HStoreFile> selected,
    final CompactionLifeCycleTracker tracker, final CompactionRequest request,
    final User user) throws IOException {
  if (coprocEnvironments.isEmpty()) {
    return;
  }
  execOperation(new RegionObserverOperationWithoutResult(user) {
    @Override
    public void call(RegionObserver observer) throws IOException {
      observer.postCompactSelection(this, store, selected, tracker, request);
    }
  });
}
 

public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> c,
                                  Store store,
                                  InternalScanner scanner,
                                  ScanType scanType,
                                  CompactionLifeCycleTracker tracker,
                                  CompactionRequest request) throws IOException {
    return preCompact(c,store,scanner,scanType,request);
}
 

@Override
public void postCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store, StoreFile resultFile, CompactionLifeCycleTracker tracker, CompactionRequest request) throws IOException {
    try {
        if(tableEnvMatch){
            Tracer.compact();
        }
    } catch (Throwable t) {
        throw CoprocessorUtils.getIOException(t);
    }
}
 

@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store, InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker, CompactionRequest request) throws IOException {
    try {
        // We can't return null, there's a check in org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCompact
        // return a dummy implementation instead
        if (scanner == null || scanner == DummyScanner.INSTANCE)
            return DummyScanner.INSTANCE;

        if(tableEnvMatch){
            SIDriver driver=SIDriver.driver();
            SimpleCompactionContext context = new SimpleCompactionContext();
            SICompactionState state = new SICompactionState(driver.getTxnSupplier(),
                    driver.getConfiguration().getActiveTransactionMaxCacheSize(), context, driver.getRejectingExecutorService());
            SConfiguration conf = driver.getConfiguration();
            PurgeConfig purgeConfig;
            if (conf.getOlapCompactionAutomaticallyPurgeDeletedRows()) {
                if (request.isMajor())
                    purgeConfig = PurgeConfig.purgeDuringMajorCompactionConfig();
                else
                    purgeConfig = PurgeConfig.purgeDuringMinorCompactionConfig();
            } else {
                purgeConfig = PurgeConfig.noPurgeConfig();
            }
            SICompactionScanner siScanner = new SICompactionScanner(
                    state, scanner, purgeConfig,
                    conf.getOlapCompactionResolutionShare(), conf.getLocalCompactionResolutionBufferSize(), context);
            siScanner.start();
            return siScanner;
        }
        return scanner;
    } catch (Throwable t) {
        throw CoprocessorUtils.getIOException(t);
    }
}
 

@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store, InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker, CompactionRequest request) throws IOException {
    try {
        if (LOG.isDebugEnabled())
            SpliceLogUtils.debug(LOG, "BackupEndpointObserver.preCompact()");

        BackupUtils.waitForBackupToComplete(tableName, regionName);
        isCompacting.set(true);
        SpliceLogUtils.info(LOG, "setting isCompacting=true for %s:%s", tableName, regionName);
        return scanner == null ? DummyScanner.INSTANCE : scanner;
    } catch (Throwable t) {
        throw CoprocessorUtils.getIOException(t);
    }
}
 

@Override
public void postCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store, StoreFile resultFile, CompactionLifeCycleTracker tracker, CompactionRequest request) throws IOException {
    try {
        isCompacting.set(false);
        SpliceLogUtils.info(LOG, "setting isCompacting=false for %s:%s", tableName, regionName);
        if (LOG.isDebugEnabled()) {
            String filePath =  resultFile != null?resultFile.getPath().toString():null;
            SpliceLogUtils.debug(LOG, "Compaction result file %s", filePath);
        }
    } catch (Throwable t) {
        throw CoprocessorUtils.getIOException(t);
    }
}
 

@Test
public void testPreCompactScannerOpen() throws IOException {
  RegionCoprocessorHost host = new RegionCoprocessorHost(region, rsServices, conf);
  ScanInfo oldScanInfo = getScanInfo();
  HStore store = mock(HStore.class);
  when(store.getScanInfo()).thenReturn(oldScanInfo);
  ScanInfo newScanInfo = host.preCompactScannerOpen(store, ScanType.COMPACT_DROP_DELETES,
    mock(CompactionLifeCycleTracker.class), mock(CompactionRequest.class), mock(User.class));
  verifyScanInfo(newScanInfo);
}