org.apache.hadoop.hbase.io.Reference Java Examples

The following examples show how to use org.apache.hadoop.hbase.io.Reference. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: IndexHalfStoreFileReader.java    From phoenix with Apache License 2.0 6 votes vote down vote up
/**
 * @param fs
 * @param p
 * @param cacheConf
 * @param in
 * @param size
 * @param r
 * @param conf
 * @param indexMaintainers
 * @param viewConstants
 * @param regionInfo
 * @param regionStartKeyInHFile
 * @param splitKey
 * @throws IOException
 */
public IndexHalfStoreFileReader(final FileSystem fs, final Path p, final CacheConfig cacheConf,
        final FSDataInputStreamWrapper in, long size, final Reference r,
        final Configuration conf,
        final Map<ImmutableBytesWritable, IndexMaintainer> indexMaintainers,
        final byte[][] viewConstants, final HRegionInfo regionInfo,
        byte[] regionStartKeyInHFile, byte[] splitKey) throws IOException {
    super(fs, p, in, size, cacheConf, conf);
    this.splitkey = splitKey == null ? r.getSplitKey() : splitKey;
    // Is it top or bottom half?
    this.top = Reference.isTopFileRegion(r.getFileRegion());
    this.splitRow = CellUtil.cloneRow(KeyValue.createKeyValueFromKey(splitkey));
    this.indexMaintainers = indexMaintainers;
    this.viewConstants = viewConstants;
    this.regionInfo = regionInfo;
    this.regionStartKeyInHFile = regionStartKeyInHFile;
    this.offset = regionStartKeyInHFile.length;
}
 
Example #2
Source File: IndexHalfStoreFileReader.java    From phoenix with Apache License 2.0 6 votes vote down vote up
/**
 * @param fs
 * @param p
 * @param cacheConf
 * @param in
 * @param size
 * @param r
 * @param conf
 * @param indexMaintainers
 * @param viewConstants
 * @param regionInfo
 * @param regionStartKeyInHFile
 * @param splitKey
 * @throws IOException
 */
public IndexHalfStoreFileReader(final FileSystem fs, final Path p, final CacheConfig cacheConf,
        final FSDataInputStreamWrapper in, long size, final Reference r,
        final Configuration conf,
        final Map<ImmutableBytesWritable, IndexMaintainer> indexMaintainers,
        final byte[][] viewConstants, final RegionInfo regionInfo,
        byte[] regionStartKeyInHFile, byte[] splitKey, boolean primaryReplicaStoreFile,
        AtomicInteger refCount, RegionInfo currentRegion) throws IOException {
    super(fs, p, in, size, cacheConf, primaryReplicaStoreFile, refCount, false,
            conf);
    this.splitkey = splitKey == null ? r.getSplitKey() : splitKey;
    // Is it top or bottom half?
    this.top = Reference.isTopFileRegion(r.getFileRegion());
    this.splitRow = CellUtil.cloneRow(new KeyValue.KeyOnlyKeyValue(splitkey));
    this.indexMaintainers = indexMaintainers;
    this.viewConstants = viewConstants;
    this.childRegionInfo = regionInfo;
    this.regionStartKeyInHFile = regionStartKeyInHFile;
    this.offset = regionStartKeyInHFile.length;
    this.refCount = refCount;
    this.currentRegion = currentRegion;
}
 
Example #3
Source File: TestStoreFileInfo.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testOpenErrorMessageReference() throws IOException {
  // Test file link exception
  // Try to open nonsense hfilelink. Make sure exception is from HFileLink.
  Path p = new Path(TEST_UTIL.getDataTestDirOnTestFS(),"4567.abcd");
  FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
  fs.mkdirs(p.getParent());
  Reference r = Reference.createBottomReference(HConstants.EMPTY_START_ROW);
  r.write(fs, p);
  StoreFileInfo sfi = new StoreFileInfo(TEST_UTIL.getConfiguration(), fs, p, true);
  try {
    ReaderContext context = sfi.createReaderContext(false, 1000, ReaderType.PREAD);
    sfi.createReader(context, null);
    throw new IllegalStateException();
  } catch (FileNotFoundException fnfe) {
    assertTrue(fnfe.getMessage().contains("->"));
  }
}
 
Example #4
Source File: RegionCoprocessorHost.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * @param fs fileystem to read from
 * @param p path to the file
 * @param in {@link FSDataInputStreamWrapper}
 * @param size Full size of the file
 * @param cacheConf
 * @param r original reference file. This will be not null only when reading a split file.
 * @param reader the base reader instance
 * @return The reader to use
 * @throws IOException
 */
public StoreFileReader postStoreFileReaderOpen(final FileSystem fs, final Path p,
    final FSDataInputStreamWrapper in, final long size, final CacheConfig cacheConf,
    final Reference r, final StoreFileReader reader) throws IOException {
  if (this.coprocEnvironments.isEmpty()) {
    return reader;
  }
  return execOperationWithResult(
      new ObserverOperationWithResult<RegionObserver, StoreFileReader>(regionObserverGetter, reader) {
        @Override
        public StoreFileReader call(RegionObserver observer) throws IOException {
          return observer.postStoreFileReaderOpen(this, fs, p, in, size, cacheConf, r,
              getResult());
        }
      });
}
 
Example #5
Source File: RegionCoprocessorHost.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * @param fs fileystem to read from
 * @param p path to the file
 * @param in {@link FSDataInputStreamWrapper}
 * @param size Full size of the file
 * @param cacheConf
 * @param r original reference file. This will be not null only when reading a split file.
 * @return a Reader instance to use instead of the base reader if overriding
 * default behavior, null otherwise
 * @throws IOException
 */
public StoreFileReader preStoreFileReaderOpen(final FileSystem fs, final Path p,
    final FSDataInputStreamWrapper in, final long size, final CacheConfig cacheConf,
    final Reference r) throws IOException {
  if (coprocEnvironments.isEmpty()) {
    return null;
  }
  return execOperationWithResult(
      new ObserverOperationWithResult<RegionObserver, StoreFileReader>(regionObserverGetter, null) {
        @Override
        public StoreFileReader call(RegionObserver observer) throws IOException {
          return observer.preStoreFileReaderOpen(this, fs, p, in, size, cacheConf, r,
              getResult());
        }
      });
}
 
Example #6
Source File: StoreFileInfo.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * helper function to compute HDFS blocks distribution of a given reference file.For reference
 * file, we don't compute the exact value. We use some estimate instead given it might be good
 * enough. we assume bottom part takes the first half of reference file, top part takes the second
 * half of the reference file. This is just estimate, given midkey ofregion != midkey of HFile,
 * also the number and size of keys vary. If this estimate isn't good enough, we can improve it
 * later.
 * @param fs The FileSystem
 * @param reference The reference
 * @param status The reference FileStatus
 * @return HDFS blocks distribution
 */
private static HDFSBlocksDistribution computeRefFileHDFSBlockDistribution(final FileSystem fs,
    final Reference reference, final FileStatus status) throws IOException {
  if (status == null) {
    return null;
  }

  long start = 0;
  long length = 0;

  if (Reference.isTopFileRegion(reference.getFileRegion())) {
    start = status.getLen() / 2;
    length = status.getLen() - status.getLen() / 2;
  } else {
    start = 0;
    length = status.getLen() / 2;
  }
  return FSUtils.computeHDFSBlocksDistribution(fs, status, start, length);
}
 
Example #7
Source File: HRegionFileSystem.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Write out a merge reference under the given merges directory. Package local
 * so it doesnt leak out of regionserver.
 * @param mergedRegion {@link RegionInfo} of the merged region
 * @param familyName Column Family Name
 * @param f File to create reference.
 * @param mergedDir
 * @return Path to created reference.
 * @throws IOException
 */
public Path mergeStoreFile(RegionInfo mergedRegion, String familyName, HStoreFile f,
    Path mergedDir) throws IOException {
  Path referenceDir = new Path(new Path(mergedDir,
      mergedRegion.getEncodedName()), familyName);
  // A whole reference to the store file.
  Reference r = Reference.createTopReference(regionInfoForFs.getStartKey());
  // Add the referred-to regions name as a dot separated suffix.
  // See REF_NAME_REGEX regex above. The referred-to regions name is
  // up in the path of the passed in <code>f</code> -- parentdir is family,
  // then the directory above is the region name.
  String mergingRegionName = regionInfoForFs.getEncodedName();
  // Write reference with same file id only with the other region name as
  // suffix and into the new region location (under same family).
  Path p = new Path(referenceDir, f.getPath().getName() + "."
      + mergingRegionName);
  return r.write(fs, p);
}
 
Example #8
Source File: RangerAuthorizationCoprocessor.java    From ranger with Apache License 2.0 6 votes vote down vote up
@Override
public StoreFileReader postStoreFileReaderOpen(ObserverContext<RegionCoprocessorEnvironment> ctx, FileSystem fs,	Path p, FSDataInputStreamWrapper in, long size,
											   CacheConfig cacheConf, Reference r, StoreFileReader reader) throws IOException {
	final StoreFileReader ret;

	if(LOG.isDebugEnabled()) {
		LOG.debug("==> RangerAuthorizationCoprocessor.postStoreFileReaderOpen()");
	}

	try {
		activatePluginClassLoader();
		ret = implRegionObserver.postStoreFileReaderOpen(ctx, fs, p, in, size, cacheConf, r, reader);
	} finally {
		deactivatePluginClassLoader();
	}

	if(LOG.isDebugEnabled()) {
		LOG.debug("<== RangerAuthorizationCoprocessor.postStoreFileReaderOpen()");
	}

	return ret;
}
 
Example #9
Source File: IndexHalfStoreFileReader.java    From phoenix with Apache License 2.0 6 votes vote down vote up
/**
 * @param fs
 * @param p
 * @param cacheConf
 * @param r
 * @param conf
 * @param indexMaintainers
 * @param viewConstants
 * @param regionInfo
 * @param regionStartKeyInHFile
 * @param splitKey
 * @throws IOException
 */
public IndexHalfStoreFileReader(final FileSystem fs, final Path p, final CacheConfig cacheConf,
        final Reference r, final Configuration conf,
        final Map<ImmutableBytesWritable, IndexMaintainer> indexMaintainers,
        final byte[][] viewConstants, final HRegionInfo regionInfo,
        final byte[] regionStartKeyInHFile, byte[] splitKey) throws IOException {
    super(fs, p, cacheConf, conf);
    this.splitkey = splitKey == null ? r.getSplitKey() : splitKey;
    // Is it top or bottom half?
    this.top = Reference.isTopFileRegion(r.getFileRegion());
    this.splitRow = CellUtil.cloneRow(KeyValue.createKeyValueFromKey(splitkey));
    this.indexMaintainers = indexMaintainers;
    this.viewConstants = viewConstants;
    this.regionInfo = regionInfo;
    this.regionStartKeyInHFile = regionStartKeyInHFile;
    this.offset = regionStartKeyInHFile.length;
}
 
Example #10
Source File: IndexSplitTransaction.java    From phoenix with Apache License 2.0 6 votes vote down vote up
private Path splitStoreFile(HRegionInfo hri, String familyName, StoreFile f, byte[] splitRow,
        boolean top, HRegionFileSystem fs) throws IOException {
    f.closeReader(true);
    Path splitDir =
            new Path(fs.getSplitsDir(hri), familyName);
    // A reference to the bottom half of the hsf store file.
    Reference r =
            top ? Reference.createTopReference(splitRow) : Reference
                    .createBottomReference(splitRow);
    // Add the referred-to regions name as a dot separated suffix.
    // See REF_NAME_REGEX regex above. The referred-to regions name is
    // up in the path of the passed in <code>f</code> -- parentdir is family,
    // then the directory above is the region name.
    String parentRegionName = this.parent.getRegionInfo().getEncodedName();
    // Write reference with same file id only with the other region name as
    // suffix and into the new region location (under same family).
    Path p = new Path(splitDir, f.getPath().getName() + "." + parentRegionName);
    return r.write(fs.getFileSystem(), p);
}
 
Example #11
Source File: RangerAuthorizationCoprocessor.java    From ranger with Apache License 2.0 6 votes vote down vote up
@Override
public StoreFileReader preStoreFileReaderOpen(ObserverContext<RegionCoprocessorEnvironment> ctx, FileSystem fs, Path p, FSDataInputStreamWrapper in, long size,
											  CacheConfig cacheConf, Reference r, StoreFileReader reader) throws IOException {
	final StoreFileReader ret;

	if(LOG.isDebugEnabled()) {
		LOG.debug("==> RangerAuthorizationCoprocessor.preStoreFileReaderOpen()");
	}

	try {
		activatePluginClassLoader();
		ret = implRegionObserver.preStoreFileReaderOpen(ctx, fs, p, in, size, cacheConf, r, reader);
	} finally {
		deactivatePluginClassLoader();
	}

	if(LOG.isDebugEnabled()) {
		LOG.debug("<== RangerAuthorizationCoprocessor.preStoreFileReaderOpen()");
	}

	return ret;
}
 
Example #12
Source File: StoreFileInfo.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Create a Store File Info from an HFileLink and a Reference
 * @param conf The {@link Configuration} to use
 * @param fs The current file system to use
 * @param fileStatus The {@link FileStatus} of the file
 * @param reference The reference instance
 * @param link The link instance
 */
public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus,
    final Reference reference, final HFileLink link) {
  this.fs = fs;
  this.conf = conf;
  this.primaryReplica = false;
  this.initialPath = (fileStatus == null) ? null : fileStatus.getPath();
  this.createdTimestamp = (fileStatus == null) ? 0 : fileStatus.getModificationTime();
  this.reference = reference;
  this.link = link;
  this.noReadahead = this.conf.getBoolean(STORE_FILE_READER_NO_READAHEAD,
    DEFAULT_STORE_FILE_READER_NO_READAHEAD);
}
 
Example #13
Source File: BulkLoadHFilesTool.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Split a storefile into a top and bottom half, maintaining the metadata, recreating bloom
 * filters, etc.
 */
@VisibleForTesting
static void splitStoreFile(Configuration conf, Path inFile, ColumnFamilyDescriptor familyDesc,
    byte[] splitKey, Path bottomOut, Path topOut) throws IOException {
  // Open reader with no block cache, and not in-memory
  Reference topReference = Reference.createTopReference(splitKey);
  Reference bottomReference = Reference.createBottomReference(splitKey);

  copyHFileHalf(conf, inFile, topOut, topReference, familyDesc);
  copyHFileHalf(conf, inFile, bottomOut, bottomReference, familyDesc);
}
 
Example #14
Source File: TestCatalogJanitor.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Test clearing a split parent.
 */
@Test
public void testCleanParent() throws IOException, InterruptedException {
  TableDescriptor td = createTableDescriptorForCurrentMethod();
  // Create regions.
  RegionInfo parent =
    createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
  RegionInfo splita =
    createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
  RegionInfo splitb =
    createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
  // Test that when both daughter regions are in place, that we do not remove the parent.
  Result r = createResult(parent, splita, splitb);
  // Add a reference under splitA directory so we don't clear out the parent.
  Path rootdir = this.masterServices.getMasterFileSystem().getRootDir();
  Path tabledir = CommonFSUtils.getTableDir(rootdir, td.getTableName());
  Path parentdir = new Path(tabledir, parent.getEncodedName());
  Path storedir = HStore.getStoreHomedir(tabledir, splita, td.getColumnFamilies()[0].getName());
  Reference ref = Reference.createTopReference(Bytes.toBytes("ccc"));
  long now = System.currentTimeMillis();
  // Reference name has this format: StoreFile#REF_NAME_PARSER
  Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
  FileSystem fs = this.masterServices.getMasterFileSystem().getFileSystem();
  Path path = ref.write(fs, p);
  assertTrue(fs.exists(path));
  LOG.info("Created reference " + path);
  // Add a parentdir for kicks so can check it gets removed by the catalogjanitor.
  fs.mkdirs(parentdir);
  assertFalse(this.janitor.cleanParent(parent, r));
  ProcedureTestingUtility.waitAllProcedures(masterServices.getMasterProcedureExecutor());
  assertTrue(fs.exists(parentdir));
  // Remove the reference file and try again.
  assertTrue(fs.delete(p, true));
  assertTrue(this.janitor.cleanParent(parent, r));
  // Parent cleanup is run async as a procedure. Make sure parentdir is removed.
  ProcedureTestingUtility.waitAllProcedures(masterServices.getMasterProcedureExecutor());
  assertTrue(!fs.exists(parentdir));
}
 
Example #15
Source File: TestCatalogJanitor.java    From hbase with Apache License 2.0 5 votes vote down vote up
private Path createReferences(final MasterServices services, final TableDescriptor td,
  final RegionInfo parent, final RegionInfo daughter, final byte[] midkey, final boolean top)
  throws IOException {
  Path rootdir = services.getMasterFileSystem().getRootDir();
  Path tabledir = CommonFSUtils.getTableDir(rootdir, parent.getTable());
  Path storedir = HStore.getStoreHomedir(tabledir, daughter, td.getColumnFamilies()[0].getName());
  Reference ref =
    top ? Reference.createTopReference(midkey) : Reference.createBottomReference(midkey);
  long now = System.currentTimeMillis();
  // Reference name has this format: StoreFile#REF_NAME_PARSER
  Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
  FileSystem fs = services.getMasterFileSystem().getFileSystem();
  ref.write(fs, p);
  return p;
}
 
Example #16
Source File: SimpleRegionObserver.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public StoreFileReader preStoreFileReaderOpen(ObserverContext<RegionCoprocessorEnvironment> ctx,
    FileSystem fs, Path p, FSDataInputStreamWrapper in, long size, CacheConfig cacheConf,
    Reference r, StoreFileReader reader) throws IOException {
  ctPreStoreFileReaderOpen.incrementAndGet();
  return null;
}
 
Example #17
Source File: SimpleRegionObserver.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public StoreFileReader postStoreFileReaderOpen(ObserverContext<RegionCoprocessorEnvironment> ctx,
    FileSystem fs, Path p, FSDataInputStreamWrapper in, long size, CacheConfig cacheConf,
    Reference r, StoreFileReader reader) throws IOException {
  ctPostStoreFileReaderOpen.incrementAndGet();
  return reader;
}
 
Example #18
Source File: StoreFileInfo.java    From hbase with Apache License 2.0 4 votes vote down vote up
/** @return True if the store file is a top Reference */
public boolean isTopReference() {
  return this.reference != null && Reference.isTopFileRegion(this.reference.getFileRegion());
}
 
Example #19
Source File: TestSplitTransactionOnCluster.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Test
public void testSplitCompactWithPriority() throws Exception {
  final TableName tableName = TableName.valueOf(name.getMethodName());
  // Create table then get the single region for our new table.
  byte[] cf = Bytes.toBytes("cf");
  TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName)
    .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf)).build();
  admin.createTable(htd);

  assertNotEquals("Unable to retrieve regions of the table", -1,
    TESTING_UTIL.waitFor(10000, () -> cluster.getRegions(tableName).size() == 1));

  HRegion region = cluster.getRegions(tableName).get(0);
  HStore store = region.getStore(cf);
  int regionServerIndex = cluster.getServerWith(region.getRegionInfo().getRegionName());
  HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);

  Table table = TESTING_UTIL.getConnection().getTable(tableName);
  // insert data
  insertData(tableName, admin, table);
  insertData(tableName, admin, table, 20);
  insertData(tableName, admin, table, 40);

  // Compaction Request
  store.triggerMajorCompaction();
  Optional<CompactionContext> compactionContext = store.requestCompaction();
  assertTrue(compactionContext.isPresent());
  assertFalse(compactionContext.get().getRequest().isAfterSplit());
  assertEquals(compactionContext.get().getRequest().getPriority(), 13);

  // Split
  long procId =
    cluster.getMaster().splitRegion(region.getRegionInfo(), Bytes.toBytes("row4"), 0, 0);

  // wait for the split to complete or get interrupted.  If the split completes successfully,
  // the procedure will return true; if the split fails, the procedure would throw exception.
  ProcedureTestingUtility.waitProcedure(cluster.getMaster().getMasterProcedureExecutor(),
    procId);
  Thread.sleep(3000);
  assertNotEquals("Table is not split properly?", -1,
    TESTING_UTIL.waitFor(3000,
      () -> cluster.getRegions(tableName).size() == 2));
  // we have 2 daughter regions
  HRegion hRegion1 = cluster.getRegions(tableName).get(0);
  HRegion hRegion2 = cluster.getRegions(tableName).get(1);
  HStore hStore1 = hRegion1.getStore(cf);
  HStore hStore2 = hRegion2.getStore(cf);

  // For hStore1 && hStore2, set mock reference to one of the storeFiles
  StoreFileInfo storeFileInfo1 = new ArrayList<>(hStore1.getStorefiles()).get(0).getFileInfo();
  StoreFileInfo storeFileInfo2 = new ArrayList<>(hStore2.getStorefiles()).get(0).getFileInfo();
  Field field = StoreFileInfo.class.getDeclaredField("reference");
  field.setAccessible(true);
  field.set(storeFileInfo1, Mockito.mock(Reference.class));
  field.set(storeFileInfo2, Mockito.mock(Reference.class));
  hStore1.triggerMajorCompaction();
  hStore2.triggerMajorCompaction();

  compactionContext = hStore1.requestCompaction();
  assertTrue(compactionContext.isPresent());
  // since we set mock reference to one of the storeFiles, we will get isAfterSplit=true &&
  // highest priority for hStore1's compactionContext
  assertTrue(compactionContext.get().getRequest().isAfterSplit());
  assertEquals(compactionContext.get().getRequest().getPriority(), Integer.MIN_VALUE + 1000);

  compactionContext =
    hStore2.requestCompaction(Integer.MIN_VALUE + 10, CompactionLifeCycleTracker.DUMMY, null);
  assertTrue(compactionContext.isPresent());
  // compaction request contains higher priority than default priority of daughter region
  // compaction (Integer.MIN_VALUE + 1000), hence we are expecting request priority to
  // be accepted.
  assertTrue(compactionContext.get().getRequest().isAfterSplit());
  assertEquals(compactionContext.get().getRequest().getPriority(), Integer.MIN_VALUE + 10);
  admin.disableTable(tableName);
  admin.deleteTable(tableName);
}
 
Example #20
Source File: BulkLoadHFilesTool.java    From hbase with Apache License 2.0 4 votes vote down vote up
/**
 * Copy half of an HFile into a new HFile.
 */
private static void copyHFileHalf(Configuration conf, Path inFile, Path outFile,
    Reference reference, ColumnFamilyDescriptor familyDescriptor) throws IOException {
  FileSystem fs = inFile.getFileSystem(conf);
  CacheConfig cacheConf = CacheConfig.DISABLED;
  HalfStoreFileReader halfReader = null;
  StoreFileWriter halfWriter = null;
  try {
    ReaderContext context = new ReaderContextBuilder()
        .withFileSystemAndPath(fs, inFile).build();
    HFileInfo hfile = new HFileInfo(context, conf);
    halfReader = new HalfStoreFileReader(context, hfile, cacheConf, reference,
      new AtomicInteger(0), conf);
    hfile.initMetaAndIndex(halfReader.getHFileReader());
    Map<byte[], byte[]> fileInfo = halfReader.loadFileInfo();

    int blocksize = familyDescriptor.getBlocksize();
    Algorithm compression = familyDescriptor.getCompressionType();
    BloomType bloomFilterType = familyDescriptor.getBloomFilterType();
    HFileContext hFileContext = new HFileContextBuilder().withCompression(compression)
      .withChecksumType(HStore.getChecksumType(conf))
      .withBytesPerCheckSum(HStore.getBytesPerChecksum(conf)).withBlockSize(blocksize)
      .withDataBlockEncoding(familyDescriptor.getDataBlockEncoding()).withIncludesTags(true)
      .build();
    halfWriter = new StoreFileWriter.Builder(conf, cacheConf, fs).withFilePath(outFile)
      .withBloomType(bloomFilterType).withFileContext(hFileContext).build();
    HFileScanner scanner = halfReader.getScanner(false, false, false);
    scanner.seekTo();
    do {
      halfWriter.append(scanner.getCell());
    } while (scanner.next());

    for (Map.Entry<byte[], byte[]> entry : fileInfo.entrySet()) {
      if (shouldCopyHFileMetaKey(entry.getKey())) {
        halfWriter.appendFileInfo(entry.getKey(), entry.getValue());
      }
    }
  } finally {
    if (halfReader != null) {
      try {
        halfReader.close(cacheConf.shouldEvictOnClose());
      } catch (IOException e) {
        LOG.warn("failed to close hfile reader for " + inFile, e);
      }
    }
    if (halfWriter != null) {
      halfWriter.close();
    }
  }
}
 
Example #21
Source File: RegionObserver.java    From hbase with Apache License 2.0 3 votes vote down vote up
/**
 * Called before creation of Reader for a store file.
 * Calling {@link org.apache.hadoop.hbase.coprocessor.ObserverContext#bypass()} has no
 * effect in this hook.
 *
 * @param ctx the environment provided by the region server
 * @param fs fileystem to read from
 * @param p path to the file
 * @param in {@link FSDataInputStreamWrapper}
 * @param size Full size of the file
 * @param cacheConf
 * @param r original reference file. This will be not null only when reading a split file.
 * @param reader the base reader, if not {@code null}, from previous RegionObserver in the chain
 * @return a Reader instance to use instead of the base reader if overriding
 * default behavior, null otherwise
 * @deprecated For Phoenix only, StoreFileReader is not a stable interface.
 */
@Deprecated
// Passing InterfaceAudience.Private args FSDataInputStreamWrapper, CacheConfig and Reference.
// This is fine as the hook is deprecated any way.
default StoreFileReader preStoreFileReaderOpen(ObserverContext<RegionCoprocessorEnvironment> ctx,
    FileSystem fs, Path p, FSDataInputStreamWrapper in, long size, CacheConfig cacheConf,
    Reference r, StoreFileReader reader) throws IOException {
  return reader;
}
 
Example #22
Source File: RegionObserver.java    From hbase with Apache License 2.0 3 votes vote down vote up
/**
 * Called after the creation of Reader for a store file.
 *
 * @param ctx the environment provided by the region server
 * @param fs fileystem to read from
 * @param p path to the file
 * @param in {@link FSDataInputStreamWrapper}
 * @param size Full size of the file
 * @param cacheConf
 * @param r original reference file. This will be not null only when reading a split file.
 * @param reader the base reader instance
 * @return The reader to use
 * @deprecated For Phoenix only, StoreFileReader is not a stable interface.
 */
@Deprecated
// Passing InterfaceAudience.Private args FSDataInputStreamWrapper, CacheConfig and Reference.
// This is fine as the hook is deprecated any way.
default StoreFileReader postStoreFileReaderOpen(ObserverContext<RegionCoprocessorEnvironment> ctx,
    FileSystem fs, Path p, FSDataInputStreamWrapper in, long size, CacheConfig cacheConf,
    Reference r, StoreFileReader reader) throws IOException {
  return reader;
}
 
Example #23
Source File: StoreFileInfo.java    From hbase with Apache License 2.0 2 votes vote down vote up
/**
 * @return the Reference object associated to this StoreFileInfo.
 *   null if the StoreFile is not a
 *   reference.
 */
public Reference getReference() {
  return this.reference;
}
 
Example #24
Source File: StoreFileInfo.java    From hbase with Apache License 2.0 2 votes vote down vote up
/**
 * Create a Store File Info from an HFileLink
 * @param conf The {@link Configuration} to use
 * @param fs The current file system to use
 * @param fileStatus The {@link FileStatus} of the file
 * @param reference The reference instance
 */
public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus,
    final Reference reference) {
  this(conf, fs, fileStatus, reference, null);
}