Java Code Examples for org.apache.hadoop.hbase.io.encoding.DataBlockEncoding#NONE

/**
 * Serialize column family to data block encoding map to configuration.
 * Invoked while configuring the MR job for incremental load.
 *
 * @param table to read the properties from
 * @param conf to persist serialized values into
 * @throws IOException
 *           on failure to read column family descriptors
 */
@VisibleForTesting
static void configureDataBlockEncoding(HTableDescriptor tableDescriptor, Configuration conf)
        throws UnsupportedEncodingException {
    if (tableDescriptor == null) {
        // could happen with mock table instance
        return;
    }
    StringBuilder dataBlockEncodingConfigValue = new StringBuilder();
    Collection<HColumnDescriptor> families = tableDescriptor.getFamilies();
    int i = 0;
    for (HColumnDescriptor familyDescriptor : families) {
        if (i++ > 0) {
            dataBlockEncodingConfigValue.append('&');
        }
        dataBlockEncodingConfigValue.append(URLEncoder.encode(familyDescriptor.getNameAsString(), "UTF-8"));
        dataBlockEncodingConfigValue.append('=');
        DataBlockEncoding encoding = familyDescriptor.getDataBlockEncoding();
        if (encoding == null) {
            encoding = DataBlockEncoding.NONE;
        }
        dataBlockEncodingConfigValue.append(URLEncoder.encode(encoding.toString(), "UTF-8"));
    }
    conf.set(DATABLOCK_ENCODING_FAMILIES_CONF_KEY, dataBlockEncodingConfigValue.toString());
}
 

private void testEncodingWithCacheInternals(boolean useTag) throws IOException {
  List<KeyValue> kvs = generator.generateTestKeyValues(60, useTag);
  HFileBlock block = getSampleHFileBlock(kvs, useTag);
  HFileBlock cacheBlock = createBlockOnDisk(kvs, block, useTag);

  LruBlockCache blockCache =
      new LruBlockCache(8 * 1024 * 1024, 32 * 1024);
  BlockCacheKey cacheKey = new BlockCacheKey("test", 0);
  blockCache.cacheBlock(cacheKey, cacheBlock);

  HeapSize heapSize = blockCache.getBlock(cacheKey, false, false, true);
  assertTrue(heapSize instanceof HFileBlock);

  HFileBlock returnedBlock = (HFileBlock) heapSize;

  if (blockEncoder.getDataBlockEncoding() ==
      DataBlockEncoding.NONE) {
    assertEquals(block.getBufferReadOnly(), returnedBlock.getBufferReadOnly());
  } else {
    if (BlockType.ENCODED_DATA != returnedBlock.getBlockType()) {
      System.out.println(blockEncoder);
    }
    assertEquals(BlockType.ENCODED_DATA, returnedBlock.getBlockType());
  }
}
 

public static HFileDataBlockEncoder createFromFileInfo(
    HFileInfo fileInfo) throws IOException {
  DataBlockEncoding encoding = DataBlockEncoding.NONE;
  byte[] dataBlockEncodingType = fileInfo.get(DATA_BLOCK_ENCODING);
  if (dataBlockEncodingType != null) {
    String dataBlockEncodingStr = Bytes.toString(dataBlockEncodingType);
    try {
      encoding = DataBlockEncoding.valueOf(dataBlockEncodingStr);
    } catch (IllegalArgumentException ex) {
      throw new IOException("Invalid data block encoding type in file info: "
        + dataBlockEncodingStr, ex);
    }
  }

  if (encoding == DataBlockEncoding.NONE) {
    return NoOpDataBlockEncoder.INSTANCE;
  }
  return new HFileDataBlockEncoderImpl(encoding);
}
 

public void majorCompactionWithDataBlockEncoding(boolean inCacheOnly) throws Exception {
  Map<HStore, HFileDataBlockEncoder> replaceBlockCache = new HashMap<>();
  for (HStore store : r.getStores()) {
    HFileDataBlockEncoder blockEncoder = store.getDataBlockEncoder();
    replaceBlockCache.put(store, blockEncoder);
    final DataBlockEncoding inCache = DataBlockEncoding.PREFIX;
    final DataBlockEncoding onDisk = inCacheOnly ? DataBlockEncoding.NONE : inCache;
    ((HStore) store).setDataBlockEncoderInTest(new HFileDataBlockEncoderImpl(onDisk));
  }

  majorCompaction();

  // restore settings
  for (Entry<HStore, HFileDataBlockEncoder> entry : replaceBlockCache.entrySet()) {
    ((HStore) entry.getKey()).setDataBlockEncoderInTest(entry.getValue());
  }
}
 

/**
 * Serialize column family to data block encoding map to configuration.
 * Invoked while configuring the MR job for incremental load.
 *
 * @param table to read the properties from
 * @param conf to persist serialized values into
 * @throws IOException
 *           on failure to read column family descriptors
 */
static String configureDataBlockEncoding(TableDescriptor tableDescriptor) throws UnsupportedEncodingException {
  
    StringBuilder dataBlockEncodingConfigValue = new StringBuilder();
    
    if (tableDescriptor == null) {
        // could happen with mock table instance
        return dataBlockEncodingConfigValue.toString();
    }
    ColumnFamilyDescriptor[] families = tableDescriptor.getColumnFamilies();
    int i = 0;
    for (ColumnFamilyDescriptor familyDescriptor : families) {
        if (i++ > 0) {
            dataBlockEncodingConfigValue.append('&');
        }
        dataBlockEncodingConfigValue.append(
                URLEncoder.encode(familyDescriptor.getNameAsString(), "UTF-8"));
        dataBlockEncodingConfigValue.append('=');
        DataBlockEncoding encoding = familyDescriptor.getDataBlockEncoding();
        if (encoding == null) {
            encoding = DataBlockEncoding.NONE;
        }
        dataBlockEncodingConfigValue.append(URLEncoder.encode(encoding.toString(),
                "UTF-8"));
    }
    return dataBlockEncodingConfigValue.toString();
}
 

/**
 * @return All possible data block encoding configurations
 */
@Parameters
public static Collection<Object[]> getAllConfigurations() {
  List<Object[]> configurations = new ArrayList<>();

  for (DataBlockEncoding diskAlgo : DataBlockEncoding.values()) {
    for (boolean includesMemstoreTS : new boolean[] { false, true }) {
      HFileDataBlockEncoder dbe = (diskAlgo == DataBlockEncoding.NONE) ?
          NoOpDataBlockEncoder.INSTANCE : new HFileDataBlockEncoderImpl(diskAlgo);
      configurations.add(new Object[] { dbe, new Boolean(includesMemstoreTS) });
    }
  }

  return configurations;
}
 

@Parameterized.Parameters
public static Collection<Object[]> parameters() {
  DataBlockEncoding[] dataBlockEncodings = DataBlockEncoding.values();
  Object[][] params = new Object[dataBlockEncodings.length * 2 - 2][];
  int i = 0;
  for (DataBlockEncoding dataBlockEncoding : dataBlockEncodings) {
    if (dataBlockEncoding == DataBlockEncoding.NONE) {
      continue;
    }
    params[i++] = new Object[]{false, dataBlockEncoding};
    params[i++] = new Object[]{true, dataBlockEncoding};
  }
  return Arrays.asList(params);
}
 

public static Collection<Object[]> generateParams(Compression.Algorithm algo,
    boolean useDataBlockEncoding) {
  List<Object[]> parameters = new ArrayList<>();
  for (BloomType bloomType : BloomType.values()) {
    DataBlockEncoding dataBlockEncoding =
        useDataBlockEncoding ? DataBlockEncoding.PREFIX : DataBlockEncoding.NONE;
    parameters.add(new Object[] { algo, bloomType, dataBlockEncoding });
  }
  return parameters;
}
 

@Override
public DataBlockEncoding getEffectiveEncodingInCache(boolean isCompaction) {
  if (!useEncodedScanner(isCompaction)) {
    return DataBlockEncoding.NONE;
  }
  return encoding;
}
 

protected void finishFileInfo() throws IOException {
  if (lastCell != null) {
    // Make a copy. The copy is stuffed into our fileinfo map. Needs a clean
    // byte buffer. Won't take a tuple.
    byte [] lastKey = PrivateCellUtil.getCellKeySerializedAsKeyValueKey(this.lastCell);
    fileInfo.append(HFileInfo.LASTKEY, lastKey, false);
  }

  // Average key length.
  int avgKeyLen =
      entryCount == 0 ? 0 : (int) (totalKeyLength / entryCount);
  fileInfo.append(HFileInfo.AVG_KEY_LEN, Bytes.toBytes(avgKeyLen), false);
  fileInfo.append(HFileInfo.CREATE_TIME_TS, Bytes.toBytes(hFileContext.getFileCreateTime()),
    false);

  // Average value length.
  int avgValueLen =
      entryCount == 0 ? 0 : (int) (totalValueLength / entryCount);
  fileInfo.append(HFileInfo.AVG_VALUE_LEN, Bytes.toBytes(avgValueLen), false);
  if (hFileContext.isIncludesTags()) {
    // When tags are not being written in this file, MAX_TAGS_LEN is excluded
    // from the FileInfo
    fileInfo.append(HFileInfo.MAX_TAGS_LEN, Bytes.toBytes(this.maxTagsLength), false);
    boolean tagsCompressed = (hFileContext.getDataBlockEncoding() != DataBlockEncoding.NONE)
      && hFileContext.isCompressTags();
    fileInfo.append(HFileInfo.TAGS_COMPRESSED, Bytes.toBytes(tagsCompressed), false);
  }
}
 

public static DataBlockEncoding dataBlockEncodingFromThrift(TDataBlockEncoding in) {
  switch (in.getValue()) {
    case 0: return DataBlockEncoding.NONE;
    case 2: return DataBlockEncoding.PREFIX;
    case 3: return DataBlockEncoding.DIFF;
    case 4: return DataBlockEncoding.FAST_DIFF;
    case 7: return DataBlockEncoding.ROW_INDEX_V1;
    default: return DataBlockEncoding.NONE;
  }
}
 

@Override
public boolean useEncodedScanner() {
  return encoding != DataBlockEncoding.NONE;
}
 

@Override
void perform() throws IOException {
  TableDescriptor selected = selectTable(disabledTables);
  if (selected == null) {
    return;
  }
  ColumnFamilyDescriptor columnDesc = selectFamily(selected);
  if (columnDesc == null){
    return;
  }

  Admin admin = connection.getAdmin();
  try {
    TableName tableName = selected.getTableName();
    // possible DataBlockEncoding ids
    DataBlockEncoding[] possibleIds = {DataBlockEncoding.NONE, DataBlockEncoding.PREFIX,
            DataBlockEncoding.DIFF, DataBlockEncoding.FAST_DIFF, DataBlockEncoding.ROW_INDEX_V1};
    short id = possibleIds[RandomUtils.nextInt(0, possibleIds.length)].getId();
    LOG.info("Altering encoding of column family: " + columnDesc + " to: " + id +
        " in table: " + tableName);

    ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(columnDesc)
        .setDataBlockEncoding(DataBlockEncoding.getEncodingById(id))
        .build();
    TableDescriptor td = TableDescriptorBuilder.newBuilder(selected)
        .modifyColumnFamily(cfd)
        .build();
    admin.modifyTable(td);

    // assertion
    TableDescriptor freshTableDesc = admin.getDescriptor(tableName);
    ColumnFamilyDescriptor freshColumnDesc = freshTableDesc.getColumnFamily(columnDesc.getName());
    Assert.assertEquals("Encoding of column family: " + columnDesc + " was not altered",
        freshColumnDesc.getDataBlockEncoding().getId(), id);
    Assert.assertTrue(
      "After alter encoding of column family, Table: " + tableName + " is not disabled",
      admin.isTableDisabled(tableName));
    disabledTables.put(tableName, freshTableDesc);
    LOG.info("Altered encoding of column family: " + freshColumnDesc + " to: " + id +
      " in table: " + tableName);
  } catch (Exception e) {
    LOG.warn("Caught exception in action: " + this.getClass());
    throw e;
  } finally {
    admin.close();
  }
}
 

@Override
public DataBlockEncoding getDataBlockEncoding() {
  return DataBlockEncoding.NONE;
}
 

@Override
public DataBlockEncoding getEffectiveEncodingInCache(boolean isCompaction) {
  return DataBlockEncoding.NONE;
}
 

DataBlockEncoding getDataBlockEncoding() {
  if (blockType == BlockType.ENCODED_DATA) {
    return DataBlockEncoding.getEncodingById(getDataBlockEncodingId());
  }
  return DataBlockEncoding.NONE;
}
 

public boolean useEncodedScanner(boolean isCompaction) {
  if (isCompaction && encoding == DataBlockEncoding.NONE) {
    return false;
  }
  return encoding != DataBlockEncoding.NONE;
}
 

/**
 * Check statistics for given HFile for different data block encoders.
 * @param scanner Of file which will be compressed.
 * @param kvLimit Maximal count of KeyValue which will be processed.
 * @throws IOException thrown if scanner is invalid
 */
public void checkStatistics(final KeyValueScanner scanner, final int kvLimit)
    throws IOException {
  scanner.seek(KeyValue.LOWESTKEY);

  KeyValue currentKV;

  byte[] previousKey = null;
  byte[] currentKey;

  DataBlockEncoding[] encodings = DataBlockEncoding.values();

  ByteArrayOutputStream uncompressedOutputStream =
      new ByteArrayOutputStream();

  int j = 0;
  while ((currentKV = KeyValueUtil.ensureKeyValue(scanner.next())) != null && j < kvLimit) {
    // Iterates through key/value pairs
    j++;
    currentKey = currentKV.getKey();
    if (previousKey != null) {
      for (int i = 0; i < previousKey.length && i < currentKey.length &&
          previousKey[i] == currentKey[i]; ++i) {
        totalKeyRedundancyLength++;
      }
    }

    // Add tagsLen zero to cells don't include tags. Since the process of
    // scanner converts byte array to KV would abandon tagsLen part if tagsLen
    // is zero. But we still needs the tagsLen part to check if current cell
    // include tags. If USE_TAG is true, HFile contains cells with tags,
    // if the cell tagsLen equals 0, it means other cells may have tags.
    if (USE_TAG && currentKV.getTagsLength() == 0) {
      uncompressedOutputStream.write(currentKV.getBuffer(),
          currentKV.getOffset(), currentKV.getLength());
      // write tagsLen = 0.
      uncompressedOutputStream.write(Bytes.toBytes((short) 0));
    } else {
      uncompressedOutputStream.write(currentKV.getBuffer(),
          currentKV.getOffset(), currentKV.getLength());
    }

    if(includesMemstoreTS) {
      WritableUtils.writeVLong(
          new DataOutputStream(uncompressedOutputStream), currentKV.getSequenceId());
    }

    previousKey = currentKey;

    int kLen = currentKV.getKeyLength();
    int vLen = currentKV.getValueLength();
    int cfLen = currentKV.getFamilyLength(currentKV.getFamilyOffset());
    int restLen = currentKV.getLength() - kLen - vLen;

    totalKeyLength += kLen;
    totalValueLength += vLen;
    totalPrefixLength += restLen;
    totalCFLength += cfLen;
  }

  rawKVs = uncompressedOutputStream.toByteArray();
  for (DataBlockEncoding encoding : encodings) {
    if (encoding == DataBlockEncoding.NONE) {
      continue;
    }
    DataBlockEncoder d = encoding.getEncoder();
    HFileContext meta = new HFileContextBuilder()
        .withDataBlockEncoding(encoding)
        .withCompression(Compression.Algorithm.NONE)
        .withIncludesMvcc(includesMemstoreTS)
        .withIncludesTags(USE_TAG).build();
    codecs.add(new EncodedDataBlock(d, encoding, rawKVs, meta ));
  }
}
 

private StoreFileWriter getNewWriter(Configuration conf, BulkImportPartition partition)
        throws IOException {

    Compression.Algorithm compression = Compression.getCompressionAlgorithmByName(compressionAlgorithm);
    BloomType bloomType = BloomType.ROW;
    Integer blockSize = HConstants.DEFAULT_BLOCKSIZE;
    DataBlockEncoding encoding = DataBlockEncoding.NONE;
    Configuration tempConf = new Configuration(conf);
    tempConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f);
    HFileContextBuilder contextBuilder = new HFileContextBuilder()
            .withCompression(compression)
            .withChecksumType(HStore.getChecksumType(conf))
            .withBytesPerCheckSum(HStore.getBytesPerChecksum(conf))
            .withBlockSize(blockSize);

    if (HFile.getFormatVersion(conf) >= HFile.MIN_FORMAT_VERSION_WITH_TAGS) {
        contextBuilder.withIncludesTags(true);
    }

    contextBuilder.withDataBlockEncoding(encoding);
    HFileContext hFileContext = contextBuilder.build();
    try {
        Path familyPath = new Path(partition.getFilePath());
        // Get favored nodes as late as possible. This is the best we can do. If the region gets moved after this
        // point, locality is not guaranteed.
        InetSocketAddress favoredNode = getFavoredNode(partition);
        StoreFileWriter.Builder builder =
                new StoreFileWriter.Builder(conf, new CacheConfig(tempConf), new HFileSystem(fs))
                        .withOutputDir(familyPath).withBloomType(bloomType)
                        .withFileContext(hFileContext);

        if (favoredNode != null) {
            InetSocketAddress[] favoredNodes = new InetSocketAddress[1];
            favoredNodes[0] = favoredNode;
            builder.withFavoredNodes(favoredNodes);
        }

        return builder.build();
    } catch (Exception e) {
        throw new IOException(e);
    }
}
 

/**
 * Do data block encoding with specified options.
 * @param encoding What kind of data block encoding will be used.
 */
public HFileDataBlockEncoderImpl(DataBlockEncoding encoding) {
  this.encoding = encoding != null ? encoding : DataBlockEncoding.NONE;
}