org.apache.parquet.format.Encoding Java Examples

VarLengthValuesColumn(ParquetRecordReader parentReader, ColumnDescriptor descriptor,
                      ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, V v,
                      SchemaElement schemaElement) throws ExecutionSetupException {

  super(parentReader, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
  variableWidthVector = (VariableWidthVector) valueVec;

  if (columnChunkMetaData.getEncodings().contains(Encoding.PLAIN_DICTIONARY)) {
    usingDictionary = true;
    // We didn't implement the fixed length optimization when a Parquet Dictionary is used; as there are
    // no data point about this use-case. Will also enable bulk processing by default since early data
    // profiling (for detecting the best processing strategy to use) is disabled when the column precision
    // is already set.
    bulkReaderState.columnPrecInfo.columnPrecisionType = ColumnPrecisionType.DT_PRECISION_IS_VARIABLE;
    bulkReaderState.columnPrecInfo.bulkProcess         = true;
  }
  else {
    usingDictionary = false;
  }
}
 

Set<org.apache.parquet.column.Encoding> fromFormatEncodings(List<Encoding> encodings) {
  Set<org.apache.parquet.column.Encoding> converted = new HashSet<org.apache.parquet.column.Encoding>();

  for (Encoding encoding : encodings) {
    converted.add(getEncoding(encoding));
  }

  // make converted unmodifiable, drop reference to modifiable copy
  converted = Collections.unmodifiableSet(converted);

  // atomically update the cache
  Set<org.apache.parquet.column.Encoding> cached = cachedEncodingSets.putIfAbsent(converted, converted);

  if (cached == null) {
    // cached == null signifies that converted was *not* in the cache previously
    // so we can return converted instead of throwing it away, it has now
    // been cached
    cached = converted;
  }

  return cached;
}
 

@Deprecated
public void writeDataPageHeader(
    int uncompressedSize,
    int compressedSize,
    int valueCount,
    org.apache.parquet.column.Encoding rlEncoding,
    org.apache.parquet.column.Encoding dlEncoding,
    org.apache.parquet.column.Encoding valuesEncoding,
    OutputStream to) throws IOException {
  writePageHeader(newDataPageHeader(uncompressedSize,
                                    compressedSize,
                                    valueCount,
                                    rlEncoding,
                                    dlEncoding,
                                    valuesEncoding), to);
}
 

public void writeDataPageV1Header(
    int uncompressedSize,
    int compressedSize,
    int valueCount,
    org.apache.parquet.column.Encoding rlEncoding,
    org.apache.parquet.column.Encoding dlEncoding,
    org.apache.parquet.column.Encoding valuesEncoding,
    int crc,
    OutputStream to) throws IOException {
  writePageHeader(newDataPageHeader(uncompressedSize,
                                    compressedSize,
                                    valueCount,
                                    rlEncoding,
                                    dlEncoding,
                                    valuesEncoding,
                                    crc), to);
}
 

private PageHeader newDataPageHeader(
    int uncompressedSize, int compressedSize,
    int valueCount,
    org.apache.parquet.column.Encoding rlEncoding,
    org.apache.parquet.column.Encoding dlEncoding,
    org.apache.parquet.column.Encoding valuesEncoding,
    int crc) {
  PageHeader pageHeader = new PageHeader(PageType.DATA_PAGE, uncompressedSize, compressedSize);
  pageHeader.setCrc(crc);
  pageHeader.setData_page_header(new DataPageHeader(
      valueCount,
      getEncoding(valuesEncoding),
      getEncoding(dlEncoding),
      getEncoding(rlEncoding)));
  return pageHeader;
}
 

public void writeDictionaryPageHeader(
    int uncompressedSize, int compressedSize, int valueCount,
    org.apache.parquet.column.Encoding valuesEncoding, int crc, OutputStream to) throws IOException {
  PageHeader pageHeader = new PageHeader(PageType.DICTIONARY_PAGE, uncompressedSize, compressedSize);
  pageHeader.setCrc(crc);
  pageHeader.setDictionary_page_header(new DictionaryPageHeader(valueCount, getEncoding(valuesEncoding)));
  writePageHeader(pageHeader, to);
}
 

VarLengthColumn(ParquetRecordReader parentReader, ColumnDescriptor descriptor,
                ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, V v,
                SchemaElement schemaElement) throws ExecutionSetupException {
  super(parentReader, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
    if (columnChunkMetaData.getEncodings().contains(Encoding.PLAIN_DICTIONARY)) {
      usingDictionary = true;
    }
    else {
      usingDictionary = false;
    }
}
 

public void writeDictionaryPageHeader(
  int uncompressedSize, int compressedSize, int valueCount,
  org.apache.parquet.column.Encoding valuesEncoding, OutputStream to) throws IOException {
  PageHeader pageHeader = new PageHeader(PageType.DICTIONARY_PAGE, uncompressedSize, compressedSize);
  pageHeader.setDictionary_page_header(new DictionaryPageHeader(valueCount, getEncoding(valuesEncoding)));
  writePageHeader(pageHeader, to);
}
 

private PageHeader newDataPageV2Header(
    int uncompressedSize, int compressedSize,
    int valueCount, int nullCount, int rowCount,
    org.apache.parquet.column.Encoding dataEncoding,
    int rlByteLength, int dlByteLength) {
  // TODO: pageHeader.crc = ...;
  DataPageHeaderV2 dataPageHeaderV2 = new DataPageHeaderV2(
      valueCount, nullCount, rowCount,
      getEncoding(dataEncoding),
      dlByteLength, rlByteLength);
  PageHeader pageHeader = new PageHeader(PageType.DATA_PAGE_V2, uncompressedSize, compressedSize);
  pageHeader.setData_page_header_v2(dataPageHeaderV2);
  return pageHeader;
}
 

public void writeDataPageV2Header(
    int uncompressedSize, int compressedSize,
    int valueCount, int nullCount, int rowCount,
    org.apache.parquet.column.Encoding dataEncoding,
    int rlByteLength, int dlByteLength,
    OutputStream to) throws IOException {
  writePageHeader(
      newDataPageV2Header(
          uncompressedSize, compressedSize,
          valueCount, nullCount, rowCount,
          dataEncoding,
          rlByteLength, dlByteLength), to);
}
 

public void writeDataPageV1Header(
  int uncompressedSize,
  int compressedSize,
  int valueCount,
  org.apache.parquet.column.Encoding rlEncoding,
  org.apache.parquet.column.Encoding dlEncoding,
  org.apache.parquet.column.Encoding valuesEncoding,
  OutputStream to) throws IOException {
  writePageHeader(newDataPageHeader(uncompressedSize,
    compressedSize,
    valueCount,
    rlEncoding,
    dlEncoding,
    valuesEncoding), to);
}
 

@Deprecated
public void writeDataPageV2Header(
    int uncompressedSize, int compressedSize,
    int valueCount, int nullCount, int rowCount,
    org.apache.parquet.column.statistics.Statistics statistics,
    org.apache.parquet.column.Encoding dataEncoding,
    int rlByteLength, int dlByteLength,
    OutputStream to) throws IOException {
  writePageHeader(
      newDataPageV2Header(
          uncompressedSize, compressedSize,
          valueCount, nullCount, rowCount,
          dataEncoding,
          rlByteLength, dlByteLength), to);
}
 

private PageHeader newDataPageHeader(
  int uncompressedSize, int compressedSize,
  int valueCount,
  org.apache.parquet.column.Encoding rlEncoding,
  org.apache.parquet.column.Encoding dlEncoding,
  org.apache.parquet.column.Encoding valuesEncoding) {
  PageHeader pageHeader = new PageHeader(PageType.DATA_PAGE, uncompressedSize, compressedSize);
  pageHeader.setData_page_header(new DataPageHeader(
    valueCount,
    getEncoding(valuesEncoding),
    getEncoding(dlEncoding),
    getEncoding(rlEncoding)));
  return pageHeader;
}
 

@Deprecated
public void writeDataPageHeader(
    int uncompressedSize,
    int compressedSize,
    int valueCount,
    org.apache.parquet.column.statistics.Statistics statistics,
    org.apache.parquet.column.Encoding rlEncoding,
    org.apache.parquet.column.Encoding dlEncoding,
    org.apache.parquet.column.Encoding valuesEncoding,
    OutputStream to) throws IOException {
  writePageHeader(
      newDataPageHeader(uncompressedSize, compressedSize, valueCount,
          rlEncoding, dlEncoding, valuesEncoding),
      to);
}
 

private List<Encoding> toFormatEncodings(Set<org.apache.parquet.column.Encoding> encodings) {
  List<Encoding> converted = new ArrayList<Encoding>(encodings.size());
  for (org.apache.parquet.column.Encoding encoding : encodings) {
    converted.add(getEncoding(encoding));
  }
  return converted;
}
 

private static Set<org.apache.parquet.column.Encoding> readEncodings(List<Encoding> encodings)
{
    Set<org.apache.parquet.column.Encoding> columnEncodings = new HashSet<>();
    for (Encoding encoding : encodings) {
        columnEncodings.add(org.apache.parquet.column.Encoding.valueOf(encoding.name()));
    }
    return Collections.unmodifiableSet(columnEncodings);
}
 

public Encoding getEncoding(org.apache.parquet.column.Encoding encoding) {
  return Encoding.valueOf(encoding.name());
}
 

public org.apache.parquet.column.Encoding getEncoding(Encoding encoding) {
  return org.apache.parquet.column.Encoding.valueOf(encoding.name());
}