org.apache.parquet.column.page.PageWriteStore Java Examples

The following examples show how to use org.apache.parquet.column.page.PageWriteStore. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ParquetWriter.java    From iceberg with Apache License 2.0 6 votes vote down vote up 
private void startRowGroup() {
  try {
    this.nextRowGroupSize = Math.min(writer.getNextRowGroupSize(), targetRowGroupSize);
  } catch (IOException e) {
    throw new RuntimeIOException(e);
  }
  this.nextCheckRecordCount = Math.min(Math.max(recordCount / 2, 100), 10000);
  this.recordCount = 0;

  PageWriteStore pageStore = pageStoreCtorParquet.newInstance(
      compressor, parquetSchema, props.getAllocator(), this.columnIndexTruncateLength);

  this.flushPageStoreToWriter = flushToWriter.bind(pageStore);
  this.writeStore = props.newColumnWriteStore(parquetSchema, pageStore);

  model.setColumnStore(writeStore);
}
Example #2
Source File: ParquetWriter.java    From iceberg with Apache License 2.0 6 votes vote down vote up 
private void startRowGroup() {
  try {
    this.nextRowGroupSize = min(writer.getNextRowGroupSize(), targetRowGroupSize);
  } catch (IOException e) {
    throw new RuntimeIOException(e);
  }
  this.nextCheckRecordCount = min(max(recordCount / 2, 100), 10000);
  this.recordCount = 0;

  PageWriteStore pageStore = pageStoreCtor.newInstance(
      compressor, parquetSchema, props.getAllocator());

  this.flushPageStoreToWriter = flushToWriter.bind(pageStore);
  this.writeStore = props.newColumnWriteStore(parquetSchema, pageStore);

  model.setColumnStore(writeStore);
}
Example #3
Source File: ColumnWriteStoreBase.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up 
@Deprecated
ColumnWriteStoreBase(
    final PageWriteStore pageWriteStore,
    final ParquetProperties props) {
  this.props = props;
  this.thresholdTolerance = (long) (props.getPageSizeThreshold() * THRESHOLD_TOLERANCE_RATIO);

  this.columns = new TreeMap<>();

  this.rowCountForNextSizeCheck = min(props.getMinRowCountForPageSizeCheck(), props.getPageRowCountLimit());

  columnWriterProvider = new ColumnWriterProvider() {
    @Override
    public ColumnWriter getColumnWriter(ColumnDescriptor path) {
      ColumnWriterBase column = columns.get(path);
      if (column == null) {
        column = createColumnWriter(path, pageWriteStore.getPageWriter(path), null, props);
        columns.put(path, column);
      }
      return column;
    }
  };
}
Example #4
Source File: ColumnWriteStoreBase.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up 
ColumnWriteStoreBase(
    MessageType schema,
    PageWriteStore pageWriteStore,
    ParquetProperties props) {
  this.props = props;
  this.thresholdTolerance = (long) (props.getPageSizeThreshold() * THRESHOLD_TOLERANCE_RATIO);
  Map<ColumnDescriptor, ColumnWriterBase> mcolumns = new TreeMap<>();
  for (ColumnDescriptor path : schema.getColumns()) {
    PageWriter pageWriter = pageWriteStore.getPageWriter(path);
    mcolumns.put(path, createColumnWriter(path, pageWriter, null, props));
  }
  this.columns = unmodifiableMap(mcolumns);

  this.rowCountForNextSizeCheck = min(props.getMinRowCountForPageSizeCheck(), props.getPageRowCountLimit());

  columnWriterProvider = new ColumnWriterProvider() {
    @Override
    public ColumnWriter getColumnWriter(ColumnDescriptor path) {
      return columns.get(path);
    }
  };
}
Example #5
Source File: ColumnWriteStoreBase.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up 
ColumnWriteStoreBase(
  MessageType schema,
  PageWriteStore pageWriteStore,
  BloomFilterWriteStore bloomFilterWriteStore,
  ParquetProperties props) {
  this.props = props;
  this.thresholdTolerance = (long) (props.getPageSizeThreshold() * THRESHOLD_TOLERANCE_RATIO);
  Map<ColumnDescriptor, ColumnWriterBase> mcolumns = new TreeMap<>();
  for (ColumnDescriptor path : schema.getColumns()) {
    PageWriter pageWriter = pageWriteStore.getPageWriter(path);
    if (props.isBloomFilterEnabled(path)) {
      BloomFilterWriter bloomFilterWriter = bloomFilterWriteStore.getBloomFilterWriter(path);
      mcolumns.put(path, createColumnWriter(path, pageWriter, bloomFilterWriter, props));
    } else {
      mcolumns.put(path, createColumnWriter(path, pageWriter, null, props));
    }
  }
  this.columns = unmodifiableMap(mcolumns);

  this.rowCountForNextSizeCheck = props.getMinRowCountForPageSizeCheck();

  columnWriterProvider = new ColumnWriterProvider() {
    @Override
    public ColumnWriter getColumnWriter(ColumnDescriptor path) {
      return columns.get(path);
    }
  };
}
Example #6
Source File: ParquetProperties.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up 
public ColumnWriteStore newColumnWriteStore(MessageType schema,
                                            PageWriteStore pageStore) {
  switch (writerVersion) {
    case PARQUET_1_0:
      return new ColumnWriteStoreV1(schema, pageStore, this);
    case PARQUET_2_0:
      return new ColumnWriteStoreV2(schema, pageStore, this);
    default:
      throw new IllegalArgumentException("unknown version " + writerVersion);
  }
}
Example #7
Source File: ParquetProperties.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up 
public ColumnWriteStore newColumnWriteStore(MessageType schema,
                                            PageWriteStore pageStore,
                                            BloomFilterWriteStore bloomFilterWriteStore) {
  switch (writerVersion) {
  case PARQUET_1_0:
    return new ColumnWriteStoreV1(schema, pageStore, bloomFilterWriteStore, this);
  case PARQUET_2_0:
    return new ColumnWriteStoreV2(schema, pageStore, bloomFilterWriteStore, this);
  default:
    throw new IllegalArgumentException("unknown version " + writerVersion);
  }
}
Example #8
Source File: ColumnChunkPageWriteStoreExposer.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up 
public static void flushPageStore(PageWriteStore pageStore, ParquetFileWriter w) throws IOException {
  ((ColumnChunkPageWriteStore) pageStore).flushToFileWriter(w);
}
Example #9
Source File: ColumnWriteStoreV1.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up 
public ColumnWriteStoreV1(MessageType schema, PageWriteStore pageWriteStore, ParquetProperties props) {
  super(schema, pageWriteStore, props);
}
Example #10
Source File: ColumnWriteStoreV1.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up 
@Deprecated
public ColumnWriteStoreV1(final PageWriteStore pageWriteStore,
    final ParquetProperties props) {
  super(pageWriteStore, props);
}
Example #11
Source File: ColumnWriteStoreV1.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up 
public ColumnWriteStoreV1(MessageType schema, PageWriteStore pageWriteStore,
                          BloomFilterWriteStore bloomFilterWriteStore,
                          ParquetProperties props) {
  super(schema, pageWriteStore, bloomFilterWriteStore, props);
}
Example #12
Source File: ColumnWriteStoreV2.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up 
public ColumnWriteStoreV2(MessageType schema, PageWriteStore pageWriteStore, ParquetProperties props) {
  super(schema, pageWriteStore, props);
}
Example #13
Source File: ColumnWriteStoreV2.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up 
public ColumnWriteStoreV2(MessageType schema, PageWriteStore pageWriteStore,
                          BloomFilterWriteStore bloomFilterWriteStore,
                          ParquetProperties props) {
  super(schema, pageWriteStore, bloomFilterWriteStore, props);
}