Java Code Examples for org.apache.parquet.Preconditions#checkArgument()

protected DecimalMetadata decimalMetadata() {
  DecimalMetadata meta = null;
  if (logicalTypeAnnotation instanceof LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) {
    LogicalTypeAnnotation.DecimalLogicalTypeAnnotation decimalType = (LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) logicalTypeAnnotation;
    if (newLogicalTypeSet) {
      if (scaleAlreadySet) {
        Preconditions.checkArgument(this.scale == decimalType.getScale(),
          "Decimal scale should match with the scale of the logical type");
      }
      if (precisionAlreadySet) {
        Preconditions.checkArgument(this.precision == decimalType.getPrecision(),
          "Decimal precision should match with the precision of the logical type");
      }
      scale = decimalType.getScale();
      precision = decimalType.getPrecision();
    }
    Preconditions.checkArgument(precision > 0,
        "Invalid DECIMAL precision: " + precision);
    Preconditions.checkArgument(this.scale >= 0,
        "Invalid DECIMAL scale: " + this.scale);
    Preconditions.checkArgument(this.scale <= precision,
        "Invalid DECIMAL scale: cannot be greater than precision");
    meta = new DecimalMetadata(precision, scale);
  }
  return meta;
}
 

/**
 * Calculate optimal size according to the number of distinct values and false positive probability.
 *
 * @param n: The number of distinct values.
 * @param p: The false positive probability.
 *
 * @return optimal number of bits of given n and p.
 */
public static int optimalNumOfBits(long n, double p) {
  Preconditions.checkArgument((p > 0.0 && p < 1.0),
    "FPP should be less than 1.0 and great than 0.0");
  final double m = -8 * n / Math.log(1 - Math.pow(p, 1.0 / 8));
  int numBits = (int)m ;

  // Handle overflow.
  if (numBits > UPPER_BOUND_BYTES << 3 || m < 0) {
    numBits = UPPER_BOUND_BYTES << 3;
  }

  // Round numBits up to (k * BITS_PER_BLOCK)
  numBits = (numBits + BITS_PER_BLOCK -1) & ~BITS_PER_BLOCK;

  if (numBits < (LOWER_BOUND_BYTES << 3)) {
    numBits = LOWER_BOUND_BYTES << 3;
  }

  return numBits;
}
 

/**
  * Fills specified buffer with uncompressed data. Returns actual number
  * of bytes of uncompressed data. A return value of 0 indicates that
  * {@link #needsInput()} should be called in order to determine if more
  * input data is required.
  *
  * @param buffer   Buffer for the compressed data
  * @param off Start offset of the data
  * @param len Size of the buffer
  * @return The actual number of bytes of uncompressed data.
  * @throws IOException if reading or decompression fails
  */
 @Override
 public synchronized int decompress(byte[] buffer, int off, int len) throws IOException {
   SnappyUtil.validateBuffer(buffer, off, len);
if (inputBuffer.position() == 0 && !outputBuffer.hasRemaining()) {
     return 0;
   }
   
   if (!outputBuffer.hasRemaining()) {
     inputBuffer.rewind();
     Preconditions.checkArgument(inputBuffer.position() == 0, "Invalid position of 0.");
     Preconditions.checkArgument(outputBuffer.position() == 0, "Invalid position of 0.");
     // There is compressed input, decompress it now.
     int decompressedSize = Snappy.uncompressedLength(inputBuffer);
     if (decompressedSize > outputBuffer.capacity()) {
       ByteBuffer oldBuffer = outputBuffer;
       outputBuffer = ByteBuffer.allocateDirect(decompressedSize);
       CleanUtil.cleanDirectBuffer(oldBuffer);
     }

     // Reset the previous outputBuffer (i.e. set position to 0)
     outputBuffer.clear();
     int size = Snappy.uncompress(inputBuffer, outputBuffer);
     outputBuffer.limit(size);
     // We've decompressed the entire input, reset the input now
     inputBuffer.clear();
     inputBuffer.limit(0);
     finished = true;
   }

   // Return compressed output up to 'len'
   int numBytes = Math.min(len, outputBuffer.remaining());
   outputBuffer.get(buffer, off, numBytes);
   return numBytes;	    
 }
 

/**
 * Set the Parquet format dictionary page size.
 *
 * @param dictionaryPageSize an integer size in bytes
 * @return this builder for method chaining.
 */
public Builder withDictionaryPageSize(int dictionaryPageSize) {
  Preconditions.checkArgument(dictionaryPageSize > 0,
      "Invalid dictionary page size (negative): %s", dictionaryPageSize);
  this.dictPageSize = dictionaryPageSize;
  return this;
}
 

public RunLengthBitPackingHybridDecoder(int bitWidth, InputStream in) {
  LOG.debug("decoding bitWidth {}", bitWidth);

  Preconditions.checkArgument(bitWidth >= 0 && bitWidth <= 32, "bitWidth must be >= 0 and <= 32");
  this.bitWidth = bitWidth;
  this.packer = Packer.LITTLE_ENDIAN.newBytePacker(bitWidth);
  this.in = in;
}
 

public RunLengthBitPackingHybridEncoder(int bitWidth, int initialCapacity, int pageSize, ByteBufferAllocator allocator) {
  LOG.debug("Encoding: RunLengthBitPackingHybridEncoder with "
    + "bithWidth: {} initialCapacity {}", bitWidth, initialCapacity);

  Preconditions.checkArgument(bitWidth >= 0 && bitWidth <= 32, "bitWidth must be >= 0 and <= 32");

  this.bitWidth = bitWidth;
  this.baos = new CapacityByteArrayOutputStream(initialCapacity, pageSize, allocator);
  this.packBuffer = new byte[bitWidth];
  this.bufferedValues = new int[8];
  this.packer = Packer.LITTLE_ENDIAN.newBytePacker(bitWidth);
  reset(false);
}
 

/**
 * Decodes {@link Binary} values from a {@link DictionaryPage}.
 *
 * If the given {@code length} is null, the values will be read as length-
 * prefixed values with a 4-byte little-endian length. If length is not
 * null, it will be used as the length for all fixed-length {@code Binary}
 * values read from the page.
 *
 * @param dictionaryPage a {@code DictionaryPage} of encoded binary values
 * @param length a fixed length of binary arrays, or null if not fixed
 * @throws IOException if there is an exception while decoding the dictionary page
 */
public PlainBinaryDictionary(DictionaryPage dictionaryPage, Integer length) throws IOException {
  super(dictionaryPage);
  final ByteBuffer dictionaryBytes = dictionaryPage.getBytes().toByteBuffer();
  binaryDictionaryContent = new Binary[dictionaryPage.getDictionarySize()];
  // dictionary values are stored in order: size (4 bytes LE) followed by {size} bytes
  int offset = dictionaryBytes.position();
  if (length == null) {
    // dictionary values are stored in order: size (4 bytes LE) followed by {size} bytes
    for (int i = 0; i < binaryDictionaryContent.length; i++) {
      int len = readIntLittleEndian(dictionaryBytes, offset);
      // read the length
      offset += 4;
      // wrap the content in a binary
      binaryDictionaryContent[i] = Binary.fromConstantByteBuffer(dictionaryBytes, offset, len);
      // increment to the next value
      offset += len;
    }
  } else {
    // dictionary values are stored as fixed-length arrays
    Preconditions.checkArgument(length > 0,
        "Invalid byte array length: " + length);
    for (int i = 0; i < binaryDictionaryContent.length; i++) {
      // wrap the content in a Binary
      binaryDictionaryContent[i] = Binary.fromConstantByteBuffer(
          dictionaryBytes, offset, length);
      // increment to the next value
      offset += length;
    }
  }
}
 

public Object borrowDirectDecompressor(){
  Preconditions.checkArgument(supportDirectDecompressor, "Tried to get a direct Decompressor from a non-direct codec.");
  try {
    return directDecompressorPool.borrowObject();
  } catch (Exception e) {
    throw new ParquetCompressionCodecException(e);
  }
}
 

/**
 * Set the Parquet format page size.
 *
 * @param pageSize an integer size in bytes
 * @return this builder for method chaining.
 */
public Builder withPageSize(int pageSize) {
  Preconditions.checkArgument(pageSize > 0,
      "Invalid page size (negative): %s", pageSize);
  this.pageSize = pageSize;
  return this;
}
 

/**
 * Initializes the internal state for decoding ints of `bitWidth`.
 */
private void initWidthAndPacker(int bitWidth) {
	Preconditions.checkArgument(bitWidth >= 0 && bitWidth <= 32, "bitWidth must be >= 0 and <= 32");
	this.bitWidth = bitWidth;
	this.bytesWidth = BytesUtils.paddedByteCountFromBits(bitWidth);
	this.packer = Packer.LITTLE_ENDIAN.newBytePacker(bitWidth);
}
 

protected void checkTypeName(PrimitiveType.PrimitiveTypeName expectedName) {
	PrimitiveType.PrimitiveTypeName actualName = descriptor.getPrimitiveType().getPrimitiveTypeName();
	Preconditions.checkArgument(
			actualName == expectedName,
			"Expected type name: %s, actual type name: %s",
			expectedName,
			actualName);
}
 

public void writeList(GroupType schema, Schema avroSchema, Object value) {
  recordConsumer.startGroup(); // group wrapper (original type LIST)
  if (value instanceof Collection) {
    writeCollection(schema, avroSchema, (Collection) value);
  } else {
    Class<?> arrayClass = value.getClass();
    Preconditions.checkArgument(arrayClass.isArray(),
        "Cannot write unless collection or array: " + arrayClass.getName());
    writeJavaArray(schema, avroSchema, arrayClass, value);
  }
  recordConsumer.endGroup();
}
 

private ColumnOrder requireValidColumnOrder(ColumnOrder columnOrder) {
  if (primitive == PrimitiveTypeName.INT96) {
    Preconditions.checkArgument(columnOrder.getColumnOrderName() == ColumnOrderName.UNDEFINED,
        "The column order {} is not supported by INT96", columnOrder);
  }
  if (getLogicalTypeAnnotation() != null) {
    Preconditions.checkArgument(getLogicalTypeAnnotation().isValidColumnOrder(columnOrder),
      "The column order {} is not supported by {} ({})", columnOrder, primitive, getLogicalTypeAnnotation());
  }
  return columnOrder;
}
 

@Override
public void decompress(ByteBuffer input, int compressedSize, ByteBuffer output, int uncompressedSize)
    throws IOException {
  Preconditions.checkArgument(compressedSize == uncompressedSize,
      "Non-compressed data did not have matching compressed and uncompressed sizes.");
  output.clear();
  output.put((ByteBuffer) input.duplicate().position(0).limit(compressedSize));
}
 

public Builder withColumnIndexTruncateLength(int length) {
  Preconditions.checkArgument(length > 0, "Invalid column index min/max truncate length (negative or zero) : %s", length);
  this.columnIndexTruncateLength = length;
  return this;
}
 

@Override
public Converter getConverter(int fieldIndex) {
  Preconditions.checkArgument(
      fieldIndex == 0, "Illegal field index: " + fieldIndex);
  return elementConverter;
}
 

@Override
public Converter getConverter(int fieldIndex) {
  Preconditions.checkArgument(
      fieldIndex == 0, "Illegal field index: " + fieldIndex);
  return elementConverter;
}
 

protected ReadSupport<T> getReadSupport() {
  // if readSupport is null, the protected constructor must have been used
  Preconditions.checkArgument(readSupport != null,
      "[BUG] Classes that extend Builder should override getReadSupport()");
  return readSupport;
}
 

/**
 * Construct a type builder that returns the {@link Type} that was built
 * when the builder is finished. The {@code returnClass} must be the
 * expected {@code Type} class.
 *
 * @param returnClass a {@code Type} to return from {@link #named(String)}
 */
protected Builder(Class<P> returnClass) {
  Preconditions.checkArgument(Type.class.isAssignableFrom(returnClass),
      "The requested return class must extend Type");
  this.returnClass = returnClass;
  this.parent = null;
}
 

/**
 * Creates a 3-level list structure annotated with LIST with elements of the
 * given elementType. The repeated level is inserted automatically and the
 * elementType's repetition should be the correct repetition of the elements,
 * required for non-null and optional for nullable.
 *
 * @param listRepetition the repetition of the entire list structure
 * @param name the name of the list structure type
 * @param elementType the type of elements contained by the list
 * @return a GroupType that represents the list
 */
public static GroupType listOfElements(Repetition listRepetition, String name, Type elementType) {
  Preconditions.checkArgument(elementType.getName().equals(ELEMENT_NAME),
      "List element type must be named 'element'");
  return listWrapper(
      listRepetition,
      name,
      LogicalTypeAnnotation.listType(),
      new GroupType(Repetition.REPEATED, "list", elementType)
  );
}