Java Code Examples for org.apache.parquet.column.values.ValuesReader#skip()

@Test
public void testSkipInBinaryDictionary() throws Exception {
  ValuesWriter cw = newPlainBinaryDictionaryValuesWriter(1000, 10000);
  writeRepeated(100, cw, "a");
  writeDistinct(100, cw, "b");
  assertEquals(PLAIN_DICTIONARY, cw.getEncoding());

  // Test skip and skip-n with dictionary encoding
  ByteBufferInputStream stream = cw.getBytes().toInputStream();
  DictionaryValuesReader cr = initDicReader(cw, BINARY);
  cr.initFromPage(200, stream);
  for (int i = 0; i < 100; i += 2) {
    assertEquals(Binary.fromString("a" + i % 10), cr.readBytes());
    cr.skip();
  }
  int skipCount;
  for (int i = 0; i < 100; i += skipCount + 1) {
    skipCount = (100 - i) / 2;
    assertEquals(Binary.fromString("b" + i), cr.readBytes());
    cr.skip(skipCount);
  }

  // Ensure fallback
  writeDistinct(1000, cw, "c");
  assertEquals(PLAIN, cw.getEncoding());

  // Test skip and skip-n with plain encoding (after fallback)
  ValuesReader plainReader = new BinaryPlainValuesReader();
  plainReader.initFromPage(1200, cw.getBytes().toInputStream());
  plainReader.skip(200);
  for (int i = 0; i < 100; i += 2) {
    assertEquals("c" + i, plainReader.readBytes().toStringUsingUTF8());
    plainReader.skip();
  }
  for (int i = 100; i < 1000; i += skipCount + 1) {
    skipCount = (1000 - i) / 2;
    assertEquals(Binary.fromString("c" + i), plainReader.readBytes());
    plainReader.skip(skipCount);
  }
}
 

private void roundTripLong(FallbackValuesWriter<PlainLongDictionaryValuesWriter, PlainValuesWriter> cw,  ValuesReader reader, int maxDictionaryByteSize) throws IOException {
  int fallBackThreshold = maxDictionaryByteSize / 8;
  for (long i = 0; i < 100; i++) {
    cw.writeLong(i);
    if (i < fallBackThreshold) {
      assertEquals(cw.getEncoding(), PLAIN_DICTIONARY);
    } else {
      assertEquals(cw.getEncoding(), PLAIN);
    }
  }

  reader.initFromPage(100, cw.getBytes().toInputStream());

  for (long i = 0; i < 100; i++) {
    assertEquals(i, reader.readLong());
  }

  // Test skip with plain encoding
  reader.initFromPage(100, cw.getBytes().toInputStream());
  for (int i = 0; i < 100; i += 2) {
    assertEquals(i, reader.readLong());
    reader.skip();
  }

  // Test skip-n with plain encoding
  reader.initFromPage(100, cw.getBytes().toInputStream());
  int skipCount;
  for (int i = 0; i < 100; i += skipCount + 1) {
    skipCount = (100 - i) / 2;
    assertEquals(i, reader.readLong());
    reader.skip(skipCount);
  }
}
 

private void roundTripDouble(FallbackValuesWriter<PlainDoubleDictionaryValuesWriter, PlainValuesWriter> cw,  ValuesReader reader, int maxDictionaryByteSize) throws IOException {
  int fallBackThreshold = maxDictionaryByteSize / 8;
  for (double i = 0; i < 100; i++) {
    cw.writeDouble(i);
    if (i < fallBackThreshold) {
      assertEquals(cw.getEncoding(), PLAIN_DICTIONARY);
    } else {
      assertEquals(cw.getEncoding(), PLAIN);
    }
  }

  reader.initFromPage(100, cw.getBytes().toInputStream());

  for (double i = 0; i < 100; i++) {
    assertEquals(i, reader.readDouble(), 0.00001);
  }

  // Test skip with plain encoding
  reader.initFromPage(100, cw.getBytes().toInputStream());
  for (int i = 0; i < 100; i += 2) {
    assertEquals(i, reader.readDouble(), 0.0);
    reader.skip();
  }

  // Test skip-n with plain encoding
  reader.initFromPage(100, cw.getBytes().toInputStream());
  int skipCount;
  for (int i = 0; i < 100; i += skipCount + 1) {
    skipCount = (100 - i) / 2;
    assertEquals(i, reader.readDouble(), 0.0);
    reader.skip(skipCount);
  }
}
 

private void roundTripInt(FallbackValuesWriter<PlainIntegerDictionaryValuesWriter, PlainValuesWriter> cw,  ValuesReader reader, int maxDictionaryByteSize) throws IOException {
  int fallBackThreshold = maxDictionaryByteSize / 4;
  for (int i = 0; i < 100; i++) {
    cw.writeInteger(i);
    if (i < fallBackThreshold) {
      assertEquals(cw.getEncoding(), PLAIN_DICTIONARY);
    } else {
      assertEquals(cw.getEncoding(), PLAIN);
    }
  }

  reader.initFromPage(100, cw.getBytes().toInputStream());

  for (int i = 0; i < 100; i++) {
    assertEquals(i, reader.readInteger());
  }

  // Test skip with plain encoding
  reader.initFromPage(100, cw.getBytes().toInputStream());
  for (int i = 0; i < 100; i += 2) {
    assertEquals(i, reader.readInteger());
    reader.skip();
  }

  // Test skip-n with plain encoding
  reader.initFromPage(100, cw.getBytes().toInputStream());
  int skipCount;
  for (int i = 0; i < 100; i += skipCount + 1) {
    skipCount = (100 - i) / 2;
    assertEquals(i, reader.readInteger());
    reader.skip(skipCount);
  }
}
 

private void roundTripFloat(FallbackValuesWriter<PlainFloatDictionaryValuesWriter, PlainValuesWriter> cw,  ValuesReader reader, int maxDictionaryByteSize) throws IOException {
  int fallBackThreshold = maxDictionaryByteSize / 4;
  for (float i = 0; i < 100; i++) {
    cw.writeFloat(i);
    if (i < fallBackThreshold) {
      assertEquals(cw.getEncoding(), PLAIN_DICTIONARY);
    } else {
      assertEquals(cw.getEncoding(), PLAIN);
    }
  }

  reader.initFromPage(100, cw.getBytes().toInputStream());

  for (float i = 0; i < 100; i++) {
    assertEquals(i, reader.readFloat(), 0.00001);
  }

  // Test skip with plain encoding
  reader.initFromPage(100, cw.getBytes().toInputStream());
  for (int i = 0; i < 100; i += 2) {
    assertEquals(i, reader.readFloat(), 0.0f);
    reader.skip();
  }

  // Test skip-n with plain encoding
  reader.initFromPage(100, cw.getBytes().toInputStream());
  int skipCount;
  for (int i = 0; i < 100; i += skipCount + 1) {
    skipCount = (100 - i) / 2;
    assertEquals(i, reader.readFloat(), 0.0f);
    reader.skip(skipCount);
  }
}
 

private void validateEncodeDecode(int bitLength, int[] vals, String expected) throws IOException {
  for (PACKING_TYPE type : PACKING_TYPE.values()) {
    LOG.debug("{}", type);
    final int bound = (int)Math.pow(2, bitLength) - 1;
    ValuesWriter w = type.getWriter(bound);
    for (int i : vals) {
      w.writeInteger(i);
    }
    byte[] bytes = w.getBytes().toByteArray();
    LOG.debug("vals ("+bitLength+"): " + TestBitPacking.toString(vals));
    LOG.debug("bytes: {}", TestBitPacking.toString(bytes));
    assertEquals(type.toString(), expected, TestBitPacking.toString(bytes));
    ValuesReader r = type.getReader(bound);
    r.initFromPage(vals.length, ByteBufferInputStream.wrap(ByteBuffer.wrap(bytes)));
    int[] result = new int[vals.length];
    for (int i = 0; i < result.length; i++) {
      result[i] = r.readInteger();
    }
    LOG.debug("result: {}", TestBitPacking.toString(result));
    assertArrayEquals(type + " result: " + TestBitPacking.toString(result), vals, result);

    // Test skipping
    r.initFromPage(vals.length, ByteBufferInputStream.wrap(ByteBuffer.wrap(bytes)));
    for (int i = 0; i < vals.length; i += 2) {
      assertEquals(vals[i], r.readInteger());
      r.skip();
    }

    // Test n-skipping
    r.initFromPage(vals.length, ByteBufferInputStream.wrap(ByteBuffer.wrap(bytes)));
    int skipCount;
    for (int i = 0; i < vals.length; i += skipCount + 1) {
      skipCount = (vals.length - i) / 2;
      assertEquals(vals[i], r.readInteger());
      r.skip(skipCount);
    }
  }
}