org.apache.orc.StripeInformation Java Examples

The following examples show how to use org.apache.orc.StripeInformation. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: OrcRowInputFormat.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private Tuple2<Long, Long> getOffsetAndLengthForSplit(FileInputSplit split, List<StripeInformation> stripes) {
	long splitStart = split.getStart();
	long splitEnd = splitStart + split.getLength();

	long readStart = Long.MAX_VALUE;
	long readEnd = Long.MIN_VALUE;

	for (StripeInformation s : stripes) {
		if (splitStart <= s.getOffset() && s.getOffset() < splitEnd) {
			// stripe starts in split, so it is included
			readStart = Math.min(readStart, s.getOffset());
			readEnd = Math.max(readEnd, s.getOffset() + s.getLength());
		}
	}

	if (readStart < Long.MAX_VALUE) {
		// at least one split is included
		return Tuple2.of(readStart, readEnd - readStart);
	} else {
		return Tuple2.of(0L, 0L);
	}
}
Example #2
Source File: OrcRowInputFormat.java    From flink with Apache License 2.0 6 votes vote down vote up 
private Tuple2<Long, Long> getOffsetAndLengthForSplit(FileInputSplit split, List<StripeInformation> stripes) {
	long splitStart = split.getStart();
	long splitEnd = splitStart + split.getLength();

	long readStart = Long.MAX_VALUE;
	long readEnd = Long.MIN_VALUE;

	for (StripeInformation s : stripes) {
		if (splitStart <= s.getOffset() && s.getOffset() < splitEnd) {
			// stripe starts in split, so it is included
			readStart = Math.min(readStart, s.getOffset());
			readEnd = Math.max(readEnd, s.getOffset() + s.getLength());
		}
	}

	if (readStart < Long.MAX_VALUE) {
		// at least one split is included
		return Tuple2.of(readStart, readEnd - readStart);
	} else {
		return Tuple2.of(0L, 0L);
	}
}
Example #3
Source File: DremioORCRecordUtils.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up 
/**
 * This function is a copy of original implementation from hive-private repository
 */
@Override
public OrcProto.StripeFooter readStripeFooter(StripeInformation stripe) throws IOException {
  if (file == null) {
    open();
  }
  long offset = stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength();
  int tailLength = (int) stripe.getFooterLength();

  // read the footer
  ByteBuffer tailBuf = ByteBuffer.allocate(tailLength);
  file.readFully(offset, tailBuf.array(), tailBuf.arrayOffset(), tailLength);
  return OrcProto.StripeFooter.parseFrom(InStream.createCodedInputStream("footer",
    Lists.<DiskRange>newArrayList(new BufferChunk(tailBuf, 0)),
    tailLength, codec, bufferSize));
}
Example #4
Source File: OrcShimV200.java    From flink with Apache License 2.0 6 votes vote down vote up 
@VisibleForTesting
public static Tuple2<Long, Long> getOffsetAndLengthForSplit(
		long splitStart, long splitLength, List<StripeInformation> stripes) {
	long splitEnd = splitStart + splitLength;
	long readStart = Long.MAX_VALUE;
	long readEnd = Long.MIN_VALUE;

	for (StripeInformation s : stripes) {
		if (splitStart <= s.getOffset() && s.getOffset() < splitEnd) {
			// stripe starts in split, so it is included
			readStart = Math.min(readStart, s.getOffset());
			readEnd = Math.max(readEnd, s.getOffset() + s.getLength());
		}
	}

	if (readStart < Long.MAX_VALUE) {
		// at least one split is included
		return Tuple2.of(readStart, readEnd - readStart);
	} else {
		return Tuple2.of(0L, 0L);
	}
}
Example #5
Source File: OrcRowInputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private static void assertOffsetAndLen(
		OrcSplitReader reader, long offset, long length) throws IllegalAccessException {
	List<StripeInformation> stripes = getStripes(reader.getRecordReader());
	long min = Long.MAX_VALUE;
	long max = Long.MIN_VALUE;
	for (StripeInformation stripe : stripes) {
		if (stripe.getOffset() < min) {
			min = stripe.getOffset();
		}
		if (stripe.getOffset() + stripe.getLength() > max) {
			max = stripe.getOffset() + stripe.getLength();
		}
	}

	assertEquals(offset, min);
	assertEquals(length, max - min);
}
Example #6
Source File: OrcFileAppender.java    From iceberg with Apache License 2.0 5 votes vote down vote up 
@Override
public List<Long> splitOffsets() {
  Preconditions.checkState(isClosed, "File is not yet closed");
  try (Reader reader = ORC.newFileReader(file.toInputFile(), conf)) {
    List<StripeInformation> stripes = reader.getStripes();
    return Collections.unmodifiableList(Lists.transform(stripes, StripeInformation::getOffset));
  } catch (IOException e) {
    throw new RuntimeIOException(e, "Can't close ORC reader %s", file.location());
  }
}
Example #7
Source File: DremioORCRecordUtils.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up 
@Override
public OrcProto.StripeFooter readStripeFooter(StripeInformation stripe) throws IOException {
  if (file == null) {
    open();
  }
  long offset = stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength();
  int tailLength = (int) stripe.getFooterLength();

  // read the footer
  ByteBuffer tailBuf = ByteBuffer.allocate(tailLength);
  file.readFully(offset, tailBuf.array(), tailBuf.arrayOffset(), tailLength);
  return OrcProto.StripeFooter.parseFrom(
    InStream.createCodedInputStream("footer", singleton(
      new BufferChunk(tailBuf, 0)), tailLength, codec, bufferSize));
}
Example #8
Source File: OrcRowInputFormatTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSplitStripesCustomSplits() throws IOException {
	// mock list of stripes
	List<StripeInformation> stripes = new ArrayList<>();
	StripeInformation stripe1 = mock(StripeInformation.class);
	when(stripe1.getOffset()).thenReturn(10L);
	when(stripe1.getLength()).thenReturn(90L);
	StripeInformation stripe2 = mock(StripeInformation.class);
	when(stripe2.getOffset()).thenReturn(100L);
	when(stripe2.getLength()).thenReturn(100L);
	StripeInformation stripe3 = mock(StripeInformation.class);
	when(stripe3.getOffset()).thenReturn(200L);
	when(stripe3.getLength()).thenReturn(100L);
	StripeInformation stripe4 = mock(StripeInformation.class);
	when(stripe4.getOffset()).thenReturn(300L);
	when(stripe4.getLength()).thenReturn(100L);
	StripeInformation stripe5 = mock(StripeInformation.class);
	when(stripe5.getOffset()).thenReturn(400L);
	when(stripe5.getLength()).thenReturn(100L);
	stripes.add(stripe1);
	stripes.add(stripe2);
	stripes.add(stripe3);
	stripes.add(stripe4);
	stripes.add(stripe5);

	// split ranging 2 stripes
	assertEquals(new Tuple2<>(10L, 190L), getOffsetAndLengthForSplit(0, 150, stripes));

	// split ranging 0 stripes
	assertEquals(new Tuple2<>(0L, 0L), getOffsetAndLengthForSplit(150, 10, stripes));

	// split ranging 1 stripe
	assertEquals(new Tuple2<>(200L, 100L), getOffsetAndLengthForSplit(160, 41, stripes));

	// split ranging 2 stripe
	assertEquals(new Tuple2<>(300L, 200L), getOffsetAndLengthForSplit(201, 299, stripes));

}
Example #9
Source File: OrcRowInputFormat.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@VisibleForTesting
List<StripeInformation> getStripes(Reader orcReader) {
	return orcReader.getStripes();
}
Example #10
Source File: OrcRowInputFormatTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Test
public void testSplitStripesCustomSplits() throws IOException {
	rowOrcInputFormat =
		new OrcRowInputFormat(getPath(TEST_FILE_FLAT), TEST_SCHEMA_FLAT, new Configuration());

	OrcRowInputFormat spy = spy(rowOrcInputFormat);

	// mock list of stripes
	List<StripeInformation> stripes = new ArrayList<>();
	StripeInformation stripe1 = mock(StripeInformation.class);
	when(stripe1.getOffset()).thenReturn(10L);
	when(stripe1.getLength()).thenReturn(90L);
	StripeInformation stripe2 = mock(StripeInformation.class);
	when(stripe2.getOffset()).thenReturn(100L);
	when(stripe2.getLength()).thenReturn(100L);
	StripeInformation stripe3 = mock(StripeInformation.class);
	when(stripe3.getOffset()).thenReturn(200L);
	when(stripe3.getLength()).thenReturn(100L);
	StripeInformation stripe4 = mock(StripeInformation.class);
	when(stripe4.getOffset()).thenReturn(300L);
	when(stripe4.getLength()).thenReturn(100L);
	StripeInformation stripe5 = mock(StripeInformation.class);
	when(stripe5.getOffset()).thenReturn(400L);
	when(stripe5.getLength()).thenReturn(100L);
	stripes.add(stripe1);
	stripes.add(stripe2);
	stripes.add(stripe3);
	stripes.add(stripe4);
	stripes.add(stripe5);
	doReturn(stripes).when(spy).getStripes(any());

	// mock options to check configuration of ORC reader
	Reader.Options options = spy(new Reader.Options());
	doReturn(options).when(spy).getOptions(any());

	spy.openInputFormat();
	// split ranging 2 stripes
	spy.open(new FileInputSplit(0, new Path(getPath(TEST_FILE_FLAT)), 0, 150, new String[]{}));
	verify(options).range(eq(10L), eq(190L));
	// split ranging 0 stripes
	spy.open(new FileInputSplit(1, new Path(getPath(TEST_FILE_FLAT)), 150, 10, new String[]{}));
	verify(options).range(eq(0L), eq(0L));
	// split ranging 1 stripe
	spy.open(new FileInputSplit(2, new Path(getPath(TEST_FILE_FLAT)), 160, 41, new String[]{}));
	verify(options).range(eq(200L), eq(100L));
	// split ranging 2 stripe
	spy.open(new FileInputSplit(3, new Path(getPath(TEST_FILE_FLAT)), 201, 299, new String[]{}));
	verify(options).range(eq(300L), eq(200L));
}
Example #11
Source File: OrcRowInputFormat.java    From flink with Apache License 2.0 4 votes vote down vote up 
@VisibleForTesting
List<StripeInformation> getStripes(Reader orcReader) {
	return orcReader.getStripes();
}
Example #12
Source File: OrcRowInputFormatTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Test
public void testSplitStripesCustomSplits() throws IOException {
	rowOrcInputFormat =
		new OrcRowInputFormat(getPath(TEST_FILE_FLAT), TEST_SCHEMA_FLAT, new Configuration());

	OrcRowInputFormat spy = spy(rowOrcInputFormat);

	// mock list of stripes
	List<StripeInformation> stripes = new ArrayList<>();
	StripeInformation stripe1 = mock(StripeInformation.class);
	when(stripe1.getOffset()).thenReturn(10L);
	when(stripe1.getLength()).thenReturn(90L);
	StripeInformation stripe2 = mock(StripeInformation.class);
	when(stripe2.getOffset()).thenReturn(100L);
	when(stripe2.getLength()).thenReturn(100L);
	StripeInformation stripe3 = mock(StripeInformation.class);
	when(stripe3.getOffset()).thenReturn(200L);
	when(stripe3.getLength()).thenReturn(100L);
	StripeInformation stripe4 = mock(StripeInformation.class);
	when(stripe4.getOffset()).thenReturn(300L);
	when(stripe4.getLength()).thenReturn(100L);
	StripeInformation stripe5 = mock(StripeInformation.class);
	when(stripe5.getOffset()).thenReturn(400L);
	when(stripe5.getLength()).thenReturn(100L);
	stripes.add(stripe1);
	stripes.add(stripe2);
	stripes.add(stripe3);
	stripes.add(stripe4);
	stripes.add(stripe5);
	doReturn(stripes).when(spy).getStripes(any());

	// mock options to check configuration of ORC reader
	Reader.Options options = spy(new Reader.Options());
	doReturn(options).when(spy).getOptions(any());

	spy.openInputFormat();
	// split ranging 2 stripes
	spy.open(new FileInputSplit(0, new Path(getPath(TEST_FILE_FLAT)), 0, 150, new String[]{}));
	verify(options).range(eq(10L), eq(190L));
	// split ranging 0 stripes
	spy.open(new FileInputSplit(1, new Path(getPath(TEST_FILE_FLAT)), 150, 10, new String[]{}));
	verify(options).range(eq(0L), eq(0L));
	// split ranging 1 stripe
	spy.open(new FileInputSplit(2, new Path(getPath(TEST_FILE_FLAT)), 160, 41, new String[]{}));
	verify(options).range(eq(200L), eq(100L));
	// split ranging 2 stripe
	spy.open(new FileInputSplit(3, new Path(getPath(TEST_FILE_FLAT)), 201, 299, new String[]{}));
	verify(options).range(eq(300L), eq(200L));
}
Example #13
Source File: DremioORCRecordUtils.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up 
/**
 * This function is a copy of original implementation from hive-private repository
 */
@Override
public OrcIndex readRowIndex(StripeInformation stripe,
                             OrcProto.StripeFooter footer,
                             boolean[] included,
                             OrcProto.RowIndex[] indexes,
                             boolean[] sargColumns,
                             OrcProto.BloomFilterIndex[] bloomFilterIndices
) throws IOException {
  if (file == null) {
    open();
  }
  if (footer == null) {
    footer = readStripeFooter(stripe);
  }
  if (indexes == null) {
    indexes = new OrcProto.RowIndex[typeCount];
  }
  if (bloomFilterIndices == null) {
    bloomFilterIndices = new OrcProto.BloomFilterIndex[typeCount];
  }
  long offset = stripe.getOffset();
  List<OrcProto.Stream> streams = footer.getStreamsList();
  for (int i = 0; i < streams.size(); i++) {
    OrcProto.Stream stream = streams.get(i);
    OrcProto.Stream nextStream = null;
    if (i < streams.size() - 1) {
      nextStream = streams.get(i+1);
    }
    int col = stream.getColumn();
    int len = (int) stream.getLength();
    // row index stream and bloom filter are interlaced, check if the sarg column contains bloom
    // filter and combine the io to read row index and bloom filters for that column together
    if (stream.hasKind() && (stream.getKind() == OrcProto.Stream.Kind.ROW_INDEX)) {
      boolean readBloomFilter = false;
      if (sargColumns != null && sargColumns[col] &&
        nextStream.getKind() == OrcProto.Stream.Kind.BLOOM_FILTER) {
        len += nextStream.getLength();
        i += 1;
        readBloomFilter = true;
      }
      if ((included == null || included[col]) && indexes[col] == null) {
        byte[] buffer = new byte[len];
        file.readFully(offset, buffer, 0, buffer.length);
        ByteBuffer bb = ByteBuffer.wrap(buffer);
        ByteBuffer rowIndexBB = bb.duplicate();
        rowIndexBB.position(0);
        rowIndexBB.limit((int)stream.getLength());
        indexes[col] = OrcProto.RowIndex.parseFrom(InStream.create("index",
          Lists.<DiskRange>newArrayList(new BufferChunk(rowIndexBB, 0)), stream.getLength(),
          codec, bufferSize, null));
        if (readBloomFilter) {
          ByteBuffer bloomFilterBB = bb.duplicate();
          bloomFilterBB.position((int)stream.getLength());
          bloomFilterBB.limit(buffer.length);
          bloomFilterIndices[col] = OrcProto.BloomFilterIndex.parseFrom(InStream.create(
            "bloom_filter", Lists.<DiskRange>newArrayList(new BufferChunk(bloomFilterBB, 0)),
            nextStream.getLength(), codec, bufferSize, null));
        }
      }
    }
    offset += len;
  }

  OrcIndex index = new OrcIndex(indexes, bloomFilterIndices);
  return index;
}
Example #14
Source File: OrcRowInputFormatTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
private static List<StripeInformation> getStripes(RecordReader reader) throws IllegalAccessException {
	return (List<StripeInformation>) readDeclaredField(reader, "stripes", true);
}
Example #15
Source File: Writer.java    From tajo with Apache License 2.0 2 votes vote down vote up 
/**
 * Fast stripe append to ORC file. This interface is used for fast ORC file
 * merge with other ORC files. When merging, the file to be merged should pass
 * stripe in binary form along with stripe information and stripe statistics.
 * After appending last stripe of a file, use appendUserMetadata() to append
 * any user metadata.
 * @param stripe - stripe as byte array
 * @param offset - offset within byte array
 * @param length - length of stripe within byte array
 * @param stripeInfo - stripe information
 * @param stripeStatistics - stripe statistics (Protobuf objects can be
 *                         merged directly)
 * @throws IOException
 */
public void appendStripe(byte[] stripe, int offset, int length,
                         StripeInformation stripeInfo,
                         OrcProto.StripeStatistics stripeStatistics) throws IOException;