Java Code Examples for org.apache.orc.StripeInformation#getOffset()

The following examples show how to use org.apache.orc.StripeInformation#getOffset() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: OrcRowInputFormat.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private Tuple2<Long, Long> getOffsetAndLengthForSplit(FileInputSplit split, List<StripeInformation> stripes) {
	long splitStart = split.getStart();
	long splitEnd = splitStart + split.getLength();

	long readStart = Long.MAX_VALUE;
	long readEnd = Long.MIN_VALUE;

	for (StripeInformation s : stripes) {
		if (splitStart <= s.getOffset() && s.getOffset() < splitEnd) {
			// stripe starts in split, so it is included
			readStart = Math.min(readStart, s.getOffset());
			readEnd = Math.max(readEnd, s.getOffset() + s.getLength());
		}
	}

	if (readStart < Long.MAX_VALUE) {
		// at least one split is included
		return Tuple2.of(readStart, readEnd - readStart);
	} else {
		return Tuple2.of(0L, 0L);
	}
}
Example 2
Source File: OrcRowInputFormat.java    From flink with Apache License 2.0 6 votes vote down vote up 
private Tuple2<Long, Long> getOffsetAndLengthForSplit(FileInputSplit split, List<StripeInformation> stripes) {
	long splitStart = split.getStart();
	long splitEnd = splitStart + split.getLength();

	long readStart = Long.MAX_VALUE;
	long readEnd = Long.MIN_VALUE;

	for (StripeInformation s : stripes) {
		if (splitStart <= s.getOffset() && s.getOffset() < splitEnd) {
			// stripe starts in split, so it is included
			readStart = Math.min(readStart, s.getOffset());
			readEnd = Math.max(readEnd, s.getOffset() + s.getLength());
		}
	}

	if (readStart < Long.MAX_VALUE) {
		// at least one split is included
		return Tuple2.of(readStart, readEnd - readStart);
	} else {
		return Tuple2.of(0L, 0L);
	}
}
Example 3
Source File: DremioORCRecordUtils.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up 
/**
 * This function is a copy of original implementation from hive-private repository
 */
@Override
public OrcProto.StripeFooter readStripeFooter(StripeInformation stripe) throws IOException {
  if (file == null) {
    open();
  }
  long offset = stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength();
  int tailLength = (int) stripe.getFooterLength();

  // read the footer
  ByteBuffer tailBuf = ByteBuffer.allocate(tailLength);
  file.readFully(offset, tailBuf.array(), tailBuf.arrayOffset(), tailLength);
  return OrcProto.StripeFooter.parseFrom(InStream.createCodedInputStream("footer",
    Lists.<DiskRange>newArrayList(new BufferChunk(tailBuf, 0)),
    tailLength, codec, bufferSize));
}
Example 4
Source File: OrcShimV200.java    From flink with Apache License 2.0 6 votes vote down vote up 
@VisibleForTesting
public static Tuple2<Long, Long> getOffsetAndLengthForSplit(
		long splitStart, long splitLength, List<StripeInformation> stripes) {
	long splitEnd = splitStart + splitLength;
	long readStart = Long.MAX_VALUE;
	long readEnd = Long.MIN_VALUE;

	for (StripeInformation s : stripes) {
		if (splitStart <= s.getOffset() && s.getOffset() < splitEnd) {
			// stripe starts in split, so it is included
			readStart = Math.min(readStart, s.getOffset());
			readEnd = Math.max(readEnd, s.getOffset() + s.getLength());
		}
	}

	if (readStart < Long.MAX_VALUE) {
		// at least one split is included
		return Tuple2.of(readStart, readEnd - readStart);
	} else {
		return Tuple2.of(0L, 0L);
	}
}
Example 5
Source File: OrcRowInputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private static void assertOffsetAndLen(
		OrcSplitReader reader, long offset, long length) throws IllegalAccessException {
	List<StripeInformation> stripes = getStripes(reader.getRecordReader());
	long min = Long.MAX_VALUE;
	long max = Long.MIN_VALUE;
	for (StripeInformation stripe : stripes) {
		if (stripe.getOffset() < min) {
			min = stripe.getOffset();
		}
		if (stripe.getOffset() + stripe.getLength() > max) {
			max = stripe.getOffset() + stripe.getLength();
		}
	}

	assertEquals(offset, min);
	assertEquals(length, max - min);
}
Example 6
Source File: DremioORCRecordUtils.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up 
@Override
public OrcProto.StripeFooter readStripeFooter(StripeInformation stripe) throws IOException {
  if (file == null) {
    open();
  }
  long offset = stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength();
  int tailLength = (int) stripe.getFooterLength();

  // read the footer
  ByteBuffer tailBuf = ByteBuffer.allocate(tailLength);
  file.readFully(offset, tailBuf.array(), tailBuf.arrayOffset(), tailLength);
  return OrcProto.StripeFooter.parseFrom(
    InStream.createCodedInputStream("footer", singleton(
      new BufferChunk(tailBuf, 0)), tailLength, codec, bufferSize));
}
Example 7
Source File: DremioORCRecordUtils.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up 
/**
 * This function is a copy of original implementation from hive-private repository
 */
@Override
public OrcIndex readRowIndex(StripeInformation stripe,
                             OrcProto.StripeFooter footer,
                             boolean[] included,
                             OrcProto.RowIndex[] indexes,
                             boolean[] sargColumns,
                             OrcProto.BloomFilterIndex[] bloomFilterIndices
) throws IOException {
  if (file == null) {
    open();
  }
  if (footer == null) {
    footer = readStripeFooter(stripe);
  }
  if (indexes == null) {
    indexes = new OrcProto.RowIndex[typeCount];
  }
  if (bloomFilterIndices == null) {
    bloomFilterIndices = new OrcProto.BloomFilterIndex[typeCount];
  }
  long offset = stripe.getOffset();
  List<OrcProto.Stream> streams = footer.getStreamsList();
  for (int i = 0; i < streams.size(); i++) {
    OrcProto.Stream stream = streams.get(i);
    OrcProto.Stream nextStream = null;
    if (i < streams.size() - 1) {
      nextStream = streams.get(i+1);
    }
    int col = stream.getColumn();
    int len = (int) stream.getLength();
    // row index stream and bloom filter are interlaced, check if the sarg column contains bloom
    // filter and combine the io to read row index and bloom filters for that column together
    if (stream.hasKind() && (stream.getKind() == OrcProto.Stream.Kind.ROW_INDEX)) {
      boolean readBloomFilter = false;
      if (sargColumns != null && sargColumns[col] &&
        nextStream.getKind() == OrcProto.Stream.Kind.BLOOM_FILTER) {
        len += nextStream.getLength();
        i += 1;
        readBloomFilter = true;
      }
      if ((included == null || included[col]) && indexes[col] == null) {
        byte[] buffer = new byte[len];
        file.readFully(offset, buffer, 0, buffer.length);
        ByteBuffer bb = ByteBuffer.wrap(buffer);
        ByteBuffer rowIndexBB = bb.duplicate();
        rowIndexBB.position(0);
        rowIndexBB.limit((int)stream.getLength());
        indexes[col] = OrcProto.RowIndex.parseFrom(InStream.create("index",
          Lists.<DiskRange>newArrayList(new BufferChunk(rowIndexBB, 0)), stream.getLength(),
          codec, bufferSize, null));
        if (readBloomFilter) {
          ByteBuffer bloomFilterBB = bb.duplicate();
          bloomFilterBB.position((int)stream.getLength());
          bloomFilterBB.limit(buffer.length);
          bloomFilterIndices[col] = OrcProto.BloomFilterIndex.parseFrom(InStream.create(
            "bloom_filter", Lists.<DiskRange>newArrayList(new BufferChunk(bloomFilterBB, 0)),
            nextStream.getLength(), codec, bufferSize, null));
        }
      }
    }
    offset += len;
  }

  OrcIndex index = new OrcIndex(indexes, bloomFilterIndices);
  return index;
}