Java Code Examples for org.apache.kylin.common.util.BytesUtil#sizeForValue()

The following examples show how to use org.apache.kylin.common.util.BytesUtil#sizeForValue() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TrieDictionaryForest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private void initSizeOfId() {
    if (trees.isEmpty()) {
        this.sizeOfId = 1;
        return;
    }
    int maxOffset = accuOffset.get(accuOffset.size() - 1);
    TrieDictionary<T> lastTree = trees.get(trees.size() - 1);
    this.sizeOfId = BytesUtil.sizeForValue(baseId + maxOffset + lastTree.getMaxId() + 1L);
}
Example 2
Source File: TrieDictionary.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private void init(byte[] trieBytes) {
    this.trieBytes = trieBytes;
    if (BytesUtil.compareBytes(MAGIC, 0, trieBytes, 0, MAGIC.length) != 0)
        throw new IllegalArgumentException("Wrong file type (magic does not match)");

    try {
        DataInputStream headIn = new DataInputStream(//
                new ByteArrayInputStream(trieBytes, MAGIC_SIZE_I, trieBytes.length - MAGIC_SIZE_I));
        this.headSize = headIn.readShort();
        this.bodyLen = headIn.readInt();
        this.sizeChildOffset = headIn.read();
        this.sizeNoValuesBeneath = headIn.read();
        this.baseId = headIn.readShort();
        this.maxValueLength = headIn.readShort();
        if (maxValueLength < 0) {
            throw new IllegalStateException("maxValueLength is negative (" + maxValueLength
                    + "). Dict value is too long, whose length is larger than " + Short.MAX_VALUE);
        }

        String converterName = headIn.readUTF();
        if (converterName.isEmpty() == false)
            setConverterByName(converterName);

        this.nValues = BytesUtil.readUnsigned(trieBytes, headSize + sizeChildOffset, sizeNoValuesBeneath);
        this.sizeOfId = BytesUtil.sizeForValue(baseId + nValues + 1L); // note baseId could raise 1 byte in ID space, +1 to reserve all 0xFF for NULL case
        this.childOffsetMask = ~((long) (BIT_IS_LAST_CHILD | BIT_IS_END_OF_VALUE) << ((sizeChildOffset - 1) * 8));
        this.firstByteOffset = sizeChildOffset + sizeNoValuesBeneath + 1; // the offset from begin of node to its first value byte
        enableCache();
    } catch (Exception e) {
        if (e instanceof RuntimeException)
            throw (RuntimeException) e;
        else
            throw new RuntimeException(e);
    }
}
Example 3
Source File: TrieDictionaryForest.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private void initSizeOfId() {
    if (trees.isEmpty()) {
        this.sizeOfId = 1;
        return;
    }
    int maxOffset = accuOffset.get(accuOffset.size() - 1);
    TrieDictionary<T> lastTree = trees.get(trees.size() - 1);
    this.sizeOfId = BytesUtil.sizeForValue(baseId + maxOffset + lastTree.getMaxId() + 1L);
}
Example 4
Source File: TrieDictionary.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private void init(byte[] trieBytes) {
    this.trieBytes = trieBytes;
    if (BytesUtil.compareBytes(MAGIC, 0, trieBytes, 0, MAGIC.length) != 0)
        throw new IllegalArgumentException("Wrong file type (magic does not match)");

    try {
        DataInputStream headIn = new DataInputStream(//
                new ByteArrayInputStream(trieBytes, MAGIC_SIZE_I, trieBytes.length - MAGIC_SIZE_I));
        this.headSize = headIn.readShort();
        this.bodyLen = headIn.readInt();
        this.sizeChildOffset = headIn.read();
        this.sizeNoValuesBeneath = headIn.read();
        this.baseId = headIn.readShort();
        this.maxValueLength = headIn.readShort();
        if (maxValueLength < 0) {
            throw new IllegalStateException("maxValueLength is negative (" + maxValueLength
                    + "). Dict value is too long, whose length is larger than " + Short.MAX_VALUE);
        }

        String converterName = headIn.readUTF();
        if (converterName.isEmpty() == false)
            setConverterByName(converterName);

        this.nValues = BytesUtil.readUnsigned(trieBytes, headSize + sizeChildOffset, sizeNoValuesBeneath);
        this.sizeOfId = BytesUtil.sizeForValue(baseId + nValues + 1L); // note baseId could raise 1 byte in ID space, +1 to reserve all 0xFF for NULL case
        this.childOffsetMask = ~((long) (BIT_IS_LAST_CHILD | BIT_IS_END_OF_VALUE) << ((sizeChildOffset - 1) * 8));
        this.firstByteOffset = sizeChildOffset + sizeNoValuesBeneath + 1; // the offset from begin of node to its first value byte
        enableCache();
    } catch (Exception e) {
        if (e instanceof RuntimeException)
            throw (RuntimeException) e;
        else
            throw new RuntimeException(e);
    }
}
Example 5
Source File: TrieDictionary.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private void init(byte[] trieBytes) {
    this.trieBytes = trieBytes;
    if (BytesUtil.compareBytes(HEAD_MAGIC, 0, trieBytes, 0, HEAD_MAGIC.length) != 0)
        throw new IllegalArgumentException("Wrong file type (magic does not match)");

    try {
        DataInputStream headIn = new DataInputStream( //
                new ByteArrayInputStream(trieBytes, HEAD_SIZE_I, trieBytes.length - HEAD_SIZE_I));
        this.headSize = headIn.readShort();
        this.bodyLen = headIn.readInt();
        this.sizeChildOffset = headIn.read();
        this.sizeNoValuesBeneath = headIn.read();
        this.baseId = headIn.readShort();
        this.maxValueLength = headIn.readShort();

        String converterName = headIn.readUTF();
        if (converterName.isEmpty() == false)
            this.bytesConvert = (BytesConverter<T>) ClassUtil.forName(converterName, BytesConverter.class).newInstance();

        this.nValues = BytesUtil.readUnsigned(trieBytes, headSize + sizeChildOffset, sizeNoValuesBeneath);
        this.sizeOfId = BytesUtil.sizeForValue(baseId + nValues + 1); // note baseId could raise 1 byte in ID space, +1 to reserve all 0xFF for NULL case
        this.childOffsetMask = ~((BIT_IS_LAST_CHILD | BIT_IS_END_OF_VALUE) << ((sizeChildOffset - 1) * 8));
        this.firstByteOffset = sizeChildOffset + sizeNoValuesBeneath + 1; // the offset from begin of node to its first value byte
    } catch (Exception e) {
        if (e instanceof RuntimeException)
            throw (RuntimeException) e;
        else
            throw new RuntimeException(e);
    }

    if (enableCache) {
        valueToIdCache = new SoftReference<HashMap>(new HashMap());
        idToValueCache = new SoftReference<Object[]>(new Object[nValues]);
    }
}