Java Code Examples for org.apache.kylin.common.util.BytesUtil#readLong()

The following examples show how to use org.apache.kylin.common.util.BytesUtil#readLong() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: IntegerDimEnc.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
@Override
public String decode(byte[] bytes, int offset, int len) {
    if (isNull(bytes, offset, len)) {
        return null;
    }

    long integer = BytesUtil.readLong(bytes, offset, len) - CAP[fixedLen];

    //only take useful bytes
    integer = integer & MASK[fixedLen];
    boolean positive = (integer & ((0x80L) << ((fixedLen - 1) << 3))) == 0;
    if (!positive) {
        integer |= (~MASK[fixedLen]);
    }

    return String.valueOf(integer);
}
Example 2
Source File: AppendDictSlice.java    From kylin with Apache License 2.0 6 votes vote down vote up 
public byte[] getFirstValue() {
    int nodeOffset = headSize;
    ByteArrayOutputStream bytes = new ByteArrayOutputStream();
    while (true) {
        int valueLen = BytesUtil.readUnsigned(trieBytes, nodeOffset + firstByteOffset - 1, 1);
        bytes.write(trieBytes, nodeOffset + firstByteOffset, valueLen);
        if (checkFlag(nodeOffset, BIT_IS_END_OF_VALUE)) {
            break;
        }
        nodeOffset = headSize
                + (int) (BytesUtil.readLong(trieBytes, nodeOffset, sizeChildOffset) & childOffsetMask);
        if (nodeOffset == headSize) {
            break;
        }
    }
    return bytes.toByteArray();
}
Example 3
Source File: AppendDictSlice.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
public byte[] getFirstValue() {
    int nodeOffset = headSize;
    ByteArrayOutputStream bytes = new ByteArrayOutputStream();
    while (true) {
        int valueLen = BytesUtil.readUnsigned(trieBytes, nodeOffset + firstByteOffset - 1, 1);
        bytes.write(trieBytes, nodeOffset + firstByteOffset, valueLen);
        if (checkFlag(nodeOffset, BIT_IS_END_OF_VALUE)) {
            break;
        }
        nodeOffset = headSize
                + (int) (BytesUtil.readLong(trieBytes, nodeOffset, sizeChildOffset) & childOffsetMask);
        if (nodeOffset == headSize) {
            break;
        }
    }
    return bytes.toByteArray();
}
Example 4
Source File: IntegerDimEnc.java    From kylin with Apache License 2.0 6 votes vote down vote up 
@Override
public String decode(byte[] bytes, int offset, int len) {
    if (isNull(bytes, offset, len)) {
        return null;
    }

    long integer = BytesUtil.readLong(bytes, offset, len) - CAP[fixedLen];

    //only take useful bytes
    integer = integer & MASK[fixedLen];
    boolean positive = (integer & ((0x80L) << ((fixedLen - 1) << 3))) == 0;
    if (!positive) {
        integer |= (~MASK[fixedLen]);
    }

    return String.valueOf(integer);
}
Example 5
Source File: BooleanDimEnc.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Override
public String decode(byte[] bytes, int offset, int len) {
    if (isNull(bytes, offset, len)) {
        return null;
    }

    int x = (int) BytesUtil.readLong(bytes, offset, len);
    if (x >= ALLOWED_VALUES.length) {
        throw new IllegalStateException();
    }

    return ALLOWED_VALUES[x];
}
Example 6
Source File: IntDimEnc.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Override
public String decode(byte[] bytes, int offset, int len) {
    if (isNull(bytes, offset, len)) {
        return null;
    }

    long integer = BytesUtil.readLong(bytes, offset, len);
    return String.valueOf(integer);
}
Example 7
Source File: IntDimEnc.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public String decode(byte[] bytes, int offset, int len) {
    if (isNull(bytes, offset, len)) {
        return null;
    }

    long integer = BytesUtil.readLong(bytes, offset, len);
    return String.valueOf(integer);
}
Example 8
Source File: AbstractDateDimEnc.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public String decode(byte[] bytes, int offset, int len) {
    if (isNull(bytes, offset, len)) {
        return null;
    }

    long code = BytesUtil.readLong(bytes, offset, fixedLen);
    if (code < 0)
        throw new IllegalArgumentException();

    return codec.codeToValue(code);
}
Example 9
Source File: AppendDictSlice.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private AppendDictNode rebuildTrieTreeR(int n, AppendDictNode parent) {
    AppendDictNode root = null;
    while (true) {
        int p = n + firstByteOffset;
        int childOffset = (int) (BytesUtil.readLong(trieBytes, n, sizeChildOffset) & childOffsetMask);
        int parLen = BytesUtil.readUnsigned(trieBytes, p - 1, 1);
        boolean isEndOfValue = checkFlag(n, BIT_IS_END_OF_VALUE);

        byte[] value = new byte[parLen];
        System.arraycopy(trieBytes, p, value, 0, parLen);

        AppendDictNode node = new AppendDictNode(value, isEndOfValue);
        if (isEndOfValue) {
            int id = BytesUtil.readUnsigned(trieBytes, p + parLen, sizeOfId);
            node.id = id;
        }

        if (parent == null) {
            root = node;
        } else {
            parent.addChild(node);
        }

        if (childOffset != 0) {
            rebuildTrieTreeR(childOffset + headSize, node);
        }

        if (checkFlag(n, BIT_IS_LAST_CHILD)) {
            break;
        } else {
            n += firstByteOffset + parLen + (isEndOfValue ? sizeOfId : 0);
        }
    }
    return root;
}
Example 10
Source File: BooleanDimEnc.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public String decode(byte[] bytes, int offset, int len) {
    if (isNull(bytes, offset, len)) {
        return null;
    }

    int x = (int) BytesUtil.readLong(bytes, offset, len);
    if (x >= ALLOWED_VALUES.length) {
        throw new IllegalStateException();
    }

    return ALLOWED_VALUES[x];
}
Example 11
Source File: DateDimEncTest.java    From kylin with Apache License 2.0 4 votes vote down vote up 
private long encode(String value) {
    enc.encode(value, buf, 0);
    return BytesUtil.readLong(buf, 0, buf.length);
}
Example 12
Source File: Long8Serializer.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
@Override
public Long deserialize(ByteBuffer in) {
    return BytesUtil.readLong(in);
}
Example 13
Source File: TimeDimEncTest.java    From kylin with Apache License 2.0 4 votes vote down vote up 
private long encode(String value) {
    enc.encode(value, buf, 0);
    return BytesUtil.readLong(buf, 0, buf.length);
}
Example 14
Source File: AppendDictSlice.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public boolean doCheck() {
    int offset = headSize;
    HashSet<Integer> parentSet = new HashSet<>();
    boolean lastChild = false;

    while (offset < trieBytes.length) {
        if (lastChild) {
            boolean contained = parentSet.remove(offset - headSize);
            // Can't find parent, the data is corrupted
            if (!contained) {
                return false;
            }
            lastChild = false;
        }
        int p = offset + firstByteOffset;
        int childOffset = (int) (BytesUtil.readLong(trieBytes, offset, sizeChildOffset) & childOffsetMask);
        int parLen = BytesUtil.readUnsigned(trieBytes, p - 1, 1);
        boolean isEndOfValue = checkFlag(offset, BIT_IS_END_OF_VALUE);

        // Copy value overflow, the data is corrupted
        if (trieBytes.length < p + parLen) {
            return false;
        }

        // Check id is fine
        if (isEndOfValue) {
            BytesUtil.readUnsigned(trieBytes, p + parLen, sizeOfId);
        }

        // Record it if has children
        if (childOffset != 0) {
            parentSet.add(childOffset);
        }

        // brothers done, move to next parent
        if (checkFlag(offset, BIT_IS_LAST_CHILD)) {
            lastChild = true;
        }

        // move to next node
        offset += firstByteOffset + parLen + (isEndOfValue ? sizeOfId : 0);
    }

    // ParentMap is empty, meaning all nodes has parent, the data is correct
    return parentSet.isEmpty();
}
Example 15
Source File: AppendDictSlice.java    From kylin with Apache License 2.0 4 votes vote down vote up 
/**
 * returns a code point from [0, nValues), preserving order of value
 *
 * @param n            -- the offset of current node
 * @param inp          -- input value bytes to lookup
 * @param o            -- offset in the input value bytes matched so far
 * @param inpEnd       -- end of input
 * @param roundingFlag -- =0: return -1 if not found
 *                     -- <0: return closest smaller if not found, return -1
 *                     -- >0: return closest bigger if not found, return nValues
 */
private int lookupSeqNoFromValue(int n, byte[] inp, int o, int inpEnd, int roundingFlag) {
    while (true) {
        // match the current node
        int p = n + firstByteOffset; // start of node's value
        int end = p + BytesUtil.readUnsigned(trieBytes, p - 1, 1); // end of node's value
        for (; p < end && o < inpEnd; p++, o++) { // note matching start from [0]
            if (trieBytes[p] != inp[o]) {
                return -1; // mismatch
            }
        }

        // node completely matched, is input all consumed?
        boolean isEndOfValue = checkFlag(n, BIT_IS_END_OF_VALUE);
        if (o == inpEnd) {
            return p == end && isEndOfValue ? BytesUtil.readUnsigned(trieBytes, end, sizeOfId) : -1;
        }

        // find a child to continue
        int c = headSize + (int) (BytesUtil.readLong(trieBytes, n, sizeChildOffset) & childOffsetMask);
        if (c == headSize) // has no children
            return -1;
        byte inpByte = inp[o];
        int comp;
        while (true) {
            p = c + firstByteOffset;
            comp = BytesUtil.compareByteUnsigned(trieBytes[p], inpByte);
            if (comp == 0) { // continue in the matching child, reset n and loop again
                n = c;
                break;
            } else if (comp < 0) { // try next child
                if (checkFlag(c, BIT_IS_LAST_CHILD))
                    return -1;
                c = p + BytesUtil.readUnsigned(trieBytes, p - 1, 1)
                        + (checkFlag(c, BIT_IS_END_OF_VALUE) ? sizeOfId : 0);
            } else { // children are ordered by their first value byte
                return -1;
            }
        }
    }
}
Example 16
Source File: AppendDictSlice.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public boolean doCheck() {
    int offset = headSize;
    HashSet<Integer> parentSet = new HashSet<>();
    boolean lastChild = false;

    while (offset < trieBytes.length) {
        if (lastChild) {
            boolean contained = parentSet.remove(offset - headSize);
            // Can't find parent, the data is corrupted
            if (!contained) {
                return false;
            }
            lastChild = false;
        }
        int p = offset + firstByteOffset;
        int childOffset = (int) (BytesUtil.readLong(trieBytes, offset, sizeChildOffset) & childOffsetMask);
        int parLen = BytesUtil.readUnsigned(trieBytes, p - 1, 1);
        boolean isEndOfValue = checkFlag(offset, BIT_IS_END_OF_VALUE);

        // Copy value overflow, the data is corrupted
        if (trieBytes.length < p + parLen) {
            return false;
        }

        // Check id is fine
        if (isEndOfValue) {
            BytesUtil.readUnsigned(trieBytes, p + parLen, sizeOfId);
        }

        // Record it if has children
        if (childOffset != 0) {
            parentSet.add(childOffset);
        }

        // brothers done, move to next parent
        if (checkFlag(offset, BIT_IS_LAST_CHILD)) {
            lastChild = true;
        }

        // move to next node
        offset += firstByteOffset + parLen + (isEndOfValue ? sizeOfId : 0);
    }

    // ParentMap is empty, meaning all nodes has parent, the data is correct
    return parentSet.isEmpty();
}
Example 17
Source File: AppendDictSlice.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
/**
 * returns a code point from [0, nValues), preserving order of value
 *
 * @param n            -- the offset of current node
 * @param inp          -- input value bytes to lookup
 * @param o            -- offset in the input value bytes matched so far
 * @param inpEnd       -- end of input
 * @param roundingFlag -- =0: return -1 if not found
 *                     -- <0: return closest smaller if not found, return -1
 *                     -- >0: return closest bigger if not found, return nValues
 */
private int lookupSeqNoFromValue(int n, byte[] inp, int o, int inpEnd, int roundingFlag) {
    while (true) {
        // match the current node
        int p = n + firstByteOffset; // start of node's value
        int end = p + BytesUtil.readUnsigned(trieBytes, p - 1, 1); // end of node's value
        for (; p < end && o < inpEnd; p++, o++) { // note matching start from [0]
            if (trieBytes[p] != inp[o]) {
                return -1; // mismatch
            }
        }

        // node completely matched, is input all consumed?
        boolean isEndOfValue = checkFlag(n, BIT_IS_END_OF_VALUE);
        if (o == inpEnd) {
            return p == end && isEndOfValue ? BytesUtil.readUnsigned(trieBytes, end, sizeOfId) : -1;
        }

        // find a child to continue
        int c = headSize + (int) (BytesUtil.readLong(trieBytes, n, sizeChildOffset) & childOffsetMask);
        if (c == headSize) // has no children
            return -1;
        byte inpByte = inp[o];
        int comp;
        while (true) {
            p = c + firstByteOffset;
            comp = BytesUtil.compareByteUnsigned(trieBytes[p], inpByte);
            if (comp == 0) { // continue in the matching child, reset n and loop again
                n = c;
                break;
            } else if (comp < 0) { // try next child
                if (checkFlag(c, BIT_IS_LAST_CHILD))
                    return -1;
                c = p + BytesUtil.readUnsigned(trieBytes, p - 1, 1)
                        + (checkFlag(c, BIT_IS_END_OF_VALUE) ? sizeOfId : 0);
            } else { // children are ordered by their first value byte
                return -1;
            }
        }
    }
}
Example 18
Source File: DateDimEncTest.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
private long encode(String value) {
    enc.encode(value, buf, 0);
    return BytesUtil.readLong(buf, 0, buf.length);
}
Example 19
Source File: TimeDimEncTest.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
private long encode(String value) {
    enc.encode(value, buf, 0);
    return BytesUtil.readLong(buf, 0, buf.length);
}
Example 20
Source File: FilterCodeSystemFactory.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
@Override
public Object deserialize(ByteBuffer buf) {
    return BytesUtil.readLong(buf);
}