Java Code Examples for org.apache.kylin.common.util.BytesUtil#writeUnsigned()

The following examples show how to use org.apache.kylin.common.util.BytesUtil#writeUnsigned() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TrieDictionaryBuilder.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
private int build_writeNode(Node n, int offset, boolean isLastChild, int sizeNoValuesBeneath, int sizeChildOffset, byte[] trieBytes) {
    int o = offset;

    // childOffset
    if (isLastChild)
        trieBytes[o] |= TrieDictionary.BIT_IS_LAST_CHILD;
    if (n.isEndOfValue)
        trieBytes[o] |= TrieDictionary.BIT_IS_END_OF_VALUE;
    o += sizeChildOffset;

    // nValuesBeneath
    BytesUtil.writeUnsigned(n.nValuesBeneath, trieBytes, o, sizeNoValuesBeneath);
    o += sizeNoValuesBeneath;

    // nValueBytes
    if (n.part.length > 255)
        throw new RuntimeException();
    BytesUtil.writeUnsigned(n.part.length, trieBytes, o, 1);
    o++;

    // valueBytes
    System.arraycopy(n.part, 0, trieBytes, o, n.part.length);
    o += n.part.length;

    return o;
}
Example 2
Source File: IIKeyValueCodec.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
int encodeKey(short shard, long timestamp, int col, int colValue,
		byte[] buf, int offset) {
	int i = offset;

	BytesUtil.writeUnsigned(shard, buf, i, SHARD_LEN);
	i += SHARD_LEN;
	BytesUtil.writeLong(timestamp, buf, i, TIMEPART_LEN);
	i += TIMEPART_LEN;

	BytesUtil.writeUnsigned(col, buf, i, COLNO_LEN);
	i += COLNO_LEN;

	if (colValue >= 0) {
		int colLen = infoDigest.length(col);
		BytesUtil.writeUnsigned(colValue, buf, i, colLen);
		i += colLen;
	}

	return i - offset;
}
Example 3
Source File: TrieDictionaryBuilder.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private int build_writeNode(Node n, int offset, boolean isLastChild, int sizeNoValuesBeneath, int sizeChildOffset, byte[] trieBytes) {
    int o = offset;
    if (o > _2GB)
        throw new IllegalStateException();

    // childOffset
    if (isLastChild)
        trieBytes[o] |= TrieDictionary.BIT_IS_LAST_CHILD;
    if (n.isEndOfValue)
        trieBytes[o] |= TrieDictionary.BIT_IS_END_OF_VALUE;
    o += sizeChildOffset;

    // nValuesBeneath
    BytesUtil.writeUnsigned(n.nValuesBeneath, trieBytes, o, sizeNoValuesBeneath);
    o += sizeNoValuesBeneath;

    // nValueBytes
    if (n.part.length > 255)
        throw new RuntimeException();
    BytesUtil.writeUnsigned(n.part.length, trieBytes, o, 1);
    o++;

    // valueBytes
    System.arraycopy(n.part, 0, trieBytes, o, n.part.length);
    o += n.part.length;

    return o;
}
Example 4
Source File: AppendDictNode.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private int build_writeNode(AppendDictNode n, int offset, boolean isLastChild, int sizeChildOffset, int sizeId,
        byte[] trieBytes) {
    int o = offset;

    // childOffset
    if (isLastChild)
        trieBytes[o] |= TrieDictionary.BIT_IS_LAST_CHILD;
    if (n.isEndOfValue)
        trieBytes[o] |= TrieDictionary.BIT_IS_END_OF_VALUE;
    o += sizeChildOffset;

    // nValueBytes
    if (n.part.length > 255)
        throw new RuntimeException(
                "Value length is " + n.part.length + " and larger than 255: " + Bytes.toStringBinary(n.part));
    BytesUtil.writeUnsigned(n.part.length, trieBytes, o, 1);
    o++;

    // valueBytes
    System.arraycopy(n.part, 0, trieBytes, o, n.part.length);
    o += n.part.length;

    if (n.isEndOfValue) {
        checkValidId(n.id);
        BytesUtil.writeUnsigned(n.id, trieBytes, o, sizeId);
        o += sizeId;
    }

    return o;
}
Example 5
Source File: CreateHTableJob.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private static byte[][] getSplitsByRegionCount(int regionCount) {
    byte[][] result = new byte[regionCount - 1][];
    for (int i = 1; i < regionCount; ++i) {
        byte[] split = new byte[RowConstants.ROWKEY_SHARDID_LEN];
        BytesUtil.writeUnsigned(i, split, 0, RowConstants.ROWKEY_SHARDID_LEN);
        result[i - 1] = split;
    }
    return result;
}
Example 6
Source File: LookupTableToHFileJob.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private byte[][] getSplitsByShardNum(int shardNum) {
    byte[][] result = new byte[shardNum - 1][];
    for (int i = 1; i < shardNum; ++i) {
        byte[] split = new byte[RowConstants.ROWKEY_SHARDID_LEN];
        BytesUtil.writeUnsigned(i, split, 0, RowConstants.ROWKEY_SHARDID_LEN);
        result[i - 1] = split;
    }
    return result;
}
Example 7
Source File: AppendDictNode.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private int build_writeNode(AppendDictNode n, int offset, boolean isLastChild, int sizeChildOffset, int sizeId,
        byte[] trieBytes) {
    int o = offset;

    // childOffset
    if (isLastChild)
        trieBytes[o] |= TrieDictionary.BIT_IS_LAST_CHILD;
    if (n.isEndOfValue)
        trieBytes[o] |= TrieDictionary.BIT_IS_END_OF_VALUE;
    o += sizeChildOffset;

    // nValueBytes
    if (n.part.length > 255)
        throw new RuntimeException(
                "Value length is " + n.part.length + " and larger than 255: " + Bytes.toStringBinary(n.part));
    BytesUtil.writeUnsigned(n.part.length, trieBytes, o, 1);
    o++;

    // valueBytes
    System.arraycopy(n.part, 0, trieBytes, o, n.part.length);
    o += n.part.length;

    if (n.isEndOfValue) {
        checkValidId(n.id);
        BytesUtil.writeUnsigned(n.id, trieBytes, o, sizeId);
        o += sizeId;
    }

    return o;
}
Example 8
Source File: RawSerializerTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private List<ByteArray> getValueList(int size) {
    if (size == -1) {
        return null;
    }
    List<ByteArray> valueList = new ArrayList<ByteArray>(size);
    for (Integer i = 0; i < size; i++) {
        ByteArray key = new ByteArray(1);
        BytesUtil.writeUnsigned(i, key.array(), 0, key.length());
        valueList.add(key);
    }
    return valueList;
}
Example 9
Source File: AppendDictNode.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
private void build_overwriteChildOffset(int parentOffset, int childOffset, int sizeChildOffset, byte[] trieBytes) {
    int flags = (int) trieBytes[parentOffset]
            & (TrieDictionary.BIT_IS_LAST_CHILD | TrieDictionary.BIT_IS_END_OF_VALUE);
    BytesUtil.writeUnsigned(childOffset, trieBytes, parentOffset, sizeChildOffset);
    trieBytes[parentOffset] |= flags;
}
Example 10
Source File: AppendDictNode.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public byte[] buildTrieBytes() {
    Stats stats = Stats.stats(this);
    int sizeChildOffset = stats.mbpn_sizeChildOffset;
    int sizeId = stats.mbpn_sizeId;

    // write head
    byte[] head;
    try {
        ByteArrayOutputStream byteBuf = new ByteArrayOutputStream();
        DataOutputStream headOut = new DataOutputStream(byteBuf);
        headOut.write(AppendTrieDictionary.HEAD_MAGIC);
        headOut.writeShort(0); // head size, will back fill
        headOut.writeInt(stats.mbpn_footprint); // body size
        headOut.writeInt(stats.nValues);
        headOut.write(sizeChildOffset);
        headOut.write(sizeId);
        headOut.close();
        head = byteBuf.toByteArray();
        BytesUtil.writeUnsigned(head.length, head, AppendTrieDictionary.HEAD_SIZE_I, 2);
    } catch (IOException e) {
        throw new RuntimeException(e); // shall not happen, as we are
    }

    byte[] trieBytes = new byte[stats.mbpn_footprint + head.length];
    System.arraycopy(head, 0, trieBytes, 0, head.length);

    LinkedList<AppendDictNode> open = new LinkedList<AppendDictNode>();
    IdentityHashMap<AppendDictNode, Integer> offsetMap = new IdentityHashMap<AppendDictNode, Integer>();

    // write body
    int o = head.length;
    offsetMap.put(this, o);
    o = build_writeNode(this, o, true, sizeChildOffset, sizeId, trieBytes);
    if (this.children.isEmpty() == false)
        open.addLast(this);

    while (open.isEmpty() == false) {
        AppendDictNode parent = open.removeFirst();
        build_overwriteChildOffset(offsetMap.get(parent), o - head.length, sizeChildOffset, trieBytes);
        for (int i = 0; i < parent.children.size(); i++) {
            AppendDictNode c = parent.children.get(i);
            boolean isLastChild = (i == parent.children.size() - 1);
            offsetMap.put(c, o);
            o = build_writeNode(c, o, isLastChild, sizeChildOffset, sizeId, trieBytes);
            if (c.children.isEmpty() == false)
                open.addLast(c);
        }
    }

    if (o != trieBytes.length)
        throw new RuntimeException();
    return trieBytes;
}
Example 11
Source File: TrieDictionaryBuilder.java    From kylin with Apache License 2.0 4 votes vote down vote up 
private void build_overwriteChildOffset(int parentOffset, int childOffset, int sizeChildOffset, byte[] trieBytes) {
    int flags = (int) trieBytes[parentOffset] & (TrieDictionary.BIT_IS_LAST_CHILD | TrieDictionary.BIT_IS_END_OF_VALUE);
    BytesUtil.writeUnsigned(childOffset, trieBytes, parentOffset, sizeChildOffset);
    trieBytes[parentOffset] |= flags;
}
Example 12
Source File: AppendDictNode.java    From kylin with Apache License 2.0 4 votes vote down vote up 
private void build_overwriteChildOffset(int parentOffset, int childOffset, int sizeChildOffset, byte[] trieBytes) {
    int flags = (int) trieBytes[parentOffset]
            & (TrieDictionary.BIT_IS_LAST_CHILD | TrieDictionary.BIT_IS_END_OF_VALUE);
    BytesUtil.writeUnsigned(childOffset, trieBytes, parentOffset, sizeChildOffset);
    trieBytes[parentOffset] |= flags;
}
Example 13
Source File: TrieDictionaryBuilder.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
private void build_overwriteChildOffset(int parentOffset, int childOffset, int sizeChildOffset, byte[] trieBytes) {
    int flags = (int) trieBytes[parentOffset] & (TrieDictionary.BIT_IS_LAST_CHILD | TrieDictionary.BIT_IS_END_OF_VALUE);
    BytesUtil.writeUnsigned(childOffset, trieBytes, parentOffset, sizeChildOffset);
    trieBytes[parentOffset] |= flags;
}
Example 14
Source File: CubeHBaseEndpointRPC.java    From kylin with Apache License 2.0 4 votes vote down vote up 
private byte[] getByteArrayForShort(short v) {
    byte[] split = new byte[Bytes.SIZEOF_SHORT];
    BytesUtil.writeUnsigned(v, split, 0, Bytes.SIZEOF_SHORT);
    return split;
}
Example 15
Source File: Int4Serializer.java    From kylin with Apache License 2.0 4 votes vote down vote up 
@Override
public void serialize(Integer value, ByteBuffer out) {
    BytesUtil.writeUnsigned(value, 4, out);
}
Example 16
Source File: DictionaryDimEnc.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
@Override
public void serialize(Object value, ByteBuffer buf) {
    int id = dict.getIdFromValue(value == null ? null : value.toString(), roundingFlag);
    BytesUtil.writeUnsigned(id, dict.getSizeOfId(), buf);
}
Example 17
Source File: DictionaryDimEnc.java    From kylin with Apache License 2.0 4 votes vote down vote up 
@Override
public void serialize(Object value, ByteBuffer buf) {
    int id = dict.getIdFromValue(value == null ? null : value.toString(), roundingFlag);
    BytesUtil.writeUnsigned(id, dict.getSizeOfId(), buf);
}
Example 18
Source File: TrieDictionaryBuilder.java    From kylin with Apache License 2.0 4 votes vote down vote up 
protected byte[] buildTrieBytes(int baseId) {
    checkOverflowParts(this.root);

    Stats stats = stats();
    int sizeNoValuesBeneath = stats.mbpn_sizeNoValueBeneath;
    int sizeChildOffset = stats.mbpn_sizeChildOffset;

    if (stats.mbpn_footprint <= 0) // must never happen, but let us be cautious
        throw new IllegalStateException("Too big dictionary, dictionary cannot be bigger than 2GB");
    if (stats.mbpn_footprint > _2GB)
        throw new RuntimeException("Too big dictionary, dictionary cannot be bigger than 2GB");

    // write head
    byte[] head;
    try {
        ByteArrayOutputStream byteBuf = new ByteArrayOutputStream();
        DataOutputStream headOut = new DataOutputStream(byteBuf);
        headOut.write(TrieDictionary.MAGIC);
        headOut.writeShort(0); // head size, will back fill
        headOut.writeInt((int) stats.mbpn_footprint); // body size
        headOut.write(sizeChildOffset);
        headOut.write(sizeNoValuesBeneath);
        positiveShortPreCheck(baseId, "baseId");
        headOut.writeShort(baseId);
        positiveShortPreCheck(stats.maxValueLength, "stats.maxValueLength");
        headOut.writeShort(stats.maxValueLength);
        headOut.writeUTF(bytesConverter == null ? "" : bytesConverter.getClass().getName());
        headOut.close();
        head = byteBuf.toByteArray();
        BytesUtil.writeUnsigned(head.length, head, TrieDictionary.MAGIC_SIZE_I, 2);
    } catch (IOException e) {
        throw new RuntimeException(e); // shall not happen, as we are writing in memory
    }

    byte[] trieBytes = new byte[(int) stats.mbpn_footprint + head.length];
    System.arraycopy(head, 0, trieBytes, 0, head.length);

    LinkedList<Node> open = new LinkedList<Node>();
    IdentityHashMap<Node, Integer> offsetMap = new IdentityHashMap<Node, Integer>();

    // write body
    int o = head.length;
    offsetMap.put(root, o);
    o = build_writeNode(root, o, true, sizeNoValuesBeneath, sizeChildOffset, trieBytes);
    if (root.children.isEmpty() == false)
        open.addLast(root);

    while (open.isEmpty() == false) {
        Node parent = open.removeFirst();
        build_overwriteChildOffset(offsetMap.get(parent), o - head.length, sizeChildOffset, trieBytes);
        for (int i = 0; i < parent.children.size(); i++) {
            Node c = parent.children.get(i);
            boolean isLastChild = (i == parent.children.size() - 1);
            offsetMap.put(c, o);
            o = build_writeNode(c, o, isLastChild, sizeNoValuesBeneath, sizeChildOffset, trieBytes);
            if (c.children.isEmpty() == false)
                open.addLast(c);
        }
    }

    if (o != trieBytes.length)
        throw new RuntimeException();
    return trieBytes;
}
Example 19
Source File: BitMapContainer.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
@Override
public void getValueAt(int i, ImmutableBytesWritable valueBytes) {
    int value = getValueIntAt(i);
    BytesUtil.writeUnsigned(value, temp, 0, valueLen);
    valueBytes.set(temp, 0, valueLen);
}
Example 20
Source File: AppendDictNode.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public byte[] buildTrieBytes() {
    Stats stats = Stats.stats(this);
    int sizeChildOffset = stats.mbpn_sizeChildOffset;
    int sizeId = stats.mbpn_sizeId;

    // write head
    byte[] head;
    try {
        ByteArrayOutputStream byteBuf = new ByteArrayOutputStream();
        DataOutputStream headOut = new DataOutputStream(byteBuf);
        headOut.write(AppendTrieDictionary.HEAD_MAGIC);
        headOut.writeShort(0); // head size, will back fill
        headOut.writeInt(stats.mbpn_footprint); // body size
        headOut.writeInt(stats.nValues);
        headOut.write(sizeChildOffset);
        headOut.write(sizeId);
        headOut.close();
        head = byteBuf.toByteArray();
        BytesUtil.writeUnsigned(head.length, head, AppendTrieDictionary.HEAD_SIZE_I, 2);
    } catch (IOException e) {
        throw new RuntimeException(e); // shall not happen, as we are
    }

    byte[] trieBytes = new byte[stats.mbpn_footprint + head.length];
    System.arraycopy(head, 0, trieBytes, 0, head.length);

    LinkedList<AppendDictNode> open = new LinkedList<AppendDictNode>();
    IdentityHashMap<AppendDictNode, Integer> offsetMap = new IdentityHashMap<AppendDictNode, Integer>();

    // write body
    int o = head.length;
    offsetMap.put(this, o);
    o = build_writeNode(this, o, true, sizeChildOffset, sizeId, trieBytes);
    if (this.children.isEmpty() == false)
        open.addLast(this);

    while (open.isEmpty() == false) {
        AppendDictNode parent = open.removeFirst();
        build_overwriteChildOffset(offsetMap.get(parent), o - head.length, sizeChildOffset, trieBytes);
        for (int i = 0; i < parent.children.size(); i++) {
            AppendDictNode c = parent.children.get(i);
            boolean isLastChild = (i == parent.children.size() - 1);
            offsetMap.put(c, o);
            o = build_writeNode(c, o, isLastChild, sizeChildOffset, sizeId, trieBytes);
            if (c.children.isEmpty() == false)
                open.addLast(c);
        }
    }

    if (o != trieBytes.length)
        throw new RuntimeException();
    return trieBytes;
}