Java Code Examples for org.apache.kylin.common.util.BytesUtil#writeUnsigned()

The following examples show how to use org.apache.kylin.common.util.BytesUtil#writeUnsigned() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TrieDictionaryBuilder.java From Kylin with Apache License 2.0

6 votes

private int build_writeNode(Node n, int offset, boolean isLastChild, int sizeNoValuesBeneath, int sizeChildOffset, byte[] trieBytes) {
    int o = offset;

    // childOffset
    if (isLastChild)
        trieBytes[o] |= TrieDictionary.BIT_IS_LAST_CHILD;
    if (n.isEndOfValue)
        trieBytes[o] |= TrieDictionary.BIT_IS_END_OF_VALUE;
    o += sizeChildOffset;

    // nValuesBeneath
    BytesUtil.writeUnsigned(n.nValuesBeneath, trieBytes, o, sizeNoValuesBeneath);
    o += sizeNoValuesBeneath;

    // nValueBytes
    if (n.part.length > 255)
        throw new RuntimeException();
    BytesUtil.writeUnsigned(n.part.length, trieBytes, o, 1);
    o++;

    // valueBytes
    System.arraycopy(n.part, 0, trieBytes, o, n.part.length);
    o += n.part.length;

    return o;
}

Example 2

Source File: IIKeyValueCodec.java From Kylin with Apache License 2.0

6 votes

int encodeKey(short shard, long timestamp, int col, int colValue,
		byte[] buf, int offset) {
	int i = offset;

	BytesUtil.writeUnsigned(shard, buf, i, SHARD_LEN);
	i += SHARD_LEN;
	BytesUtil.writeLong(timestamp, buf, i, TIMEPART_LEN);
	i += TIMEPART_LEN;

	BytesUtil.writeUnsigned(col, buf, i, COLNO_LEN);
	i += COLNO_LEN;

	if (colValue >= 0) {
		int colLen = infoDigest.length(col);
		BytesUtil.writeUnsigned(colValue, buf, i, colLen);
		i += colLen;
	}

	return i - offset;
}

Example 3

Source File: TrieDictionaryBuilder.java From kylin with Apache License 2.0

5 votes

private int build_writeNode(Node n, int offset, boolean isLastChild, int sizeNoValuesBeneath, int sizeChildOffset, byte[] trieBytes) {
    int o = offset;
    if (o > _2GB)
        throw new IllegalStateException();

    // childOffset
    if (isLastChild)
        trieBytes[o] |= TrieDictionary.BIT_IS_LAST_CHILD;
    if (n.isEndOfValue)
        trieBytes[o] |= TrieDictionary.BIT_IS_END_OF_VALUE;
    o += sizeChildOffset;

    // nValuesBeneath
    BytesUtil.writeUnsigned(n.nValuesBeneath, trieBytes, o, sizeNoValuesBeneath);
    o += sizeNoValuesBeneath;

    // nValueBytes
    if (n.part.length > 255)
        throw new RuntimeException();
    BytesUtil.writeUnsigned(n.part.length, trieBytes, o, 1);
    o++;

    // valueBytes
    System.arraycopy(n.part, 0, trieBytes, o, n.part.length);
    o += n.part.length;

    return o;
}

Example 4

Source File: AppendDictNode.java From kylin with Apache License 2.0

5 votes

private int build_writeNode(AppendDictNode n, int offset, boolean isLastChild, int sizeChildOffset, int sizeId,
        byte[] trieBytes) {
    int o = offset;

    // childOffset
    if (isLastChild)
        trieBytes[o] |= TrieDictionary.BIT_IS_LAST_CHILD;
    if (n.isEndOfValue)
        trieBytes[o] |= TrieDictionary.BIT_IS_END_OF_VALUE;
    o += sizeChildOffset;

    // nValueBytes
    if (n.part.length > 255)
        throw new RuntimeException(
                "Value length is " + n.part.length + " and larger than 255: " + Bytes.toStringBinary(n.part));
    BytesUtil.writeUnsigned(n.part.length, trieBytes, o, 1);
    o++;

    // valueBytes
    System.arraycopy(n.part, 0, trieBytes, o, n.part.length);
    o += n.part.length;

    if (n.isEndOfValue) {
        checkValidId(n.id);
        BytesUtil.writeUnsigned(n.id, trieBytes, o, sizeId);
        o += sizeId;
    }

    return o;
}

Example 5

Source File: CreateHTableJob.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

private static byte[][] getSplitsByRegionCount(int regionCount) {
    byte[][] result = new byte[regionCount - 1][];
    for (int i = 1; i < regionCount; ++i) {
        byte[] split = new byte[RowConstants.ROWKEY_SHARDID_LEN];
        BytesUtil.writeUnsigned(i, split, 0, RowConstants.ROWKEY_SHARDID_LEN);
        result[i - 1] = split;
    }
    return result;
}

Example 6

Source File: LookupTableToHFileJob.java From kylin with Apache License 2.0

5 votes

private byte[][] getSplitsByShardNum(int shardNum) {
    byte[][] result = new byte[shardNum - 1][];
    for (int i = 1; i < shardNum; ++i) {
        byte[] split = new byte[RowConstants.ROWKEY_SHARDID_LEN];
        BytesUtil.writeUnsigned(i, split, 0, RowConstants.ROWKEY_SHARDID_LEN);
        result[i - 1] = split;
    }
    return result;
}

Example 7

Source File: AppendDictNode.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

private int build_writeNode(AppendDictNode n, int offset, boolean isLastChild, int sizeChildOffset, int sizeId,
        byte[] trieBytes) {
    int o = offset;

    // childOffset
    if (isLastChild)
        trieBytes[o] |= TrieDictionary.BIT_IS_LAST_CHILD;
    if (n.isEndOfValue)
        trieBytes[o] |= TrieDictionary.BIT_IS_END_OF_VALUE;
    o += sizeChildOffset;

    // nValueBytes
    if (n.part.length > 255)
        throw new RuntimeException(
                "Value length is " + n.part.length + " and larger than 255: " + Bytes.toStringBinary(n.part));
    BytesUtil.writeUnsigned(n.part.length, trieBytes, o, 1);
    o++;

    // valueBytes
    System.arraycopy(n.part, 0, trieBytes, o, n.part.length);
    o += n.part.length;

    if (n.isEndOfValue) {
        checkValidId(n.id);
        BytesUtil.writeUnsigned(n.id, trieBytes, o, sizeId);
        o += sizeId;
    }

    return o;
}

Example 8

Source File: RawSerializerTest.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

private List<ByteArray> getValueList(int size) {
    if (size == -1) {
        return null;
    }
    List<ByteArray> valueList = new ArrayList<ByteArray>(size);
    for (Integer i = 0; i < size; i++) {
        ByteArray key = new ByteArray(1);
        BytesUtil.writeUnsigned(i, key.array(), 0, key.length());
        valueList.add(key);
    }
    return valueList;
}

Example 9

Source File: AppendDictNode.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

private void build_overwriteChildOffset(int parentOffset, int childOffset, int sizeChildOffset, byte[] trieBytes) {
    int flags = (int) trieBytes[parentOffset]
            & (TrieDictionary.BIT_IS_LAST_CHILD | TrieDictionary.BIT_IS_END_OF_VALUE);
    BytesUtil.writeUnsigned(childOffset, trieBytes, parentOffset, sizeChildOffset);
    trieBytes[parentOffset] |= flags;
}

Example 10

Source File: AppendDictNode.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

public byte[] buildTrieBytes() {
    Stats stats = Stats.stats(this);
    int sizeChildOffset = stats.mbpn_sizeChildOffset;
    int sizeId = stats.mbpn_sizeId;

    // write head
    byte[] head;
    try {
        ByteArrayOutputStream byteBuf = new ByteArrayOutputStream();
        DataOutputStream headOut = new DataOutputStream(byteBuf);
        headOut.write(AppendTrieDictionary.HEAD_MAGIC);
        headOut.writeShort(0); // head size, will back fill
        headOut.writeInt(stats.mbpn_footprint); // body size
        headOut.writeInt(stats.nValues);
        headOut.write(sizeChildOffset);
        headOut.write(sizeId);
        headOut.close();
        head = byteBuf.toByteArray();
        BytesUtil.writeUnsigned(head.length, head, AppendTrieDictionary.HEAD_SIZE_I, 2);
    } catch (IOException e) {
        throw new RuntimeException(e); // shall not happen, as we are
    }

    byte[] trieBytes = new byte[stats.mbpn_footprint + head.length];
    System.arraycopy(head, 0, trieBytes, 0, head.length);

    LinkedList<AppendDictNode> open = new LinkedList<AppendDictNode>();
    IdentityHashMap<AppendDictNode, Integer> offsetMap = new IdentityHashMap<AppendDictNode, Integer>();

    // write body
    int o = head.length;
    offsetMap.put(this, o);
    o = build_writeNode(this, o, true, sizeChildOffset, sizeId, trieBytes);
    if (this.children.isEmpty() == false)
        open.addLast(this);

    while (open.isEmpty() == false) {
        AppendDictNode parent = open.removeFirst();
        build_overwriteChildOffset(offsetMap.get(parent), o - head.length, sizeChildOffset, trieBytes);
        for (int i = 0; i < parent.children.size(); i++) {
            AppendDictNode c = parent.children.get(i);
            boolean isLastChild = (i == parent.children.size() - 1);
            offsetMap.put(c, o);
            o = build_writeNode(c, o, isLastChild, sizeChildOffset, sizeId, trieBytes);
            if (c.children.isEmpty() == false)
                open.addLast(c);
        }
    }

    if (o != trieBytes.length)
        throw new RuntimeException();
    return trieBytes;
}

Example 11

Source File: TrieDictionaryBuilder.java From kylin with Apache License 2.0

4 votes

private void build_overwriteChildOffset(int parentOffset, int childOffset, int sizeChildOffset, byte[] trieBytes) {
    int flags = (int) trieBytes[parentOffset] & (TrieDictionary.BIT_IS_LAST_CHILD | TrieDictionary.BIT_IS_END_OF_VALUE);
    BytesUtil.writeUnsigned(childOffset, trieBytes, parentOffset, sizeChildOffset);
    trieBytes[parentOffset] |= flags;
}

Example 12

Source File: AppendDictNode.java From kylin with Apache License 2.0

4 votes

private void build_overwriteChildOffset(int parentOffset, int childOffset, int sizeChildOffset, byte[] trieBytes) {
    int flags = (int) trieBytes[parentOffset]
            & (TrieDictionary.BIT_IS_LAST_CHILD | TrieDictionary.BIT_IS_END_OF_VALUE);
    BytesUtil.writeUnsigned(childOffset, trieBytes, parentOffset, sizeChildOffset);
    trieBytes[parentOffset] |= flags;
}

Example 13

Source File: TrieDictionaryBuilder.java From Kylin with Apache License 2.0

4 votes

private void build_overwriteChildOffset(int parentOffset, int childOffset, int sizeChildOffset, byte[] trieBytes) {
    int flags = (int) trieBytes[parentOffset] & (TrieDictionary.BIT_IS_LAST_CHILD | TrieDictionary.BIT_IS_END_OF_VALUE);
    BytesUtil.writeUnsigned(childOffset, trieBytes, parentOffset, sizeChildOffset);
    trieBytes[parentOffset] |= flags;
}

Example 14

Source File: CubeHBaseEndpointRPC.java From kylin with Apache License 2.0

4 votes

private byte[] getByteArrayForShort(short v) {
    byte[] split = new byte[Bytes.SIZEOF_SHORT];
    BytesUtil.writeUnsigned(v, split, 0, Bytes.SIZEOF_SHORT);
    return split;
}

Example 15

Source File: Int4Serializer.java From kylin with Apache License 2.0

4 votes

@Override
public void serialize(Integer value, ByteBuffer out) {
    BytesUtil.writeUnsigned(value, 4, out);
}

Example 16

Source File: DictionaryDimEnc.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

@Override
public void serialize(Object value, ByteBuffer buf) {
    int id = dict.getIdFromValue(value == null ? null : value.toString(), roundingFlag);
    BytesUtil.writeUnsigned(id, dict.getSizeOfId(), buf);
}

Example 17

Source File: DictionaryDimEnc.java From kylin with Apache License 2.0

4 votes

@Override
public void serialize(Object value, ByteBuffer buf) {
    int id = dict.getIdFromValue(value == null ? null : value.toString(), roundingFlag);
    BytesUtil.writeUnsigned(id, dict.getSizeOfId(), buf);
}

Example 18

Source File: TrieDictionaryBuilder.java From kylin with Apache License 2.0

4 votes

protected byte[] buildTrieBytes(int baseId) {
    checkOverflowParts(this.root);

    Stats stats = stats();
    int sizeNoValuesBeneath = stats.mbpn_sizeNoValueBeneath;
    int sizeChildOffset = stats.mbpn_sizeChildOffset;

    if (stats.mbpn_footprint <= 0) // must never happen, but let us be cautious
        throw new IllegalStateException("Too big dictionary, dictionary cannot be bigger than 2GB");
    if (stats.mbpn_footprint > _2GB)
        throw new RuntimeException("Too big dictionary, dictionary cannot be bigger than 2GB");

    // write head
    byte[] head;
    try {
        ByteArrayOutputStream byteBuf = new ByteArrayOutputStream();
        DataOutputStream headOut = new DataOutputStream(byteBuf);
        headOut.write(TrieDictionary.MAGIC);
        headOut.writeShort(0); // head size, will back fill
        headOut.writeInt((int) stats.mbpn_footprint); // body size
        headOut.write(sizeChildOffset);
        headOut.write(sizeNoValuesBeneath);
        positiveShortPreCheck(baseId, "baseId");
        headOut.writeShort(baseId);
        positiveShortPreCheck(stats.maxValueLength, "stats.maxValueLength");
        headOut.writeShort(stats.maxValueLength);
        headOut.writeUTF(bytesConverter == null ? "" : bytesConverter.getClass().getName());
        headOut.close();
        head = byteBuf.toByteArray();
        BytesUtil.writeUnsigned(head.length, head, TrieDictionary.MAGIC_SIZE_I, 2);
    } catch (IOException e) {
        throw new RuntimeException(e); // shall not happen, as we are writing in memory
    }

    byte[] trieBytes = new byte[(int) stats.mbpn_footprint + head.length];
    System.arraycopy(head, 0, trieBytes, 0, head.length);

    LinkedList<Node> open = new LinkedList<Node>();
    IdentityHashMap<Node, Integer> offsetMap = new IdentityHashMap<Node, Integer>();

    // write body
    int o = head.length;
    offsetMap.put(root, o);
    o = build_writeNode(root, o, true, sizeNoValuesBeneath, sizeChildOffset, trieBytes);
    if (root.children.isEmpty() == false)
        open.addLast(root);

    while (open.isEmpty() == false) {
        Node parent = open.removeFirst();
        build_overwriteChildOffset(offsetMap.get(parent), o - head.length, sizeChildOffset, trieBytes);
        for (int i = 0; i < parent.children.size(); i++) {
            Node c = parent.children.get(i);
            boolean isLastChild = (i == parent.children.size() - 1);
            offsetMap.put(c, o);
            o = build_writeNode(c, o, isLastChild, sizeNoValuesBeneath, sizeChildOffset, trieBytes);
            if (c.children.isEmpty() == false)
                open.addLast(c);
        }
    }

    if (o != trieBytes.length)
        throw new RuntimeException();
    return trieBytes;
}

Example 19

Source File: BitMapContainer.java From Kylin with Apache License 2.0

4 votes

@Override
public void getValueAt(int i, ImmutableBytesWritable valueBytes) {
    int value = getValueIntAt(i);
    BytesUtil.writeUnsigned(value, temp, 0, valueLen);
    valueBytes.set(temp, 0, valueLen);
}

Example 20

Source File: AppendDictNode.java From kylin with Apache License 2.0

4 votes

public byte[] buildTrieBytes() {
    Stats stats = Stats.stats(this);
    int sizeChildOffset = stats.mbpn_sizeChildOffset;
    int sizeId = stats.mbpn_sizeId;

    // write head
    byte[] head;
    try {
        ByteArrayOutputStream byteBuf = new ByteArrayOutputStream();
        DataOutputStream headOut = new DataOutputStream(byteBuf);
        headOut.write(AppendTrieDictionary.HEAD_MAGIC);
        headOut.writeShort(0); // head size, will back fill
        headOut.writeInt(stats.mbpn_footprint); // body size
        headOut.writeInt(stats.nValues);
        headOut.write(sizeChildOffset);
        headOut.write(sizeId);
        headOut.close();
        head = byteBuf.toByteArray();
        BytesUtil.writeUnsigned(head.length, head, AppendTrieDictionary.HEAD_SIZE_I, 2);
    } catch (IOException e) {
        throw new RuntimeException(e); // shall not happen, as we are
    }

    byte[] trieBytes = new byte[stats.mbpn_footprint + head.length];
    System.arraycopy(head, 0, trieBytes, 0, head.length);

    LinkedList<AppendDictNode> open = new LinkedList<AppendDictNode>();
    IdentityHashMap<AppendDictNode, Integer> offsetMap = new IdentityHashMap<AppendDictNode, Integer>();

    // write body
    int o = head.length;
    offsetMap.put(this, o);
    o = build_writeNode(this, o, true, sizeChildOffset, sizeId, trieBytes);
    if (this.children.isEmpty() == false)
        open.addLast(this);

    while (open.isEmpty() == false) {
        AppendDictNode parent = open.removeFirst();
        build_overwriteChildOffset(offsetMap.get(parent), o - head.length, sizeChildOffset, trieBytes);
        for (int i = 0; i < parent.children.size(); i++) {
            AppendDictNode c = parent.children.get(i);
            boolean isLastChild = (i == parent.children.size() - 1);
            offsetMap.put(c, o);
            o = build_writeNode(c, o, isLastChild, sizeChildOffset, sizeId, trieBytes);
            if (c.children.isEmpty() == false)
                open.addLast(c);
        }
    }

    if (o != trieBytes.length)
        throw new RuntimeException();
    return trieBytes;
}