Java Code Examples for org.apache.kylin.common.util.BytesUtil#readUnsigned()

public byte[] getFirstValue() {
    int nodeOffset = headSize;
    ByteArrayOutputStream bytes = new ByteArrayOutputStream();
    while (true) {
        int valueLen = BytesUtil.readUnsigned(trieBytes, nodeOffset + firstByteOffset - 1, 1);
        bytes.write(trieBytes, nodeOffset + firstByteOffset, valueLen);
        if (checkFlag(nodeOffset, BIT_IS_END_OF_VALUE)) {
            break;
        }
        nodeOffset = headSize
                + (int) (BytesUtil.readLong(trieBytes, nodeOffset, sizeChildOffset) & childOffsetMask);
        if (nodeOffset == headSize) {
            break;
        }
    }
    return bytes.toByteArray();
}
 

public void readRegisters(ByteBuffer in) throws IOException {
    byte scheme = in.get();

    if (scheme == 0) { // map scheme
        clear();
        int size = BytesUtil.readVInt(in);
        if (size > m)
            throw new IllegalArgumentException("register size (" + size + ") cannot be larger than m (" + m + ")");
        int indexLen = getRegisterIndexSize();
        for (int i = 0; i < size; i++) {
            int key = BytesUtil.readUnsigned(in, indexLen);
            registers[key] = in.get();
        }
    } else { // array scheme
        in.get(registers);
    }
}
 

public byte[] getFirstValue() {
    int nodeOffset = headSize;
    ByteArrayOutputStream bytes = new ByteArrayOutputStream();
    while (true) {
        int valueLen = BytesUtil.readUnsigned(trieBytes, nodeOffset + firstByteOffset - 1, 1);
        bytes.write(trieBytes, nodeOffset + firstByteOffset, valueLen);
        if (checkFlag(nodeOffset, BIT_IS_END_OF_VALUE)) {
            break;
        }
        nodeOffset = headSize
                + (int) (BytesUtil.readLong(trieBytes, nodeOffset, sizeChildOffset) & childOffsetMask);
        if (nodeOffset == headSize) {
            break;
        }
    }
    return bytes.toByteArray();
}
 

@Override
public int partition(SelfDefineSortableKey key, int numPartitions) {
    if (initialized == false) {
        synchronized (FlinkFactDistinctColumns.class) {
            if (initialized == false) {
                init();
            }
        }
    }

    Text keyText = key.getText();
    if (keyText.getBytes()[0] == FactDistinctColumnsReducerMapping.MARK_FOR_HLL_COUNTER) {
        Long cuboidId = Bytes.toLong(keyText.getBytes(), 1, Bytes.SIZEOF_LONG);
        return reducerMapping.getReducerIdForCuboidRowCount(cuboidId);
    } else {
        return BytesUtil.readUnsigned(keyText.getBytes(), 0, 1);
    }
}
 

@Override
public int getPartition(Object o) {
    if (initialized == false) {
        synchronized (SparkFactDistinct.class) {
            if (initialized == false) {
                init();
            }
        }
    }

    SelfDefineSortableKey skey = (SelfDefineSortableKey) o;
    Text key = skey.getText();
    if (key.getBytes()[0] == FactDistinctColumnsReducerMapping.MARK_FOR_HLL_COUNTER) {
        Long cuboidId = Bytes.toLong(key.getBytes(), 1, Bytes.SIZEOF_LONG);
        return reducerMapping.getReducerIdForCuboidRowCount(cuboidId);
    } else {
        return BytesUtil.readUnsigned(key.getBytes(), 0, 1);
    }
}
 

@Override
public int getPartition(Object o) {
    if (initialized == false) {
        synchronized (SparkFactDistinct.class) {
            if (initialized == false) {
                init();
            }
        }
    }

    SelfDefineSortableKey skey = (SelfDefineSortableKey) o;
    Text key = skey.getText();
    if (key.getBytes()[0] == FactDistinctColumnsReducerMapping.MARK_FOR_HLL_COUNTER) {
        Long cuboidId = Bytes.toLong(key.getBytes(), 1, Bytes.SIZEOF_LONG);
        return reducerMapping.getReducerIdForCuboidRowCount(cuboidId);
    } else {
        return BytesUtil.readUnsigned(key.getBytes(), 0, 1);
    }
}
 

/**
 * Visit the trie tree by pre-order
 * @param n           -- the offset of current node in trieBytes
 * @param returnValue -- where return value is written to
 */
private void visitNode(int n, byte[] returnValue, int offset, List<T> result) {
    int o = offset;

    // write current node value
    int p = n + firstByteOffset;
    int len = BytesUtil.readUnsigned(trieBytes, p - 1, 1);
    System.arraycopy(trieBytes, p, returnValue, o, len);
    o += len;

    // if the value is ended
    boolean isEndOfValue = checkFlag(n, BIT_IS_END_OF_VALUE);
    if (isEndOfValue) {
        T curNodeValue = bytesConvert.convertFromBytes(returnValue, 0, o);
        result.add(curNodeValue);
    }

    // find a child to continue
    int c = getChildOffset(n);
    if (c == headSize) // has no children
        return;
    while (true) {
        visitNode(c, returnValue, o, result);
        if (checkFlag(c, BIT_IS_LAST_CHILD))
            return;
        // go to next child
        p = c + firstByteOffset;
        c = p + BytesUtil.readUnsigned(trieBytes, p - 1, 1);
    }
}
 

private AppendDictNode rebuildTrieTreeR(int n, AppendDictNode parent) {
    AppendDictNode root = null;
    while (true) {
        int p = n + firstByteOffset;
        int childOffset = (int) (BytesUtil.readLong(trieBytes, n, sizeChildOffset) & childOffsetMask);
        int parLen = BytesUtil.readUnsigned(trieBytes, p - 1, 1);
        boolean isEndOfValue = checkFlag(n, BIT_IS_END_OF_VALUE);

        byte[] value = new byte[parLen];
        System.arraycopy(trieBytes, p, value, 0, parLen);

        AppendDictNode node = new AppendDictNode(value, isEndOfValue);
        if (isEndOfValue) {
            int id = BytesUtil.readUnsigned(trieBytes, p + parLen, sizeOfId);
            node.id = id;
        }

        if (parent == null) {
            root = node;
        } else {
            parent.addChild(node);
        }

        if (childOffset != 0) {
            rebuildTrieTreeR(childOffset + headSize, node);
        }

        if (checkFlag(n, BIT_IS_LAST_CHILD)) {
            break;
        } else {
            n += firstByteOffset + parLen + (isEndOfValue ? sizeOfId : 0);
        }
    }
    return root;
}
 

@Override
public IAdvMeasureFiller getAdvancedTupleFiller(FunctionDesc function, TupleInfo tupleInfo, Map<TblColRef, Dictionary<String>> dictionaryMap) {
    final TblColRef literalCol = getRawColumn(function);
    final Dictionary<String> rawColDict = dictionaryMap.get(literalCol);
    final int literalTupleIdx = tupleInfo.hasColumn(literalCol) ? tupleInfo.getColumnIndex(literalCol) : -1;

    return new IAdvMeasureFiller() {
        private List<ByteArray> rawList;
        private Iterator<ByteArray> rawIterator;
        private int expectRow = 0;

        @SuppressWarnings("unchecked")
        @Override
        public void reload(Object measureValue) {
            this.rawList = (List<ByteArray>) measureValue;
            this.rawIterator = rawList.iterator();
            this.expectRow = 0;
        }

        @Override
        public int getNumOfRows() {
            return rawList.size();
        }

        @Override
        public void fillTuple(Tuple tuple, int row) {
            if (expectRow++ != row)
                throw new IllegalStateException();

            ByteArray raw = rawIterator.next();
            int key = BytesUtil.readUnsigned(raw.array(), raw.offset(), raw.length());
            String colValue = rawColDict.getValueFromId(key);
            tuple.setDimensionValue(literalTupleIdx, colValue);
        }
    };
}
 

/**
 * returns a code point from [0, nValues), preserving order of value, or -1
 * if not found
 *
 * @param n           -- the offset of current node
 * @param seq         -- the code point under track
 * @param returnValue -- where return value is written to
 */
private int lookupValueFromSeqNo(int n, int seq, byte[] returnValue, int offset) {
    int o = offset;
    while (true) {
        // write current node value
        int p = n + firstByteOffset;
        int len = BytesUtil.readUnsigned(trieBytes, p - 1, 1);
        System.arraycopy(trieBytes, p, returnValue, o, len);
        o += len;

        // if the value is ended
        boolean isEndOfValue = checkFlag(n, BIT_IS_END_OF_VALUE);
        if (isEndOfValue) {
            seq--;
            if (seq < 0)
                return o - offset;
        }

        // find a child to continue
        int c = getChildOffset(n);
        if (c == headSize) // has no children
            return -1; // no child? corrupted dictionary!
        int nValuesBeneath;
        while (true) {
            nValuesBeneath = BytesUtil.readUnsigned(trieBytes, c + sizeChildOffset, sizeNoValuesBeneath);
            if (seq - nValuesBeneath < 0) { // value is under this child, reset n and loop again
                n = c;
                break;
            } else { // go to next child
                seq -= nValuesBeneath;
                if (checkFlag(c, BIT_IS_LAST_CHILD))
                    return -1; // no more child? corrupted dictionary!
                p = c + firstByteOffset;
                c = p + BytesUtil.readUnsigned(trieBytes, p - 1, 1);
            }
        }
    }
}
 

private void init(byte[] trieBytes) {
    this.trieBytes = trieBytes;
    if (BytesUtil.compareBytes(MAGIC, 0, trieBytes, 0, MAGIC.length) != 0)
        throw new IllegalArgumentException("Wrong file type (magic does not match)");

    try {
        DataInputStream headIn = new DataInputStream(//
                new ByteArrayInputStream(trieBytes, MAGIC_SIZE_I, trieBytes.length - MAGIC_SIZE_I));
        this.headSize = headIn.readShort();
        this.bodyLen = headIn.readInt();
        this.sizeChildOffset = headIn.read();
        this.sizeNoValuesBeneath = headIn.read();
        this.baseId = headIn.readShort();
        this.maxValueLength = headIn.readShort();
        if (maxValueLength < 0) {
            throw new IllegalStateException("maxValueLength is negative (" + maxValueLength
                    + "). Dict value is too long, whose length is larger than " + Short.MAX_VALUE);
        }

        String converterName = headIn.readUTF();
        if (converterName.isEmpty() == false)
            setConverterByName(converterName);

        this.nValues = BytesUtil.readUnsigned(trieBytes, headSize + sizeChildOffset, sizeNoValuesBeneath);
        this.sizeOfId = BytesUtil.sizeForValue(baseId + nValues + 1L); // note baseId could raise 1 byte in ID space, +1 to reserve all 0xFF for NULL case
        this.childOffsetMask = ~((long) (BIT_IS_LAST_CHILD | BIT_IS_END_OF_VALUE) << ((sizeChildOffset - 1) * 8));
        this.firstByteOffset = sizeChildOffset + sizeNoValuesBeneath + 1; // the offset from begin of node to its first value byte
        enableCache();
    } catch (Exception e) {
        if (e instanceof RuntimeException)
            throw (RuntimeException) e;
        else
            throw new RuntimeException(e);
    }
}
 

private void init(byte[] trieBytes) {
    this.trieBytes = trieBytes;
    if (BytesUtil.compareBytes(MAGIC, 0, trieBytes, 0, MAGIC.length) != 0)
        throw new IllegalArgumentException("Wrong file type (magic does not match)");

    try {
        DataInputStream headIn = new DataInputStream(//
                new ByteArrayInputStream(trieBytes, MAGIC_SIZE_I, trieBytes.length - MAGIC_SIZE_I));
        this.headSize = headIn.readShort();
        this.bodyLen = headIn.readInt();
        this.sizeChildOffset = headIn.read();
        this.sizeNoValuesBeneath = headIn.read();
        this.baseId = headIn.readShort();
        this.maxValueLength = headIn.readShort();
        if (maxValueLength < 0) {
            throw new IllegalStateException("maxValueLength is negative (" + maxValueLength
                    + "). Dict value is too long, whose length is larger than " + Short.MAX_VALUE);
        }

        String converterName = headIn.readUTF();
        if (converterName.isEmpty() == false)
            setConverterByName(converterName);

        this.nValues = BytesUtil.readUnsigned(trieBytes, headSize + sizeChildOffset, sizeNoValuesBeneath);
        this.sizeOfId = BytesUtil.sizeForValue(baseId + nValues + 1L); // note baseId could raise 1 byte in ID space, +1 to reserve all 0xFF for NULL case
        this.childOffsetMask = ~((long) (BIT_IS_LAST_CHILD | BIT_IS_END_OF_VALUE) << ((sizeChildOffset - 1) * 8));
        this.firstByteOffset = sizeChildOffset + sizeNoValuesBeneath + 1; // the offset from begin of node to its first value byte
        enableCache();
    } catch (Exception e) {
        if (e instanceof RuntimeException)
            throw (RuntimeException) e;
        else
            throw new RuntimeException(e);
    }
}
 

private void init(byte[] trieBytes) {
    this.trieBytes = trieBytes;
    if (BytesUtil.compareBytes(HEAD_MAGIC, 0, trieBytes, 0, HEAD_MAGIC.length) != 0)
        throw new IllegalArgumentException("Wrong file type (magic does not match)");

    try {
        DataInputStream headIn = new DataInputStream( //
                new ByteArrayInputStream(trieBytes, HEAD_SIZE_I, trieBytes.length - HEAD_SIZE_I));
        this.headSize = headIn.readShort();
        this.bodyLen = headIn.readInt();
        this.sizeChildOffset = headIn.read();
        this.sizeNoValuesBeneath = headIn.read();
        this.baseId = headIn.readShort();
        this.maxValueLength = headIn.readShort();

        String converterName = headIn.readUTF();
        if (converterName.isEmpty() == false)
            this.bytesConvert = (BytesConverter<T>) ClassUtil.forName(converterName, BytesConverter.class).newInstance();

        this.nValues = BytesUtil.readUnsigned(trieBytes, headSize + sizeChildOffset, sizeNoValuesBeneath);
        this.sizeOfId = BytesUtil.sizeForValue(baseId + nValues + 1); // note baseId could raise 1 byte in ID space, +1 to reserve all 0xFF for NULL case
        this.childOffsetMask = ~((BIT_IS_LAST_CHILD | BIT_IS_END_OF_VALUE) << ((sizeChildOffset - 1) * 8));
        this.firstByteOffset = sizeChildOffset + sizeNoValuesBeneath + 1; // the offset from begin of node to its first value byte
    } catch (Exception e) {
        if (e instanceof RuntimeException)
            throw (RuntimeException) e;
        else
            throw new RuntimeException(e);
    }

    if (enableCache) {
        valueToIdCache = new SoftReference<HashMap>(new HashMap());
        idToValueCache = new SoftReference<Object[]>(new Object[nValues]);
    }
}
 

@Override
public String decode(byte[] bytes, int offset, int len) {
    int id = BytesUtil.readUnsigned(bytes, offset, len);
    try {
        String value = dict.getValueFromId(id);
        return value;
    } catch (IllegalArgumentException e) {
        logger.error("Can't get dictionary value from " + dict + " (id = " + id + ")");
        return "";
    }
}
 

@Override
public int getPartition(SelfDefineSortableKey skey, NullWritable value, int numReduceTasks) {
    return BytesUtil.readUnsigned(skey.getText().getBytes(), 0, 1);
}
 

@Override
public int getPartition(SelfDefineSortableKey skey, NullWritable value, int numReduceTasks) {
    return BytesUtil.readUnsigned(skey.getText().getBytes(), 0, 1);
}
 

@Override
public Object deserialize(ByteBuffer in) {
    int id = BytesUtil.readUnsigned(in, dict.getSizeOfId());
    return dict.getValueFromId(id);
}
 

@Override
public Integer deserialize(ByteBuffer in) {
    return BytesUtil.readUnsigned(in, 4);
}
 

@Override
public Object deserialize(ByteBuffer in) {
    int id = BytesUtil.readUnsigned(in, dict.getSizeOfId());
    return dict.getValueFromId(id);
}
 

@Override
public Integer deserialize(ByteBuffer in) {
    return BytesUtil.readUnsigned(in, 4);
}