Java Code Examples for org.apache.kylin.common.util.BytesUtil#compareBytes()

The following examples show how to use org.apache.kylin.common.util.BytesUtil#compareBytes() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AppendDictSlice.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private void init() {
    if (BytesUtil.compareBytes(HEAD_MAGIC, 0, trieBytes, 0, HEAD_MAGIC.length) != 0)
        throw new IllegalArgumentException("Wrong file type (magic does not match)");

    try {
        DataInputStream headIn = new DataInputStream(
                new ByteArrayInputStream(trieBytes, HEAD_SIZE_I, trieBytes.length - HEAD_SIZE_I));
        this.headSize = headIn.readShort();
        this.bodyLen = headIn.readInt();
        this.nValues = headIn.readInt();
        this.sizeChildOffset = headIn.read();
        this.sizeOfId = headIn.read();

        this.childOffsetMask = ~(((long) (BIT_IS_LAST_CHILD | BIT_IS_END_OF_VALUE)) << ((sizeChildOffset - 1) * 8));
        this.firstByteOffset = sizeChildOffset + 1; // the offset from begin of node to its first value byte
    } catch (Exception e) {
        if (e instanceof RuntimeException)
            throw (RuntimeException) e;
        else
            throw new RuntimeException(e);
    }
}
Example 2
Source File: AppendDictSlice.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
public static AppendDictSlice deserializeFrom(DataInput in) throws IOException {
    byte[] headPartial = new byte[HEAD_MAGIC.length + Short.SIZE / Byte.SIZE + Integer.SIZE / Byte.SIZE];
    in.readFully(headPartial);

    if (BytesUtil.compareBytes(HEAD_MAGIC, 0, headPartial, 0, HEAD_MAGIC.length) != 0)
        throw new IllegalArgumentException("Wrong file type (magic does not match)");

    DataInputStream headIn = new DataInputStream(//
            new ByteArrayInputStream(headPartial, HEAD_SIZE_I, headPartial.length - HEAD_SIZE_I));
    int headSize = headIn.readShort();
    int bodyLen = headIn.readInt();
    headIn.close();

    byte[] all = new byte[headSize + bodyLen];
    System.arraycopy(headPartial, 0, all, 0, headPartial.length);
    in.readFully(all, headPartial.length, all.length - headPartial.length);

    return new AppendDictSlice(all);
}
Example 3
Source File: TrieDictionary.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
@Override
public void readFields(DataInput in) throws IOException {
    byte[] headPartial = new byte[MAGIC.length + Short.SIZE + Integer.SIZE];
    in.readFully(headPartial);

    if (BytesUtil.compareBytes(MAGIC, 0, headPartial, 0, MAGIC.length) != 0)
        throw new IllegalArgumentException("Wrong file type (magic does not match)");

    DataInputStream headIn = new DataInputStream(//
            new ByteArrayInputStream(headPartial, MAGIC_SIZE_I, headPartial.length - MAGIC_SIZE_I));
    int headSize = headIn.readShort();
    int bodyLen = headIn.readInt();
    headIn.close();

    byte[] all = new byte[headSize + bodyLen];
    System.arraycopy(headPartial, 0, all, 0, headPartial.length);
    in.readFully(all, headPartial.length, all.length - headPartial.length);

    init(all);
}
Example 4
Source File: AppendDictSlice.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private void init() {
    if (BytesUtil.compareBytes(HEAD_MAGIC, 0, trieBytes, 0, HEAD_MAGIC.length) != 0)
        throw new IllegalArgumentException("Wrong file type (magic does not match)");

    try {
        DataInputStream headIn = new DataInputStream(
                new ByteArrayInputStream(trieBytes, HEAD_SIZE_I, trieBytes.length - HEAD_SIZE_I));
        this.headSize = headIn.readShort();
        this.bodyLen = headIn.readInt();
        this.nValues = headIn.readInt();
        this.sizeChildOffset = headIn.read();
        this.sizeOfId = headIn.read();

        this.childOffsetMask = ~(((long) (BIT_IS_LAST_CHILD | BIT_IS_END_OF_VALUE)) << ((sizeChildOffset - 1) * 8));
        this.firstByteOffset = sizeChildOffset + 1; // the offset from begin of node to its first value byte
    } catch (Exception e) {
        if (e instanceof RuntimeException)
            throw (RuntimeException) e;
        else
            throw new RuntimeException(e);
    }
}
Example 5
Source File: AppendDictSlice.java    From kylin with Apache License 2.0 6 votes vote down vote up 
public static AppendDictSlice deserializeFrom(DataInput in) throws IOException {
    byte[] headPartial = new byte[HEAD_MAGIC.length + Short.SIZE / Byte.SIZE + Integer.SIZE / Byte.SIZE];
    in.readFully(headPartial);

    if (BytesUtil.compareBytes(HEAD_MAGIC, 0, headPartial, 0, HEAD_MAGIC.length) != 0)
        throw new IllegalArgumentException("Wrong file type (magic does not match)");

    DataInputStream headIn = new DataInputStream(//
            new ByteArrayInputStream(headPartial, HEAD_SIZE_I, headPartial.length - HEAD_SIZE_I));
    int headSize = headIn.readShort();
    int bodyLen = headIn.readInt();
    headIn.close();

    byte[] all = new byte[headSize + bodyLen];
    System.arraycopy(headPartial, 0, all, 0, headPartial.length);
    in.readFully(all, headPartial.length, all.length - headPartial.length);

    return new AppendDictSlice(all);
}
Example 6
Source File: TrieDictionary.java    From kylin with Apache License 2.0 6 votes vote down vote up 
@Override
public void readFields(DataInput in) throws IOException {
    byte[] headPartial = new byte[MAGIC.length + Short.SIZE + Integer.SIZE];
    in.readFully(headPartial);

    if (BytesUtil.compareBytes(MAGIC, 0, headPartial, 0, MAGIC.length) != 0)
        throw new IllegalArgumentException("Wrong file type (magic does not match)");

    DataInputStream headIn = new DataInputStream(//
            new ByteArrayInputStream(headPartial, MAGIC_SIZE_I, headPartial.length - MAGIC_SIZE_I));
    int headSize = headIn.readShort();
    int bodyLen = headIn.readInt();
    headIn.close();

    byte[] all = new byte[headSize + bodyLen];
    System.arraycopy(headPartial, 0, all, 0, headPartial.length);
    in.readFully(all, headPartial.length, all.length - headPartial.length);

    init(all);
}
Example 7
Source File: TrieDictionary.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
@Override
public void readFields(DataInput in) throws IOException {
    byte[] headPartial = new byte[HEAD_MAGIC.length + Short.SIZE + Integer.SIZE];
    in.readFully(headPartial);

    if (BytesUtil.compareBytes(HEAD_MAGIC, 0, headPartial, 0, HEAD_MAGIC.length) != 0)
        throw new IllegalArgumentException("Wrong file type (magic does not match)");

    DataInputStream headIn = new DataInputStream( //
            new ByteArrayInputStream(headPartial, HEAD_SIZE_I, headPartial.length - HEAD_SIZE_I));
    int headSize = headIn.readShort();
    int bodyLen = headIn.readInt();
    headIn.close();

    byte[] all = new byte[headSize + bodyLen];
    System.arraycopy(headPartial, 0, all, 0, headPartial.length);
    in.readFully(all, headPartial.length, all.length - headPartial.length);

    init(all);
}
Example 8
Source File: TrieDictionary.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private void init(byte[] trieBytes) {
    this.trieBytes = trieBytes;
    if (BytesUtil.compareBytes(MAGIC, 0, trieBytes, 0, MAGIC.length) != 0)
        throw new IllegalArgumentException("Wrong file type (magic does not match)");

    try {
        DataInputStream headIn = new DataInputStream(//
                new ByteArrayInputStream(trieBytes, MAGIC_SIZE_I, trieBytes.length - MAGIC_SIZE_I));
        this.headSize = headIn.readShort();
        this.bodyLen = headIn.readInt();
        this.sizeChildOffset = headIn.read();
        this.sizeNoValuesBeneath = headIn.read();
        this.baseId = headIn.readShort();
        this.maxValueLength = headIn.readShort();
        if (maxValueLength < 0) {
            throw new IllegalStateException("maxValueLength is negative (" + maxValueLength
                    + "). Dict value is too long, whose length is larger than " + Short.MAX_VALUE);
        }

        String converterName = headIn.readUTF();
        if (converterName.isEmpty() == false)
            setConverterByName(converterName);

        this.nValues = BytesUtil.readUnsigned(trieBytes, headSize + sizeChildOffset, sizeNoValuesBeneath);
        this.sizeOfId = BytesUtil.sizeForValue(baseId + nValues + 1L); // note baseId could raise 1 byte in ID space, +1 to reserve all 0xFF for NULL case
        this.childOffsetMask = ~((long) (BIT_IS_LAST_CHILD | BIT_IS_END_OF_VALUE) << ((sizeChildOffset - 1) * 8));
        this.firstByteOffset = sizeChildOffset + sizeNoValuesBeneath + 1; // the offset from begin of node to its first value byte
        enableCache();
    } catch (Exception e) {
        if (e instanceof RuntimeException)
            throw (RuntimeException) e;
        else
            throw new RuntimeException(e);
    }
}
Example 9
Source File: HBaseReadonlyStore.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public static Cell findCell(List<Cell> cells, byte[] familyName, byte[] columnName) {
    for (Cell c : cells) {
        if (BytesUtil.compareBytes(familyName, 0, c.getFamilyArray(), c.getFamilyOffset(), familyName.length) == 0 && //
                BytesUtil.compareBytes(columnName, 0, c.getQualifierArray(), c.getQualifierOffset(), columnName.length) == 0) {
            return c;
        }
    }
    return null;
}
Example 10
Source File: GTFilterScanner.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public boolean[] checkCache(GTRecord record) {
    if (!enabled)
        return null;

    count++;

    // disable the cache if the hit rate is bad
    if (count == CHECKPOINT) {
        if ((double) hit / (double) count < HIT_RATE_THRESHOLD) {
            enabled = false;
        }
    }

    boolean match = count > 1;
    int p = 0;
    for (int i = 0; i < colsInFilter.trueBitCount(); i++) {
        int c = colsInFilter.trueBitAt(i);
        ByteArray col = record.get(c);
        if (match) {
            match = BytesUtil.compareBytes(col.array(), col.offset(), lastValues, p, col.length()) == 0;
        }
        if (!match) {
            System.arraycopy(col.array(), col.offset(), lastValues, p, col.length());
        }
        p += col.length();
    }

    if (match) {
        hit++;
        return lastResult;
    } else {
        return null;
    }
}
Example 11
Source File: TrieDictionary.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private void init(byte[] trieBytes) {
    this.trieBytes = trieBytes;
    if (BytesUtil.compareBytes(MAGIC, 0, trieBytes, 0, MAGIC.length) != 0)
        throw new IllegalArgumentException("Wrong file type (magic does not match)");

    try {
        DataInputStream headIn = new DataInputStream(//
                new ByteArrayInputStream(trieBytes, MAGIC_SIZE_I, trieBytes.length - MAGIC_SIZE_I));
        this.headSize = headIn.readShort();
        this.bodyLen = headIn.readInt();
        this.sizeChildOffset = headIn.read();
        this.sizeNoValuesBeneath = headIn.read();
        this.baseId = headIn.readShort();
        this.maxValueLength = headIn.readShort();
        if (maxValueLength < 0) {
            throw new IllegalStateException("maxValueLength is negative (" + maxValueLength
                    + "). Dict value is too long, whose length is larger than " + Short.MAX_VALUE);
        }

        String converterName = headIn.readUTF();
        if (converterName.isEmpty() == false)
            setConverterByName(converterName);

        this.nValues = BytesUtil.readUnsigned(trieBytes, headSize + sizeChildOffset, sizeNoValuesBeneath);
        this.sizeOfId = BytesUtil.sizeForValue(baseId + nValues + 1L); // note baseId could raise 1 byte in ID space, +1 to reserve all 0xFF for NULL case
        this.childOffsetMask = ~((long) (BIT_IS_LAST_CHILD | BIT_IS_END_OF_VALUE) << ((sizeChildOffset - 1) * 8));
        this.firstByteOffset = sizeChildOffset + sizeNoValuesBeneath + 1; // the offset from begin of node to its first value byte
        enableCache();
    } catch (Exception e) {
        if (e instanceof RuntimeException)
            throw (RuntimeException) e;
        else
            throw new RuntimeException(e);
    }
}
Example 12
Source File: HBaseReadonlyStore.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public static Cell findCell(List<Cell> cells, byte[] familyName, byte[] columnName) {
    for (Cell c : cells) {
        if (BytesUtil.compareBytes(familyName, 0, c.getFamilyArray(), c.getFamilyOffset(), familyName.length) == 0 && //
                BytesUtil.compareBytes(columnName, 0, c.getQualifierArray(), c.getQualifierOffset(), columnName.length) == 0) {
            return c;
        }
    }
    return null;
}
Example 13
Source File: GTFilterScanner.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public boolean[] checkCache(GTRecord record) {
    if (!enabled)
        return null;

    count++;

    // disable the cache if the hit rate is bad
    if (count == CHECKPOINT) {
        if ((double) hit / (double) count < HIT_RATE_THRESHOLD) {
            enabled = false;
        }
    }

    boolean match = count > 1;
    int p = 0;
    for (int i = 0; i < colsInFilter.trueBitCount(); i++) {
        int c = colsInFilter.trueBitAt(i);
        ByteArray col = record.get(c);
        if (match) {
            match = BytesUtil.compareBytes(col.array(), col.offset(), lastValues, p, col.length()) == 0;
        }
        if (!match) {
            System.arraycopy(col.array(), col.offset(), lastValues, p, col.length());
        }
        p += col.length();
    }

    if (match) {
        hit++;
        return lastResult;
    } else {
        return null;
    }
}
Example 14
Source File: TrieDictionary.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private void init(byte[] trieBytes) {
    this.trieBytes = trieBytes;
    if (BytesUtil.compareBytes(HEAD_MAGIC, 0, trieBytes, 0, HEAD_MAGIC.length) != 0)
        throw new IllegalArgumentException("Wrong file type (magic does not match)");

    try {
        DataInputStream headIn = new DataInputStream( //
                new ByteArrayInputStream(trieBytes, HEAD_SIZE_I, trieBytes.length - HEAD_SIZE_I));
        this.headSize = headIn.readShort();
        this.bodyLen = headIn.readInt();
        this.sizeChildOffset = headIn.read();
        this.sizeNoValuesBeneath = headIn.read();
        this.baseId = headIn.readShort();
        this.maxValueLength = headIn.readShort();

        String converterName = headIn.readUTF();
        if (converterName.isEmpty() == false)
            this.bytesConvert = (BytesConverter<T>) ClassUtil.forName(converterName, BytesConverter.class).newInstance();

        this.nValues = BytesUtil.readUnsigned(trieBytes, headSize + sizeChildOffset, sizeNoValuesBeneath);
        this.sizeOfId = BytesUtil.sizeForValue(baseId + nValues + 1); // note baseId could raise 1 byte in ID space, +1 to reserve all 0xFF for NULL case
        this.childOffsetMask = ~((BIT_IS_LAST_CHILD | BIT_IS_END_OF_VALUE) << ((sizeChildOffset - 1) * 8));
        this.firstByteOffset = sizeChildOffset + sizeNoValuesBeneath + 1; // the offset from begin of node to its first value byte
    } catch (Exception e) {
        if (e instanceof RuntimeException)
            throw (RuntimeException) e;
        else
            throw new RuntimeException(e);
    }

    if (enableCache) {
        valueToIdCache = new SoftReference<HashMap>(new HashMap());
        idToValueCache = new SoftReference<Object[]>(new Object[nValues]);
    }
}