org.apache.kylin.common.util.BytesUtil Java Examples

@Override
public List<ByteArray> deserialize(ByteBuffer in) {
    List<ByteArray> values = new ArrayList<>();
    int size = BytesUtil.readVInt(in);
    if (size >= 0) {
        for (int i = 0; i < size; i++) {
            ByteArray ba = new ByteArray(BytesUtil.readByteArray(in));
            if (ba.length() != 0) {
                values.add(ba);
            }
        }
    } else {
        throw new RuntimeException("Read error data size:" + size);
    }
    return values;
}
 

@Override
public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
    BytesUtil.writeVInt(whenList.size(), buffer);
    for (Pair<TupleFilter, TupleExpression> whenEntry : whenList) {
        byte[] whenBytes = TupleFilterSerializer.serialize(whenEntry.getFirst(), cs);
        BytesUtil.writeByteArray(whenBytes, buffer);

        byte[] thenBytes = TupleExpressionSerializer.serialize(whenEntry.getSecond(), cs);
        BytesUtil.writeByteArray(thenBytes, buffer);
    }
    if (elseExpr != null) {
        BytesUtil.writeVInt(1, buffer);
        byte[] elseBytes = TupleExpressionSerializer.serialize(elseExpr, cs);
        BytesUtil.writeByteArray(elseBytes, buffer);
    } else {
        BytesUtil.writeVInt(-1, buffer);
    }
}
 

private List<byte[]> getRowKeysDifferentShards(byte[] halfCookedKey) {
    final short cuboidShardNum = cubeSeg.getCuboidShardNum(cuboid.getId());

    if (!cubeSeg.isEnableSharding()) {
        return Lists.newArrayList(halfCookedKey);//not shard to append at head, so it is already well cooked
    } else {
        List<byte[]> ret = Lists.newArrayList();
        for (short i = 0; i < cuboidShardNum; ++i) {
            short shard = ShardingHash.normalize(cubeSeg.getCuboidBaseShard(cuboid.getId()), i, cubeSeg.getTotalShards(cuboid.getId()));
            byte[] cookedKey = Arrays.copyOf(halfCookedKey, halfCookedKey.length);
            BytesUtil.writeShort(shard, cookedKey, 0, RowConstants.ROWKEY_SHARDID_LEN);
            ret.add(cookedKey);
        }
        return ret;
    }
}
 

private String getStatsString(byte[] region, CubeVisitResponse result) {
    StringBuilder sb = new StringBuilder();
    Stats stats = result.getStats();
    byte[] compressedRows = HBaseZeroCopyByteString.zeroCopyGetBytes(result.getCompressedRows());

    sb.append("Endpoint RPC returned from HTable ").append(cubeSeg.getStorageLocationIdentifier()).append(" Shard ")
            .append(BytesUtil.toHex(region)).append(" on host: ").append(stats.getHostname()).append(".");
    sb.append("Total scanned row: ").append(stats.getScannedRowCount()).append(". ");
    sb.append("Total scanned bytes: ").append(stats.getScannedBytes()).append(". ");
    sb.append("Total filtered row: ").append(stats.getFilteredRowCount()).append(". ");
    sb.append("Total aggred row: ").append(stats.getAggregatedRowCount()).append(". ");
    sb.append("Time elapsed in EP: ").append(stats.getServiceEndTime() - stats.getServiceStartTime())
            .append("(ms). ");
    sb.append("Server CPU usage: ").append(stats.getSystemCpuLoad()).append(", server physical mem left: ")
            .append(stats.getFreePhysicalMemorySize()).append(", server swap mem left:")
            .append(stats.getFreeSwapSpaceSize()).append(".");
    sb.append("Etc message: ").append(stats.getEtcMsg()).append(".");
    sb.append("Normal Complete: ").append(stats.getNormalComplete() == 1).append(".");
    sb.append("Compressed row size: ").append(compressedRows.length);
    return sb.toString();

}
 

@Override
public void encode(String valueStr, byte[] output, int outputOffset) {
    if (valueStr == null) {
        Arrays.fill(output, outputOffset, outputOffset + fixedLen, NULL);
        return;
    }

    long integer = Long.parseLong(valueStr);
    if (integer > CAP[fixedLen]) {
        if (avoidVerbose++ % 10000 == 0) {
            logger.warn("Expect at most " + fixedLen + " bytes, but got " + valueStr + ", will truncate, hit times:"
                    + avoidVerbose);
        }
    }

    BytesUtil.writeLong(integer, output, outputOffset, fixedLen);
}
 

@Override
public void encode(String valueStr, byte[] output, int outputOffset) {
    if (valueStr == null) {
        Arrays.fill(output, outputOffset, outputOffset + fixedLen, NULL);
        return;
    }

    long integer = Long.parseLong(valueStr);
    if (integer > CAP[fixedLen] || integer < TAIL[fixedLen]) {
        if (avoidVerbose++ % 10000 == 0) {
            logger.warn("Expect at most " + fixedLen + " bytes, but got " + valueStr + ", will truncate, hit times:" + avoidVerbose);
        }
    }

    if (integer == TAIL[fixedLen]) {
        if (avoidVerbose2++ % 10000 == 0) {
            logger.warn("Value " + valueStr + " does not fit into " + fixedLen + " bytes ");
        }
    }

    BytesUtil.writeLong(integer + CAP[fixedLen], output, outputOffset, fixedLen);//apply an offset to preserve binary order, overflow is okay
}
 

@Override
public void readFields(DataInput in) throws IOException {
    byte[] headPartial = new byte[MAGIC.length + Short.SIZE + Integer.SIZE];
    in.readFully(headPartial);

    if (BytesUtil.compareBytes(MAGIC, 0, headPartial, 0, MAGIC.length) != 0)
        throw new IllegalArgumentException("Wrong file type (magic does not match)");

    DataInputStream headIn = new DataInputStream(//
            new ByteArrayInputStream(headPartial, MAGIC_SIZE_I, headPartial.length - MAGIC_SIZE_I));
    int headSize = headIn.readShort();
    int bodyLen = headIn.readInt();
    headIn.close();

    byte[] all = new byte[headSize + bodyLen];
    System.arraycopy(headPartial, 0, all, 0, headPartial.length);
    in.readFully(all, headPartial.length, all.length - headPartial.length);

    init(all);
}
 

@Override
public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
    BytesUtil.writeVInt(whenList.size(), buffer);
    for (Pair<TupleFilter, TupleExpression> whenEntry : whenList) {
        byte[] whenBytes = TupleFilterSerializer.serialize(whenEntry.getFirst(), cs);
        BytesUtil.writeByteArray(whenBytes, buffer);

        byte[] thenBytes = TupleExpressionSerializer.serialize(whenEntry.getSecond(), cs);
        BytesUtil.writeByteArray(thenBytes, buffer);
    }
    if (elseExpr != null) {
        BytesUtil.writeVInt(1, buffer);
        byte[] elseBytes = TupleExpressionSerializer.serialize(elseExpr, cs);
        BytesUtil.writeByteArray(elseBytes, buffer);
    } else {
        BytesUtil.writeVInt(-1, buffer);
    }
}
 

@Override
public void encode(String valueStr, byte[] output, int outputOffset) {
    if (valueStr == null) {
        Arrays.fill(output, outputOffset, outputOffset + fixedLen, NULL);
        return;
    }

    long integer = Long.parseLong(valueStr);
    if (integer > CAP[fixedLen]) {
        if (avoidVerbose++ % 10000 == 0) {
            logger.warn("Expect at most " + fixedLen + " bytes, but got " + valueStr + ", will truncate, hit times:"
                    + avoidVerbose);
        }
    }

    BytesUtil.writeLong(integer, output, outputOffset, fixedLen);
}
 

@Override
public int getPartition(Object o) {
    if (initialized == false) {
        synchronized (SparkFactDistinct.class) {
            if (initialized == false) {
                init();
            }
        }
    }

    SelfDefineSortableKey skey = (SelfDefineSortableKey) o;
    Text key = skey.getText();
    if (key.getBytes()[0] == FactDistinctColumnsReducerMapping.MARK_FOR_HLL_COUNTER) {
        Long cuboidId = Bytes.toLong(key.getBytes(), 1, Bytes.SIZEOF_LONG);
        return reducerMapping.getReducerIdForCuboidRowCount(cuboidId);
    } else {
        return BytesUtil.readUnsigned(key.getBytes(), 0, 1);
    }
}
 

public static void writeDimensionEncoding(DimensionEncoding encoding, ByteBuffer out) {
    try {
        if (encoding == null) {
            BytesUtil.writeVInt(1, out);
        } else {
            BytesUtil.writeVInt(0, out);

            if (encoding instanceof DictionaryDimEnc) {
                encoding = new TrimmedDimEnc(encoding.getLengthOfEncoding());
            }
            ByteArrayOutputStream baos = new ByteArrayOutputStream();
            ObjectOutputStream oos = new ObjectOutputStream(baos);
            oos.writeObject(encoding);
            BytesUtil.writeByteArray(baos.toByteArray(), out);
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 

@Override
public TableRecordInfoDigest deserialize(ByteBuffer in) {
	TableRecordInfoDigest result = new TableRecordInfoDigest();
	result.nColumns = BytesUtil.readVInt(in);
	result.byteFormLen = BytesUtil.readVInt(in);
	result.offsets = BytesUtil.readIntArray(in);
	result.dictMaxIds = BytesUtil.readIntArray(in);
	result.lengths = BytesUtil.readIntArray(in);
	result.isMetric = BytesUtil.readBooleanArray(in);

	result.measureSerializers = new FixedLenMeasureCodec<?>[result.nColumns];
	for (int i = 0; i < result.nColumns; ++i) {
		String typeStr = BytesUtil.readAsciiString(in);
		if (typeStr == null) {
			result.measureSerializers[i] = null;
		} else {
			result.measureSerializers[i] = FixedLenMeasureCodec
					.get(DataType.getInstance(typeStr));
		}
	}

	return result;
}
 

@Override
public void serialize(IGTCodeSystem ivalue, ByteBuffer out) {
    TrimmedCubeCodeSystem value = (TrimmedCubeCodeSystem) ivalue;
    BytesUtil.writeVInt(value.dependentMetricsMap.size(), out);
    for (Map.Entry<Integer, Integer> x : value.dependentMetricsMap.entrySet()) {
        BytesUtil.writeVInt(x.getKey(), out);
        BytesUtil.writeVInt(x.getValue(), out);
    }

    BytesUtil.writeVInt(value.dimEncs.length, out);
    for (int i = 0; i < value.dimEncs.length; i++) {
        DimensionEncoding enc = value.dimEncs[i];

        writeDimensionEncoding(enc, out);
    }
}
 

public byte[] encodeRowKey(String[] keys) {
    keyByteBuffer.clear();
    for (String key : keys) {
        if (key == null) {
            throw new IllegalArgumentException("key cannot be null:" + Arrays.toString(keys));
        }
        byte[] byteKey = Bytes.toBytes(key);
        keyByteBuffer.putShort((short) byteKey.length);
        keyByteBuffer.put(byteKey);
    }
    byte[] result = new byte[RowConstants.ROWKEY_SHARDID_LEN + keyByteBuffer.position()];
    System.arraycopy(keyByteBuffer.array(), 0, result, RowConstants.ROWKEY_SHARDID_LEN, keyByteBuffer.position());
    short shard = ShardingHash.getShard(result, RowConstants.ROWKEY_SHARDID_LEN, result.length, shardNum);
    BytesUtil.writeShort(shard, result, 0, RowConstants.ROWKEY_SHARDID_LEN);
    return result;
}
 

@Override
public int partition(SelfDefineSortableKey key, int numPartitions) {
    if (initialized == false) {
        synchronized (FlinkFactDistinctColumns.class) {
            if (initialized == false) {
                init();
            }
        }
    }

    Text keyText = key.getText();
    if (keyText.getBytes()[0] == FactDistinctColumnsReducerMapping.MARK_FOR_HLL_COUNTER) {
        Long cuboidId = Bytes.toLong(keyText.getBytes(), 1, Bytes.SIZEOF_LONG);
        return reducerMapping.getReducerIdForCuboidRowCount(cuboidId);
    } else {
        return BytesUtil.readUnsigned(keyText.getBytes(), 0, 1);
    }
}
 

public static AppendDictSlice deserializeFrom(DataInput in) throws IOException {
    byte[] headPartial = new byte[HEAD_MAGIC.length + Short.SIZE / Byte.SIZE + Integer.SIZE / Byte.SIZE];
    in.readFully(headPartial);

    if (BytesUtil.compareBytes(HEAD_MAGIC, 0, headPartial, 0, HEAD_MAGIC.length) != 0)
        throw new IllegalArgumentException("Wrong file type (magic does not match)");

    DataInputStream headIn = new DataInputStream(//
            new ByteArrayInputStream(headPartial, HEAD_SIZE_I, headPartial.length - HEAD_SIZE_I));
    int headSize = headIn.readShort();
    int bodyLen = headIn.readInt();
    headIn.close();

    byte[] all = new byte[headSize + bodyLen];
    System.arraycopy(headPartial, 0, all, 0, headPartial.length);
    in.readFully(all, headPartial.length, all.length - headPartial.length);

    return new AppendDictSlice(all);
}
 

@Override
public CoprocessorRowType deserialize(ByteBuffer in) {
    int n = BytesUtil.readVInt(in);
    TblColRef[] cols = new TblColRef[n];
    int[] colSizes = new int[n];
    for (int i = 0; i < n; i++) {
        String tableName = BytesUtil.readAsciiString(in);
        String colName = BytesUtil.readAsciiString(in);
        TableDesc table = new TableDesc();
        table.setName(tableName);
        ColumnDesc col = new ColumnDesc();
        col.setTable(table);
        col.setName(colName);
        cols[i] = new TblColRef(col);

        int colSize = BytesUtil.readVInt(in);
        colSizes[i] = colSize;
    }
    return new CoprocessorRowType(cols, colSizes);
}
 

private void init() {
    if (BytesUtil.compareBytes(HEAD_MAGIC, 0, trieBytes, 0, HEAD_MAGIC.length) != 0)
        throw new IllegalArgumentException("Wrong file type (magic does not match)");

    try {
        DataInputStream headIn = new DataInputStream(
                new ByteArrayInputStream(trieBytes, HEAD_SIZE_I, trieBytes.length - HEAD_SIZE_I));
        this.headSize = headIn.readShort();
        this.bodyLen = headIn.readInt();
        this.nValues = headIn.readInt();
        this.sizeChildOffset = headIn.read();
        this.sizeOfId = headIn.read();

        this.childOffsetMask = ~(((long) (BIT_IS_LAST_CHILD | BIT_IS_END_OF_VALUE)) << ((sizeChildOffset - 1) * 8));
        this.firstByteOffset = sizeChildOffset + 1; // the offset from begin of node to its first value byte
    } catch (Exception e) {
        if (e instanceof RuntimeException)
            throw (RuntimeException) e;
        else
            throw new RuntimeException(e);
    }
}
 

private void serializeGTRecord(GTRecord gtRecord, ByteBuffer out) {
    ByteArray[] cols = gtRecord.getInternal();
    BytesUtil.writeVInt(cols.length, out);
    for (ByteArray col : cols) {
        col.exportData(out);
    }
}
 

@Override
public String decode(byte[] bytes, int offset, int len) {
    if (isNull(bytes, offset, len)) {
        return null;
    }

    long code = BytesUtil.readLong(bytes, offset, fixedLen);
    if (code < 0)
        throw new IllegalArgumentException();

    return codec.codeToValue(code);
}
 

private byte[][] getSplits(int shard) {
    byte[][] result = new byte[shard - 1][];
    for (int i = 1; i < shard; ++i) {
        byte[] split = new byte[IIKeyValueCodec.SHARD_LEN];
        BytesUtil.writeUnsigned(i, split, 0, IIKeyValueCodec.SHARD_LEN);
        result[i - 1] = split;
    }
    return result;
}
 

@Test
public void testNormal() {
    int size = 100;
    List<ByteArray> valueList = new ArrayList<ByteArray>(size);
    for (Integer i = 0; i < size; i++) {
        ByteArray key = new ByteArray(1);
        BytesUtil.writeUnsigned(i, key.array(), 0, key.length());
        valueList.add(key);
    }
    agg.aggregate(valueList);
    agg.aggregate(valueList);
    assertEquals(valueList.size() * 2, agg.getState().size());
}
 

@Override
protected void putSmallResource(String resPath, ContentWriter content, long ts) throws IOException {
    byte[] row = Bytes.toBytes(resPath);
    byte[] bytes = content.extractAllBytes();

    Table table = getConnection().getTable(TableName.valueOf(tableName));
    RollbackablePushdown pushdown = null;
    try {
        if (bytes.length > kvSizeLimit) {
            pushdown = writePushdown(resPath, ContentWriter.create(bytes));
            bytes = BytesUtil.EMPTY_BYTE_ARRAY;
        }

        Put put = new Put(row);
        put.addColumn(B_FAMILY, B_COLUMN, bytes);
        put.addColumn(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));

        table.put(put);

    } catch (Exception ex) {
        if (pushdown != null)
            pushdown.rollback();
        throw ex;
    } finally {
        if (pushdown != null)
            pushdown.close();
        IOUtils.closeQuietly(table);
    }
}
 

public boolean[] checkCache(GTRecord record) {
    if (!enabled)
        return null;

    count++;

    // disable the cache if the hit rate is bad
    if (count == CHECKPOINT) {
        if ((double) hit / (double) count < HIT_RATE_THRESHOLD) {
            enabled = false;
        }
    }

    boolean match = count > 1;
    int p = 0;
    for (int i = 0; i < colsInFilter.trueBitCount(); i++) {
        int c = colsInFilter.trueBitAt(i);
        ByteArray col = record.get(c);
        if (match) {
            match = BytesUtil.compareBytes(col.array(), col.offset(), lastValues, p, col.length()) == 0;
        }
        if (!match) {
            System.arraycopy(col.array(), col.offset(), lastValues, p, col.length());
        }
        p += col.length();
    }

    if (match) {
        hit++;
        return lastResult;
    } else {
        return null;
    }
}
 

private AppendDictNode rebuildTrieTreeR(int n, AppendDictNode parent) {
    AppendDictNode root = null;
    while (true) {
        int p = n + firstByteOffset;
        int childOffset = (int) (BytesUtil.readLong(trieBytes, n, sizeChildOffset) & childOffsetMask);
        int parLen = BytesUtil.readUnsigned(trieBytes, p - 1, 1);
        boolean isEndOfValue = checkFlag(n, BIT_IS_END_OF_VALUE);

        byte[] value = new byte[parLen];
        System.arraycopy(trieBytes, p, value, 0, parLen);

        AppendDictNode node = new AppendDictNode(value, isEndOfValue);
        if (isEndOfValue) {
            int id = BytesUtil.readUnsigned(trieBytes, p + parLen, sizeOfId);
            node.id = id;
        }

        if (parent == null) {
            root = node;
        } else {
            parent.addChild(node);
        }

        if (childOffset != 0) {
            rebuildTrieTreeR(childOffset + headSize, node);
        }

        if (checkFlag(n, BIT_IS_LAST_CHILD)) {
            break;
        } else {
            n += firstByteOffset + parLen + (isEndOfValue ? sizeOfId : 0);
        }
    }
    return root;
}
 

@Override
public GTTwoLayerAggregateParam deserialize(ByteBuffer in) {
    int ifEnabled = BytesUtil.readVInt(in);
    if (ifEnabled != 1) {
        return new GTTwoLayerAggregateParam();
    }
    ImmutableBitSet vDimMask = ImmutableBitSet.serializer.deserialize(in);
    ImmutableBitSet outLMetrics = ImmutableBitSet.serializer.deserialize(in);
    String[] outLMetricsFuncs = BytesUtil.readAsciiStringArray(in);
    int[] inLMetrics = BytesUtil.readIntArray(in);
    String[] inLMetricsFuncs = BytesUtil.readAsciiStringArray(in);

    return new GTTwoLayerAggregateParam(vDimMask, outLMetrics, outLMetricsFuncs, inLMetrics, inLMetricsFuncs);
}
 

public void writeRegisters(final ByteBuffer out) throws IOException {

        final int indexLen = getRegisterIndexSize();
        int size = size();

        // decide output scheme -- map (3*size bytes) or array (2^p bytes)
        byte scheme;
        if ((indexLen + 1) * size < m)
            scheme = 0; // map
        else
            scheme = 1; // array
        out.put(scheme);

        if (scheme == 0) { // map scheme
            BytesUtil.writeVInt(size, out);
            for (int i = 0; i < m; i++) {
                if (registers[i] > 0) {
                    BytesUtil.writeUnsigned(i, indexLen, out);
                    out.put(registers[i]);
                }
            }
        } else { // array scheme
            for (int i = 0; i < m; i++) {
                out.put(registers[i]);
            }
        }
    }
 

private GTRecord deserializeGTRecord(ByteBuffer in, GTInfo sInfo) {
    int colLength = BytesUtil.readVInt(in);
    ByteArray[] sCols = new ByteArray[colLength];
    for (int i = 0; i < colLength; i++) {
        sCols[i] = ByteArray.importData(in);
    }
    return new GTRecord(sInfo, sCols);
}
 

@SuppressWarnings({ "unchecked", "rawtypes" })
@Override
public void serialize(IFilterCodeSystem cs, ByteBuffer buffer) {
    int size = this.dynamicVariables.size();
    BytesUtil.writeVInt(size, buffer);
    for (Map.Entry<String, Object> entry : this.dynamicVariables.entrySet()) {
        BytesUtil.writeUTFString(entry.getKey(), buffer);
        cs.serialize(entry.getValue(), buffer);
    }
}
 

@Override
public BigDecimal deserialize(ByteBuffer in) {
    int scale = BytesUtil.readVInt(in);
    int n = BytesUtil.readVInt(in);

    byte[] bytes = new byte[n];
    in.get(bytes);

    return new BigDecimal(new BigInteger(bytes), scale);
}