Java Code Examples for io.airlift.slice.Slices#allocate()

public static Slice padSpaces(Slice slice, int length)
{
    int textLength = countCodePoints(slice);

    if (textLength > length) {
        throw new IllegalArgumentException("pad length is smaller than slice length");
    }

    if (textLength == length) {
        return slice;
    }

    int bufferSize = slice.length() + length - textLength;
    Slice buffer = Slices.allocate(bufferSize);

    buffer.setBytes(0, slice);

    for (int i = slice.length(); i < bufferSize; ++i) {
        buffer.setByte(i, ' ');
    }

    return buffer;
}
 

@Override
public void serialize(LongDecimalWithOverflowState state, BlockBuilder out)
{
    if (state.getLongDecimal() == null) {
        out.appendNull();
    }
    else {
        Slice slice = Slices.allocate(Long.BYTES + UnscaledDecimal128Arithmetic.UNSCALED_DECIMAL_128_SLICE_LENGTH);
        SliceOutput output = slice.getOutput();

        output.writeLong(state.getOverflow());
        output.writeBytes(state.getLongDecimal());

        VARBINARY.writeSlice(out, slice);
    }
}
 

@Override
public void serialize(LongDecimalWithOverflowAndLongState state, BlockBuilder out)
{
    if (state.getLongDecimal() == null) {
        out.appendNull();
    }
    else {
        Slice slice = Slices.allocate(Long.BYTES + Long.BYTES + UnscaledDecimal128Arithmetic.UNSCALED_DECIMAL_128_SLICE_LENGTH);
        SliceOutput output = slice.getOutput();

        output.writeLong(state.getLong());
        output.writeLong(state.getOverflow());
        output.writeBytes(state.getLongDecimal());

        VARBINARY.writeSlice(out, slice);
    }
}
 

private static List<Long> getSyncPositionsBruteForce(RcFileReader recordReader, File file)
{
    Slice slice = Slices.allocate(toIntExact(file.length()));
    try (InputStream in = new FileInputStream(file)) {
        slice.setBytes(0, in, slice.length());
    }
    catch (IOException e) {
        throw new UncheckedIOException(e);
    }

    List<Long> syncPositionsBruteForce = new ArrayList<>();
    Slice sync = Slices.allocate(SIZE_OF_INT + SIZE_OF_LONG + SIZE_OF_LONG);
    sync.setInt(0, -1);
    sync.setBytes(SIZE_OF_INT, recordReader.getSync());

    long syncPosition = 0;
    while (syncPosition >= 0) {
        syncPosition = slice.indexOf(sync, toIntExact(syncPosition));
        if (syncPosition > 0) {
            syncPositionsBruteForce.add(syncPosition);
            syncPosition++;
        }
    }
    return syncPositionsBruteForce;
}
 

@Description("Concatenates given character strings")
@ScalarFunction
@LiteralParameters({"x", "y", "u"})
@Constraint(variable = "u", expression = "x + y")
@SqlType("char(u)")
public static Slice concat(@LiteralParameter("x") Long x, @SqlType("char(x)") Slice left, @SqlType("char(y)") Slice right)
{
    int rightLength = right.length();
    if (rightLength == 0) {
        return left;
    }

    Slice paddedLeft = padSpaces(left, x.intValue());
    int leftLength = paddedLeft.length();

    Slice result = Slices.allocate(leftLength + rightLength);
    result.setBytes(0, paddedLeft);
    result.setBytes(leftLength, right);

    return result;
}
 

public static SetDigest newInstance(Slice serialized)
{
    requireNonNull(serialized, "serialized is null");
    SliceInput input = serialized.getInput();
    checkArgument(input.readByte() == UNCOMPRESSED_FORMAT, "Unexpected version");

    int hllLength = input.readInt();
    Slice serializedHll = Slices.allocate(hllLength);
    input.readBytes(serializedHll, hllLength);
    HyperLogLog hll = HyperLogLog.newInstance(serializedHll);

    Long2ShortRBTreeMap minhash = new Long2ShortRBTreeMap();
    int maxHashes = input.readInt();
    int minhashLength = input.readInt();
    // The values are stored after the keys
    SliceInput valuesInput = serialized.getInput();
    valuesInput.setPosition(input.position() + minhashLength * SIZE_OF_LONG);

    for (int i = 0; i < minhashLength; i++) {
        minhash.put(input.readLong(), valuesInput.readShort());
    }

    return new SetDigest(maxHashes, hll, minhash);
}
 

@Override
public Optional<Range> getRange()
{
    if (length > 100) {
        // The max/min values may be materialized in the plan, so we don't want them to be too large.
        // Range comparison against large values are usually nonsensical, too, so no need to support them
        // beyond a certain size. They specific choice above is arbitrary and can be adjusted if needed.
        return Optional.empty();
    }

    int codePointSize = SliceUtf8.lengthOfCodePoint(MAX_CODE_POINT);

    Slice max = Slices.allocate(codePointSize * length);
    int position = 0;
    for (int i = 0; i < length; i++) {
        position += SliceUtf8.setCodePointAt(MAX_CODE_POINT, max, position);
    }

    return Optional.of(new Range(Slices.EMPTY_SLICE, max));
}
 

@Test
public void test()
        throws IOException
{
    List<List<Slice>> groups = new ArrayList<>();
    for (int groupIndex = 0; groupIndex < 3; groupIndex++) {
        List<Slice> group = new ArrayList<>();
        for (int i = 0; i < 1000; i++) {
            Slice value = Slices.allocate(8);
            SliceOutput output = value.getOutput();
            output.writeInt(groupIndex);
            output.writeInt(i);
            group.add(value);
        }
        groups.add(group);
    }
    testWriteValue(groups);
}
 

@Description("Encode value as a big endian varbinary according to IEEE 754 single-precision floating-point format")
@ScalarFunction("to_ieee754_32")
@SqlType(StandardTypes.VARBINARY)
public static Slice toIEEE754Binary32(@SqlType(StandardTypes.REAL) long value)
{
    Slice slice = Slices.allocate(Float.BYTES);
    slice.setInt(0, Integer.reverseBytes((int) value));
    return slice;
}
 

/**
 * Serializes the model using the following format
 * int: format version
 * byte[32]: SHA256 hash of all following data
 * int: id of algorithm
 * int: length of hyperparameters section
 * byte[]: hyperparameters (currently not used)
 * long: length of data section
 * byte[]: model data
 * <p>
 * note: all multibyte values are in little endian
 */
public static Slice serialize(Model model)
{
    requireNonNull(model, "model is null");
    Integer id = MODEL_SERIALIZATION_IDS.get(model.getClass());
    requireNonNull(id, "id is null");
    int size = HYPERPARAMETERS_OFFSET;

    // hyperparameters aren't implemented yet
    byte[] hyperparameters = new byte[0];
    size += hyperparameters.length;

    int dataLengthOffset = size;
    size += SIZE_OF_LONG;
    int dataOffset = size;
    byte[] data = model.getSerializedData();
    size += data.length;

    Slice slice = Slices.allocate(size);
    slice.setInt(VERSION_OFFSET, CURRENT_FORMAT_VERSION);
    slice.setInt(ALGORITHM_OFFSET, id);
    slice.setInt(HYPERPARAMETER_LENGTH_OFFSET, hyperparameters.length);
    slice.setBytes(HYPERPARAMETERS_OFFSET, hyperparameters);
    slice.setLong(dataLengthOffset, data.length);
    slice.setBytes(dataOffset, data);

    byte[] modelHash = Hashing.sha256().hashBytes(slice.getBytes(ALGORITHM_OFFSET, slice.length() - ALGORITHM_OFFSET)).asBytes();
    checkState(modelHash.length == 32, "sha256 hash code expected to be 32 bytes");
    slice.setBytes(HASH_OFFSET, modelHash);

    return slice;
}
 

private static Slice offsetHeapSlice(String value)
{
    Slice source = Slices.utf8Slice(value);
    Slice result = Slices.allocate(source.length() + 5);
    result.setBytes(2, source);
    return result.slice(2, source.length());
}
 

@Description("Compute SpookyHashV2 32-bit hash")
@ScalarFunction
@SqlType(StandardTypes.VARBINARY)
public static Slice spookyHashV2_32(@SqlType(StandardTypes.VARBINARY) Slice slice)
{
    Slice hash = Slices.allocate(Integer.BYTES);
    hash.setInt(0, Integer.reverseBytes(SpookyHashV2.hash32(slice, 0, slice.length(), 0)));
    return hash;
}
 

@Description("Compute xxhash64 hash")
@ScalarFunction
@SqlType(StandardTypes.VARBINARY)
public static Slice xxhash64(@SqlType(StandardTypes.VARBINARY) Slice slice)
{
    Slice hash = Slices.allocate(Long.BYTES);
    hash.setLong(0, Long.reverseBytes(XxHash64.hash(slice)));
    return hash;
}
 

@Description("Encode value as a big endian varbinary according to IEEE 754 double-precision floating-point format")
@ScalarFunction("to_ieee754_64")
@SqlType(StandardTypes.VARBINARY)
public static Slice toIEEE754Binary64(@SqlType(StandardTypes.DOUBLE) double value)
{
    Slice slice = Slices.allocate(Double.BYTES);
    slice.setLong(0, Long.reverseBytes(Double.doubleToLongBits(value)));
    return slice;
}
 

public Slice getSync()
{
    Slice sync = Slices.allocate(SIZE_OF_LONG + SIZE_OF_LONG);
    sync.setLong(0, syncFirst);
    sync.setLong(SIZE_OF_LONG, syncSecond);
    return sync;
}
 

@Description("Encode value as a 64-bit 2's complement big endian varbinary")
@ScalarFunction("to_big_endian_64")
@SqlType(StandardTypes.VARBINARY)
public static Slice toBigEndian64(@SqlType(StandardTypes.BIGINT) long value)
{
    Slice slice = Slices.allocate(Long.BYTES);
    slice.setLong(0, Long.reverseBytes(value));
    return slice;
}
 

@Description("Translate characters from the source string based on original and translations strings")
@ScalarFunction
@LiteralParameters({"x", "y", "z"})
@SqlType(StandardTypes.VARCHAR)
public static Slice translate(@SqlType("varchar(x)") Slice source, @SqlType("varchar(y)") Slice from, @SqlType("varchar(z)") Slice to)
{
    int[] fromCodePoints = castToCodePoints(from);
    int[] toCodePoints = castToCodePoints(to);

    Int2IntOpenHashMap map = new Int2IntOpenHashMap(fromCodePoints.length);
    for (int index = 0; index < fromCodePoints.length; index++) {
        int fromCodePoint = fromCodePoints[index];
        map.putIfAbsent(fromCodePoint, index < toCodePoints.length ? toCodePoints[index] : -1);
    }

    int[] sourceCodePoints = castToCodePoints(source);
    int[] targetCodePoints = new int[sourceCodePoints.length];
    int targetPositions = 0;
    int targetBytes = 0;
    for (int index = 0; index < sourceCodePoints.length; index++) {
        int codePoint = sourceCodePoints[index];
        if (map.containsKey(codePoint)) {
            int translatedCodePoint = map.get(codePoint);
            if (translatedCodePoint == -1) {
                continue;
            }
            codePoint = translatedCodePoint;
        }
        targetCodePoints[targetPositions++] = codePoint;
        targetBytes += lengthOfCodePoint(codePoint);
    }

    Slice target = Slices.allocate(targetBytes);
    int offset = 0;
    for (int index = 0; index < targetPositions; index++) {
        offset += setCodePointAt(targetCodePoints[index], target, offset);
    }

    return target;
}
 

public static Slice unscaledDecimal()
{
    return Slices.allocate(UNSCALED_DECIMAL_128_SLICE_LENGTH);
}
 

public static Slice varcharToVarcharCreateSliceWithExtraParameterLength(Slice string, int extraParameter)
{
    return Slices.allocate(extraParameter);
}
 

@Override
public Slice readFully(long position, int length)
{
    return Slices.allocate(length);
}