Java Code Examples for io.airlift.slice.Slice#getBytes()

/**
 * Recursive helper function used by {@link AccumuloRowSerializer#getArrayFromBlock} and
 * {@link AccumuloRowSerializer#getMapFromBlock} to decode the Block into a Java type.
 *
 * @param type Presto type
 * @param block Block to decode
 * @param position Position in the block to get
 * @return Java object from the Block
 */
static Object readObject(Type type, Block block, int position)
{
    if (Types.isArrayType(type)) {
        Type elementType = Types.getElementType(type);
        return getArrayFromBlock(elementType, block.getObject(position, Block.class));
    }
    else if (Types.isMapType(type)) {
        return getMapFromBlock(type, block.getObject(position, Block.class));
    }
    else {
        if (type.getJavaType() == Slice.class) {
            Slice slice = (Slice) TypeUtils.readNativeValue(type, block, position);
            return type.equals(VarcharType.VARCHAR) ? slice.toStringUtf8() : slice.getBytes();
        }

        return TypeUtils.readNativeValue(type, block, position);
    }
}
 

static Object readObject(Type type, Block block, int position)
{
    if (Types.isArrayType(type)) {
        Type elementType = Types.getElementType(type);
        return getArrayFromBlock(elementType, block.getObject(position, Block.class));
    }
    else if (Types.isMapType(type)) {
        return getMapFromBlock(type, block.getObject(position, Block.class));
    }
    else {
        if (type.getJavaType() == Slice.class) {
            Slice slice = (Slice) TypeUtils.readNativeValue(type, block, position);
            return (type instanceof VarcharType) ? slice.toStringUtf8() : slice.getBytes();
        }

        return TypeUtils.readNativeValue(type, block, position);
    }
}
 

/**
 * @param type Presto type
 * @param block Block to decode
 * @param position Position in the block to get
 * @return Java object from the Block
 */
private static Object readObject(Type type, Block block, int position)
{
    if (Types.isArrayType(type)) {
        Type elementType = Types.getElementType(type);
        return getArrayFromBlock(elementType, block.getObject(position, Block.class));
    }
    else if (Types.isMapType(type)) {
        return getMapFromBlock(type, block.getObject(position, Block.class));
    }
    else {
        if (type.getJavaType() == Slice.class) {
            Slice slice = (Slice) TypeUtils.readNativeValue(type, block, position);
            return type.equals(VarcharType.VARCHAR) ? slice.toStringUtf8() : slice.getBytes();
        }

        return TypeUtils.readNativeValue(type, block, position);
    }
}
 

@Override
public Block decodeColumn(ColumnData columnData)
{
    int size = columnData.rowCount();
    BlockBuilder builder = type.createBlockBuilder(null, size);

    Slice slice = columnData.getSlice();
    for (int i = 0; i < size; i++) {
        int offset = columnData.getOffset(i);
        int length = columnData.getLength(i);
        if (nullSequence.equals(0, nullSequence.length(), slice, offset, length)) {
            builder.appendNull();
        }
        else {
            byte[] data = slice.getBytes(offset, length);
            type.writeSlice(builder, Slices.wrappedBuffer(base64Decoder.decode(data)));
        }
    }
    return builder.build();
}
 

@ScalarOperator(CAST)
@SqlType(StandardTypes.IPADDRESS)
public static Slice castFromVarbinaryToIpAddress(@SqlType("varbinary") Slice slice)
{
    if (slice.length() == 4) {
        byte[] address = slice.getBytes();
        byte[] bytes = new byte[16];
        bytes[10] = (byte) 0xff;
        bytes[11] = (byte) 0xff;
        arraycopy(address, 0, bytes, 12, 4);
        return wrappedBuffer(bytes);
    }
    if (slice.length() == 16) {
        return slice;
    }
    throw new PrestoException(INVALID_CAST_ARGUMENT, "Invalid IP address binary length: " + slice.length());
}
 

public boolean mightContain(Slice s)
{
    byte[] b = s.getBytes();
    if (USE_PRE_FILTER) {
        if (instancePreFilter.contains(b)) {
            return instance.contains(b);
        }
        else {
            preMiss++;
            return false;
        }
    }
    else {
        return instance.contains(b);
    }
}
 

@Override
public void encodeColumn(Block block, SliceOutput output, EncodeOutput encodeOutput)
{
    for (int position = 0; position < block.getPositionCount(); position++) {
        if (block.isNull(position)) {
            output.writeBytes(nullSequence);
        }
        else {
            Slice slice = type.getSlice(block, position);
            byte[] data = slice.getBytes();
            slice = Slices.wrappedBuffer(base64Encoder.encode(data));
            output.writeBytes(slice);
        }
        encodeOutput.closeEntry();
    }
}
 

@SqlType(StandardTypes.BOOLEAN)
@Nullable
@SqlNullable
public static Boolean bloomFilterPersist(@SqlNullable @SqlType(BloomFilterType.TYPE) Slice bloomFilterSlice, @SqlType(StandardTypes.VARCHAR) Slice urlSlice) throws Exception
{
    // Nothing todo
    if (urlSlice == null) {
        return true;
    }
    BloomFilter bf = getOrLoadBloomFilter(bloomFilterSlice);

    // Persist
    // we do not try catch here to make sure that errors are communicated clearly to the client
    // and typical retry logic continues to work
    String url = new String(urlSlice.getBytes());
    if (!HTTP_CLIENT.isStarted()) {
        log.warn("Http client was not started, trying to start");
        HTTP_CLIENT.start();
    }
    Request post = HTTP_CLIENT.POST(url);
    post.content(new StringContentProvider(new String(bf.toBase64())));
    post.method("PUT");
    post.send();
    log.info("Persisted " + bf.toString() + " " + url);
    return true;
}
 

public BloomFilter put(Slice s)
{
    if (s == null) {
        return this;
    }
    byte[] b = s.getBytes();
    if (b.length < 1) {
        return this;
    }
    instance.add(b);
    if (USE_PRE_FILTER) {
        instancePreFilter.add(b);
    }
    return this;
}
 

private void writeSliceBlock(BlockBuilder blockBuilder, Type columnType, int columnIndex)
{
    for (int i = 0; i < currentDataTable.getDataTable().getNumberOfRows(); i++) {
        Slice slice = getSlice(i, columnIndex);
        columnType.writeSlice(blockBuilder, slice, 0, slice.length());
        completedBytes += slice.getBytes().length;
    }
}
 

@Override
public void encodeValueInto(int depth, Block block, int position, SliceOutput output)
{
    Slice slice = type.getSlice(block, position);
    byte[] data = slice.getBytes();
    slice = Slices.wrappedBuffer(base64Encoder.encode(data));
    output.writeBytes(slice);
}
 

private void writeToOutputStream(Slice source, int sourceIndex, int length)
{
    try {
        source.getBytes(sourceIndex, outputStream, length);
    }
    catch (IOException e) {
        throw new UncheckedIOException(e);
    }
}
 

private static void assertByteCountWithoutTrailingSpace(byte[] actual, int offset, int length, int codePointCount, byte[] expected)
{
    Slice slice = wrappedBuffer(actual);
    int truncatedLength = byteCountWithoutTrailingSpace(slice, offset, length, codePointCount);
    byte[] bytes = slice.getBytes(offset, truncatedLength);
    assertEquals(bytes, expected);
}
 

private static void assertByteCount(byte[] actual, int offset, int length, int codePointCount, byte[] expected)
{
    Slice slice = wrappedBuffer(actual);
    int truncatedLength = byteCount(slice, offset, length, codePointCount);
    byte[] bytes = slice.getBytes(offset, truncatedLength);
    assertEquals(bytes, expected);
}
 

public static BigInteger unscaledDecimalToBigInteger(Slice decimal)
{
    byte[] bytes = decimal.getBytes(0, UNSCALED_DECIMAL_128_SLICE_LENGTH);
    // convert to big-endian order
    reverse(bytes);
    bytes[0] &= ~SIGN_BYTE_MASK;
    return new BigInteger(isNegative(decimal) ? -1 : 1, bytes);
}
 

public static JoniRegexp joniRegexp(Slice pattern)
{
    Regex regex;
    try {
        // When normal UTF8 encoding instead of non-strict UTF8) is used, joni can infinite loop when invalid UTF8 slice is supplied to it.
        regex = new Regex(pattern.getBytes(), 0, pattern.length(), Option.DEFAULT, NonStrictUTF8Encoding.INSTANCE, Syntax.Java);
    }
    catch (Exception e) {
        throw new PrestoException(INVALID_FUNCTION_ARGUMENT, e);
    }
    return new JoniRegexp(pattern, regex);
}
 

public static PrestoThriftBlock fromSliceBasedBlock(Block block, Type type, CreateSliceThriftBlockFunction create)
{
    int positions = block.getPositionCount();
    if (positions == 0) {
        return create.apply(null, null, null);
    }
    boolean[] nulls = null;
    int[] sizes = null;
    byte[] bytes = null;
    int bytesIndex = 0;
    for (int position = 0; position < positions; position++) {
        if (block.isNull(position)) {
            if (nulls == null) {
                nulls = new boolean[positions];
            }
            nulls[position] = true;
        }
        else {
            Slice value = type.getSlice(block, position);
            if (sizes == null) {
                sizes = new int[positions];
                int totalBytes = totalSliceBytes(block);
                if (totalBytes > 0) {
                    bytes = new byte[totalBytes];
                }
            }
            int length = value.length();
            sizes[position] = length;
            if (length > 0) {
                checkState(bytes != null);
                value.getBytes(0, bytes, bytesIndex, length);
                bytesIndex += length;
            }
        }
    }
    checkState(bytes == null || bytesIndex == bytes.length);
    return create.apply(nulls, sizes, bytes);
}
 

private static Slice jsonParse(Slice slice)
{
    try (JsonParser parser = createJsonParser(slice)) {
        byte[] in = slice.getBytes();
        SliceOutput dynamicSliceOutput = new DynamicSliceOutput(in.length);
        SORTED_MAPPER.writeValue((OutputStream) dynamicSliceOutput, SORTED_MAPPER.readValue(parser, Object.class));
        // nextToken() returns null if the input is parsed correctly,
        // but will throw an exception if there are trailing characters.
        parser.nextToken();
        return dynamicSliceOutput.slice();
    }
    catch (Exception e) {
        throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("Cannot convert '%s' to JSON", slice.toStringUtf8()));
    }
}
 

private static Object nativeContainerToOrcValue(Type type, Object nativeValue)
{
    if (nativeValue == null) {
        return null;
    }
    if (type instanceof DecimalType) {
        BigInteger unscaledValue;
        DecimalType decimalType = (DecimalType) type;
        if (decimalType.isShort()) {
            unscaledValue = BigInteger.valueOf((long) nativeValue);
        }
        else {
            unscaledValue = Decimals.decodeUnscaledValue((Slice) nativeValue);
        }
        return HiveDecimal.create(unscaledValue, decimalType.getScale());
    }
    if (type.getJavaType() == boolean.class) {
        return nativeValue;
    }
    if (type.getJavaType() == long.class) {
        return nativeValue;
    }
    if (type.getJavaType() == double.class) {
        return nativeValue;
    }
    if (type.getJavaType() == Slice.class) {
        Slice slice = (Slice) nativeValue;
        return type instanceof VarcharType ? slice.toStringUtf8() : slice.getBytes();
    }
    if (isArrayType(type)) {
        Block arrayBlock = (Block) nativeValue;
        Type elementType = type.getTypeParameters().get(0);
        List<Object> list = new ArrayList<>();
        for (int i = 0; i < arrayBlock.getPositionCount(); i++) {
            list.add(nativeContainerToOrcValue(elementType, getNativeContainerValue(elementType, arrayBlock, i)));
        }
        return list;
    }
    if (isMapType(type)) {
        Block mapBlock = (Block) nativeValue;
        Type keyType = type.getTypeParameters().get(0);
        Type valueType = type.getTypeParameters().get(1);
        Map<Object, Object> map = new HashMap<>();
        for (int i = 0; i < mapBlock.getPositionCount(); i += 2) {
            Object key = nativeContainerToOrcValue(keyType, getNativeContainerValue(keyType, mapBlock, i));
            Object value = nativeContainerToOrcValue(valueType, getNativeContainerValue(valueType, mapBlock, i + 1));
            map.put(key, value);
        }
        return map;
    }
    throw new PrestoException(GENERIC_INTERNAL_ERROR, "Unimplemented type: " + type);
}
 

private static void appendReplacement(SliceOutput result, Slice source, Regex pattern, Region region, Slice replacement)
{
    // Handle the following items:
    // 1. ${name};
    // 2. $0, $1, $123 (group 123, if exists; or group 12, if exists; or group 1);
    // 3. \\, \$, \t (literal 't').
    // 4. Anything that doesn't starts with \ or $ is considered regular bytes

    int idx = 0;

    while (idx < replacement.length()) {
        byte nextByte = replacement.getByte(idx);
        if (nextByte == '$') {
            idx++;
            if (idx == replacement.length()) { // not using checkArgument because `.toStringUtf8` is expensive
                throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Illegal replacement sequence: " + replacement.toStringUtf8());
            }
            nextByte = replacement.getByte(idx);
            int backref;
            if (nextByte == '{') { // case 1 in the above comment
                idx++;
                int startCursor = idx;
                while (idx < replacement.length()) {
                    nextByte = replacement.getByte(idx);
                    if (nextByte == '}') {
                        break;
                    }
                    idx++;
                }
                byte[] groupName = replacement.getBytes(startCursor, idx - startCursor);
                try {
                    backref = pattern.nameToBackrefNumber(groupName, 0, groupName.length, region);
                }
                catch (ValueException e) {
                    throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Illegal replacement sequence: unknown group { " + new String(groupName, StandardCharsets.UTF_8) + " }");
                }
                idx++;
            }
            else { // case 2 in the above comment
                backref = nextByte - '0';
                if (backref < 0 || backref > 9) { // not using checkArgument because `.toStringUtf8` is expensive
                    throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Illegal replacement sequence: " + replacement.toStringUtf8());
                }
                if (region.numRegs <= backref) {
                    throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Illegal replacement sequence: unknown group " + backref);
                }
                idx++;
                while (idx < replacement.length()) { // Adaptive group number: find largest group num that is not greater than actual number of groups
                    int nextDigit = replacement.getByte(idx) - '0';
                    if (nextDigit < 0 || nextDigit > 9) {
                        break;
                    }
                    int newBackref = (backref * 10) + nextDigit;
                    if (region.numRegs <= newBackref) {
                        break;
                    }
                    backref = newBackref;
                    idx++;
                }
            }
            int beg = region.beg[backref];
            int end = region.end[backref];
            if (beg != -1 && end != -1) { // the specific group doesn't exist in the current match, skip
                result.appendBytes(source.slice(beg, end - beg));
            }
        }
        else { // case 3 and 4 in the above comment
            if (nextByte == '\\') {
                idx++;
                if (idx == replacement.length()) { // not using checkArgument because `.toStringUtf8` is expensive
                    throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Illegal replacement sequence: " + replacement.toStringUtf8());
                }
                nextByte = replacement.getByte(idx);
            }
            result.appendByte(nextByte);
            idx++;
        }
    }
}