io.airlift.slice.Slices Java Examples

public static Slice padSpaces(Slice slice, int length)
{
    int textLength = countCodePoints(slice);

    if (textLength > length) {
        throw new IllegalArgumentException("pad length is smaller than slice length");
    }

    if (textLength == length) {
        return slice;
    }

    int bufferSize = slice.length() + length - textLength;
    Slice buffer = Slices.allocate(bufferSize);

    buffer.setBytes(0, slice);

    for (int i = slice.length(); i < bufferSize; ++i) {
        buffer.setByte(i, ' ');
    }

    return buffer;
}
 

private static Block createChannel(int positionCount, int arraySize, ArrayType arrayType)
{
    BlockBuilder blockBuilder = arrayType.createBlockBuilder(null, positionCount);
    for (int position = 0; position < positionCount; position++) {
        BlockBuilder entryBuilder = blockBuilder.beginBlockEntry();
        for (int i = 0; i < arraySize; i++) {
            if (arrayType.getElementType().getJavaType() == long.class) {
                arrayType.getElementType().writeLong(entryBuilder, ThreadLocalRandom.current().nextLong());
            }
            else if (arrayType.getElementType().equals(VARCHAR)) {
                arrayType.getElementType().writeSlice(entryBuilder, Slices.utf8Slice("test_string"));
            }
            else {
                throw new UnsupportedOperationException();
            }
        }
        blockBuilder.closeEntry();
    }
    return blockBuilder.build();
}
 

@Override
public <K> Map<K, FixedLengthSliceInput> readFully(Map<K, DiskRange> diskRanges)
        throws IOException
{
    ImmutableMap.Builder<K, FixedLengthSliceInput> builder = ImmutableMap.builder();

    // Assumption here: all disk ranges are in the same region. Therefore, serving them in arbitrary order
    // will not result in eviction of cache that otherwise could have served any of the DiskRanges provided.
    for (Map.Entry<K, DiskRange> entry : diskRanges.entrySet()) {
        DiskRange diskRange = entry.getValue();
        byte[] buffer = new byte[diskRange.getLength()];
        readFully(diskRange.getOffset(), buffer);
        builder.put(entry.getKey(), Slices.wrappedBuffer(buffer).getInput());
    }
    return builder.build();
}
 

private static Block createChannel(int positionCount, int arraySize, ArrayType arrayType)
{
    BlockBuilder blockBuilder = arrayType.createBlockBuilder(null, positionCount);
    for (int position = 0; position < positionCount; position++) {
        BlockBuilder entryBuilder = blockBuilder.beginBlockEntry();
        for (int i = 0; i < arraySize; i++) {
            if (arrayType.getElementType().getJavaType() == long.class) {
                arrayType.getElementType().writeLong(entryBuilder, ThreadLocalRandom.current().nextLong());
            }
            else if (arrayType.getElementType().getJavaType() == double.class) {
                arrayType.getElementType().writeDouble(entryBuilder, ThreadLocalRandom.current().nextDouble());
            }
            else if (arrayType.getElementType().getJavaType() == boolean.class) {
                arrayType.getElementType().writeBoolean(entryBuilder, ThreadLocalRandom.current().nextBoolean());
            }
            else if (arrayType.getElementType().equals(VARCHAR)) {
                arrayType.getElementType().writeSlice(entryBuilder, Slices.utf8Slice("test_string"));
            }
            else {
                throw new UnsupportedOperationException();
            }
        }
        blockBuilder.closeEntry();
    }
    return blockBuilder.build();
}
 

public Footer(
        long numberOfRows,
        OptionalInt rowsInRowGroup,
        List<StripeInformation> stripes,
        ColumnMetadata<OrcType> types,
        Optional<ColumnMetadata<ColumnStatistics>> fileStats,
        Map<String, Slice> userMetadata)
{
    this.numberOfRows = numberOfRows;
    rowsInRowGroup.ifPresent(value -> checkArgument(value > 0, "rowsInRowGroup must be at least 1"));
    this.rowsInRowGroup = rowsInRowGroup;
    this.stripes = ImmutableList.copyOf(requireNonNull(stripes, "stripes is null"));
    this.types = requireNonNull(types, "types is null");
    this.fileStats = requireNonNull(fileStats, "fileStats is null");
    requireNonNull(userMetadata, "userMetadata is null");
    this.userMetadata = ImmutableMap.copyOf(transformValues(userMetadata, Slices::copyOf));
}
 

@Test
public void testBloomFilterPerformanceContains()
{
    BloomFilter bf = BloomFilter.newInstance();
    long start = new Date().getTime();
    Random rand = new Random();
    byte[] buf = new byte[32];
    int iterations = 100000;
    for (int i = 0; i < iterations; i++) {
        rand.nextBytes(buf);
        Slice x = Slices.wrappedBuffer(buf);
        bf.mightContain(x);
    }
    long took = new Date().getTime() - start;
    assertEquals(bf.getPreMiss(), iterations);
    assertTrue(took < 10000L);
}
 

public static Slice currentTokenAsVarchar(JsonParser parser)
        throws IOException
{
    switch (parser.currentToken()) {
        case VALUE_NULL:
            return null;
        case VALUE_STRING:
        case FIELD_NAME:
            return Slices.utf8Slice(parser.getText());
        case VALUE_NUMBER_FLOAT:
            // Avoidance of loss of precision does not seem to be possible here because of Jackson implementation.
            return DoubleOperators.castToVarchar(parser.getDoubleValue());
        case VALUE_NUMBER_INT:
            // An alternative is calling getLongValue and then BigintOperators.castToVarchar.
            // It doesn't work as well because it can result in overflow and underflow exceptions for large integral numbers.
            return Slices.utf8Slice(parser.getText());
        case VALUE_TRUE:
            return BooleanOperators.castToVarchar(true);
        case VALUE_FALSE:
            return BooleanOperators.castToVarchar(false);
        default:
            throw new JsonCastException(format("Unexpected token when cast to %s: %s", StandardTypes.VARCHAR, parser.getText()));
    }
}
 

@Test
public void testNone()
{
    AllOrNoneValueSet valueSet = AllOrNoneValueSet.none(HYPER_LOG_LOG);
    assertEquals(valueSet.getType(), HYPER_LOG_LOG);
    assertTrue(valueSet.isNone());
    assertFalse(valueSet.isAll());
    assertFalse(valueSet.isSingleValue());
    assertFalse(valueSet.containsValue(Slices.EMPTY_SLICE));

    try {
        valueSet.getSingleValue();
        fail();
    }
    catch (Exception ignored) {
    }
}
 

@Override
public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput)
{
    // positionCount
    int positionCount = sliceInput.readInt();

    // dictionary
    Block dictionaryBlock = blockEncodingSerde.readBlock(sliceInput);

    // ids
    int[] ids = new int[positionCount];
    sliceInput.readBytes(Slices.wrappedIntArray(ids));

    // instance id
    long mostSignificantBits = sliceInput.readLong();
    long leastSignificantBits = sliceInput.readLong();
    long sequenceId = sliceInput.readLong();

    // We always compact the dictionary before we send it. However, dictionaryBlock comes from sliceInput, which may over-retain memory.
    // As a result, setting dictionaryIsCompacted to true is not appropriate here.
    // TODO: fix DictionaryBlock so that dictionaryIsCompacted can be set to true when the underlying block over-retains memory.
    return new DictionaryBlock(positionCount, dictionaryBlock, ids, false, new DictionaryId(mostSignificantBits, leastSignificantBits, sequenceId));
}
 

@Override
public Object evaluate(DeferredObject[] arguments) throws HiveException {
    String source = (String) arguments[0].get();
    String pattern = (String) arguments[1].get();
    Long groupIndex = 0L;
    if (arguments.length == 3) {
        groupIndex = (Long) arguments[2].get();
    }

    if (source == null) {
        return null;
    }

    if (re2JRegexp == null) {
        re2JRegexp = new Re2JRegexp(Integer.MAX_VALUE, 5, Slices.utf8Slice(pattern));
    }

    result.clear();
    result.addAll(re2JRegexp.extractAll(Slices.utf8Slice(source), groupIndex));

    return result;
}
 

@Override
public void serialize(LongDecimalWithOverflowState state, BlockBuilder out)
{
    if (state.getLongDecimal() == null) {
        out.appendNull();
    }
    else {
        Slice slice = Slices.allocate(Long.BYTES + UnscaledDecimal128Arithmetic.UNSCALED_DECIMAL_128_SLICE_LENGTH);
        SliceOutput output = slice.getOutput();

        output.writeLong(state.getOverflow());
        output.writeBytes(state.getLongDecimal());

        VARBINARY.writeSlice(out, slice);
    }
}
 

public Slice serialize()
{
    compact();

    int requiredBytes = SizeOf.SIZE_OF_BYTE + // format
            SizeOf.SIZE_OF_INT + // max buckets
            SizeOf.SIZE_OF_INT + // entry count
            SizeOf.SIZE_OF_DOUBLE * nextIndex + // values
            SizeOf.SIZE_OF_DOUBLE * nextIndex; // weights

    return Slices.allocate(requiredBytes)
            .getOutput()
            .appendByte(FORMAT_TAG)
            .appendInt(maxBuckets)
            .appendInt(nextIndex)
            .appendBytes(Slices.wrappedDoubleArray(values, 0, nextIndex))
            .appendBytes(Slices.wrappedDoubleArray(weights, 0, nextIndex))
            .getUnderlyingSlice();
}
 

@Test
public void testRange()
{
    VarcharType type = createVarcharType(5);

    Type.Range range = type.getRange().get();

    String expectedMax = new StringBuilder()
            .appendCodePoint(Character.MAX_CODE_POINT)
            .appendCodePoint(Character.MAX_CODE_POINT)
            .appendCodePoint(Character.MAX_CODE_POINT)
            .appendCodePoint(Character.MAX_CODE_POINT)
            .appendCodePoint(Character.MAX_CODE_POINT)
            .toString();

    assertEquals(Slices.utf8Slice(""), range.getMin());
    assertEquals(Slices.utf8Slice(expectedMax), range.getMax());
}
 

@Override
public PositionLinks.Factory build()
{
    return new PositionLinks.Factory()
    {
        @Override
        public PositionLinks create(List<JoinFilterFunction> searchFunctions)
        {
            return new ArrayPositionLinks(positionLinks);
        }

        @Override
        public long checksum()
        {
            return XxHash64.hash(Slices.wrappedIntArray(positionLinks));
        }
    };
}
 

@Description("Encode binary data as base64")
@ScalarFunction
@SqlType(StandardTypes.VARCHAR)
public static Slice toBase64(@SqlType(StandardTypes.VARBINARY) Slice slice)
{
    if (slice.hasByteArray()) {
        return Slices.wrappedBuffer(Base64.getEncoder().encode(slice.toByteBuffer()));
    }
    return Slices.wrappedBuffer(Base64.getEncoder().encode(slice.getBytes()));
}
 

static Slice[] toSlices(String... values)
{
    Slice[] slices = new Slice[values.length];
    for (int i = 0; i < values.length; i++) {
        if (values[i] != null) {
            slices[i] = Slices.utf8Slice(values[i]);
        }
    }

    return slices;
}
 

@Description("Removes substrings matching a regular expression")
@ScalarFunction
@LiteralParameters("x")
@SqlType("varchar(x)")
public static Slice regexpReplace(@SqlType("varchar(x)") Slice source, @SqlType(JoniRegexpType.NAME) JoniRegexp pattern)
{
    return regexpReplace(source, pattern, Slices.EMPTY_SLICE);
}
 

@Test
public void testInformationSchemaPredicatePushdownForEmptyNames()
{
    TransactionId transactionId = transactionManager.beginTransaction(false);
    ConnectorSession session = createNewSession(transactionId);
    ConnectorMetadata metadata = new InformationSchemaMetadata("test_catalog", this.metadata);
    InformationSchemaColumnHandle tableSchemaColumn = new InformationSchemaColumnHandle("table_schema");
    InformationSchemaColumnHandle tableNameColumn = new InformationSchemaColumnHandle("table_name");
    ConnectorTableHandle tableHandle = metadata.getTableHandle(session, new SchemaTableName("information_schema", "tables"));

    // Empty schema name
    InformationSchemaTableHandle filtered = metadata.applyFilter(session, tableHandle, new Constraint(TupleDomain.withColumnDomains(
            ImmutableMap.of(tableSchemaColumn, Domain.singleValue(VARCHAR, Slices.utf8Slice(""))))))
            .map(ConstraintApplicationResult::getHandle)
            .map(InformationSchemaTableHandle.class::cast)
            .orElseThrow(AssertionError::new);

    // "" schema name is valid schema name, but is (currently) valid for QualifiedTablePrefix
    assertEquals(filtered.getPrefixes(), ImmutableSet.of(new QualifiedTablePrefix("test_catalog", "")));

    // Empty table name
    filtered = metadata.applyFilter(session, tableHandle, new Constraint(TupleDomain.withColumnDomains(
            ImmutableMap.of(tableNameColumn, Domain.singleValue(VARCHAR, Slices.utf8Slice(""))))))
            .map(ConstraintApplicationResult::getHandle)
            .map(InformationSchemaTableHandle.class::cast)
            .orElseThrow(AssertionError::new);

    // "" table name is valid schema name, but is (currently) valid for QualifiedTablePrefix
    assertEquals(filtered.getPrefixes(), ImmutableSet.of(new QualifiedTablePrefix("test_catalog", "test_schema", "")));
}
 

@Test
public void testUncomparableNone()
{
    Domain domain = Domain.none(HYPER_LOG_LOG);
    assertTrue(domain.isNone());
    assertFalse(domain.isAll());
    assertFalse(domain.isSingleValue());
    assertFalse(domain.isNullableSingleValue());
    assertFalse(domain.isNullAllowed());
    assertEquals(domain.getValues(), ValueSet.none(HYPER_LOG_LOG));
    assertEquals(domain.getType(), HYPER_LOG_LOG);
    assertFalse(domain.includesNullableValue(Slices.EMPTY_SLICE));
    assertFalse(domain.includesNullableValue(null));
    assertEquals(domain.complement(), Domain.all(HYPER_LOG_LOG));
}
 

@Override
public Slice getSlice()
{
    if (isNull()) {
        return Slices.EMPTY_SLICE;
    }

    if (fieldType == FieldType.BYTE) {
        return Slices.wrappedBuffer(value.slice());
    }

    throw new PrestoException(KINESIS_CONVERSION_NOT_SUPPORTED, format("conversion %s to Slice not supported", fieldType));
}
 

private static Object optimize(@Language("SQL") String expression)
{
    assertRoundTrip(expression);

    Expression parsedExpression = planExpression(expression);

    Map<NodeRef<Expression>, Type> expressionTypes = getTypes(TEST_SESSION, METADATA, SYMBOL_TYPES, parsedExpression);
    ExpressionInterpreter interpreter = expressionOptimizer(parsedExpression, METADATA, TEST_SESSION, expressionTypes);
    return interpreter.optimize(symbol -> {
        switch (symbol.getName().toLowerCase(ENGLISH)) {
            case "bound_integer":
                return 1234L;
            case "bound_long":
                return 1234L;
            case "bound_string":
                return utf8Slice("hello");
            case "bound_double":
                return 12.34;
            case "bound_date":
                return new LocalDate(2001, 8, 22).toDateMidnight(DateTimeZone.UTC).getMillis();
            case "bound_time":
                return new LocalTime(3, 4, 5, 321).toDateTime(new DateTime(0, DateTimeZone.UTC)).getMillis();
            case "bound_timestamp":
                return new DateTime(2001, 8, 22, 3, 4, 5, 321, DateTimeZone.UTC).getMillis();
            case "bound_pattern":
                return utf8Slice("%el%");
            case "bound_timestamp_with_timezone":
                return SqlTimestampWithTimeZone.newInstance(3, new DateTime(1970, 1, 1, 1, 0, 0, 999, DateTimeZone.UTC).getMillis(), 0, getTimeZoneKey("Z"));
            case "bound_varbinary":
                return Slices.wrappedBuffer((byte) 0xab);
            case "bound_decimal_short":
                return 12345L;
            case "bound_decimal_long":
                return Decimals.encodeUnscaledValue(new BigInteger("12345678901234567890123"));
        }

        return symbol.toSymbolReference();
    });
}
 

@Override
public ParquetDataOutput compress(BytesInput bytesInput)
        throws IOException
{
    int minCompressionBufferSize = compressor.maxCompressedLength(toIntExact(bytesInput.size()));
    byte[] compressionBuffer = new byte[minCompressionBufferSize];
    byte[] bytes = bytesInput.toByteArray();
    // TODO compressedDataSize > bytes.length?
    int compressedDataSize = compressor.compress(bytes, 0, bytes.length, compressionBuffer, 0, compressionBuffer.length);
    return createDataOutput(Slices.wrappedBuffer(compressionBuffer, 0, compressedDataSize));
}
 

public StripeFooter readStripeFooter(StripeInformation stripe, AbstractAggregatedMemoryContext systemMemoryUsage)
        throws IOException
{
    long offset = stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength();
    int tailLength = toIntExact(stripe.getFooterLength());

    // read the footer
    byte[] tailBuffer = new byte[tailLength];
    orcDataSource.readFully(offset, tailBuffer);
    try (InputStream inputStream = new OrcInputStream(orcDataSource.toString(), Slices.wrappedBuffer(tailBuffer).getInput(), compressionKind, bufferSize, systemMemoryUsage)) {
        return metadataReader.readStripeFooter(hiveWriterVersion, types, inputStream);
    }
}
 

@Override
public Page getNextPage()
{
    if (done) {
        return null;
    }

    done = true;

    PageBuilder page = new PageBuilder(1, ImmutableList.of(VARCHAR));
    page.declarePosition();
    BlockBuilder column = page.getBlockBuilder(0);
    VARCHAR.writeSlice(column, Slices.utf8Slice(result));
    return page.build();
}
 

private Map<String, Slice> toUserMetadata(List<DwrfProto.UserMetadataItem> metadataList)
{
    ImmutableMap.Builder<String, Slice> mapBuilder = ImmutableMap.builder();
    for (DwrfProto.UserMetadataItem item : metadataList) {
        mapBuilder.put(item.getName(), Slices.wrappedBuffer(item.getValue().toByteArray()));
    }
    return mapBuilder.build();
}
 

@Description("Returns the hexadecimal representation of the UTF-16BE encoding of the argument")
@ScalarFunction("char2hexint")
@SqlType(StandardTypes.VARCHAR)
public static Slice char2HexInt(@SqlType(StandardTypes.VARCHAR) Slice string)
{
    Slice utf16 = Slices.wrappedBuffer(UTF_16BE.encode(string.toStringUtf8()));
    String encoded = BaseEncoding.base16().encode(utf16.getBytes());
    return Slices.utf8Slice(encoded);
}
 

public Text evaluate(Text source, Text pattern, LongWritable groupIndex) throws HiveException {
    if (source == null) {
        return null;
    }

    if (re2JRegexp == null) {
        re2JRegexp = new Re2JRegexp(Integer.MAX_VALUE, 5, Slices.utf8Slice(pattern.toString()));
    }

    result.set(re2JRegexp.extract(Slices.utf8Slice(source.toString()), groupIndex.get()).toStringUtf8());
    return result;
}
 

@Override
public Block toBlock(Type desiredType)
{
    checkArgument(desiredType.getJavaType() == Slice.class, "type doesn't match: %s", desiredType);
    Slice values = bytes == null ? Slices.EMPTY_SLICE : Slices.wrappedBuffer(bytes);
    int numberOfRecords = numberOfRecords();
    return new VariableWidthBlock(
            numberOfRecords,
            values,
            calculateOffsets(sizes, nulls, numberOfRecords),
            Optional.ofNullable(nulls));
}
 

private static void verifySlice()
{
    Slice slice = Slices.wrappedBuffer(new byte[5]);
    slice.setByte(4, 0xDE);
    slice.setByte(3, 0xAD);
    slice.setByte(2, 0xBE);
    slice.setByte(1, 0xEF);
    if (slice.getInt(1) != 0xDEADBEEF) {
        failRequirement("Slice library produced an unexpected result");
    }
}
 

@Description("Greedily removes occurrences of a pattern in a string")
@ScalarFunction
@LiteralParameters({"x", "y"})
@SqlType("varchar(x)")
public static Slice replace(@SqlType("varchar(x)") Slice str, @SqlType("varchar(y)") Slice search)
{
    return replace(str, search, Slices.EMPTY_SLICE);
}