it.unimi.dsi.fastutil.booleans.BooleanList Java Examples

private ColumnChunk readMap(GroupField field)
        throws IOException
{
    List<Type> parameters = field.getType().getTypeParameters();
    checkArgument(parameters.size() == 2, "Maps must have two type parameters, found %s", parameters.size());
    Block[] blocks = new Block[parameters.size()];

    ColumnChunk columnChunk = readColumnChunk(field.getChildren().get(0).get());
    blocks[0] = columnChunk.getBlock();
    blocks[1] = readColumnChunk(field.getChildren().get(1).get()).getBlock();
    IntList offsets = new IntArrayList();
    BooleanList valueIsNull = new BooleanArrayList();
    calculateCollectionOffsets(field, offsets, valueIsNull, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels());
    Block mapBlock = ((MapType) field.getType()).createBlockFromKeyValue(Optional.of(valueIsNull.toBooleanArray()), offsets.toIntArray(), blocks[0], blocks[1]);
    return new ColumnChunk(mapBlock, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels());
}
 

private ColumnChunk readStruct(GroupField field)
        throws IOException
{
    List<TypeSignatureParameter> fields = field.getType().getTypeSignature().getParameters();
    Block[] blocks = new Block[fields.size()];
    ColumnChunk columnChunk = null;
    List<Optional<Field>> parameters = field.getChildren();
    for (int i = 0; i < fields.size(); i++) {
        Optional<Field> parameter = parameters.get(i);
        if (parameter.isPresent()) {
            columnChunk = readColumnChunk(parameter.get());
            blocks[i] = columnChunk.getBlock();
        }
    }
    for (int i = 0; i < fields.size(); i++) {
        if (blocks[i] == null) {
            blocks[i] = RunLengthEncodedBlock.create(field.getType(), null, columnChunk.getBlock().getPositionCount());
        }
    }
    BooleanList structIsNull = StructColumnReader.calculateStructOffsets(field, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels());
    boolean[] structIsNullVector = structIsNull.toBooleanArray();
    Block rowBlock = RowBlock.fromFieldBlocks(structIsNullVector.length, Optional.of(structIsNullVector), blocks);
    return new ColumnChunk(rowBlock, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels());
}
 

private ColumnChunk readArray(GroupField field)
        throws IOException
{
    List<Type> parameters = field.getType().getTypeParameters();
    checkArgument(parameters.size() == 1, "Arrays must have a single type parameter, found %s", parameters.size());
    Field elementField = field.getChildren().get(0).get();
    ColumnChunk columnChunk = readColumnChunk(elementField);
    IntList offsets = new IntArrayList();
    BooleanList valueIsNull = new BooleanArrayList();

    calculateCollectionOffsets(field, offsets, valueIsNull, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels());
    Block arrayBlock = ArrayBlock.fromElementBlock(valueIsNull.size(), Optional.of(valueIsNull.toBooleanArray()), offsets.toIntArray(), columnChunk.getBlock());
    return new ColumnChunk(arrayBlock, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels());
}
 

/**
 * Each struct has three variants of presence:
 * 1) Struct is not defined, because one of it's optional parent fields is null
 * 2) Struct is null
 * 3) Struct is defined and not empty.
 */
public static BooleanList calculateStructOffsets(
        Field field,
        int[] fieldDefinitionLevels,
        int[] fieldRepetitionLevels)
{
    int maxDefinitionLevel = field.getDefinitionLevel();
    int maxRepetitionLevel = field.getRepetitionLevel();
    BooleanList structIsNull = new BooleanArrayList();
    boolean required = field.isRequired();
    if (fieldDefinitionLevels == null) {
        return structIsNull;
    }
    for (int i = 0; i < fieldDefinitionLevels.length; i++) {
        if (fieldRepetitionLevels[i] <= maxRepetitionLevel) {
            if (isValueNull(required, fieldDefinitionLevels[i], maxDefinitionLevel)) {
                // Struct is null
                structIsNull.add(true);
            }
            else if (fieldDefinitionLevels[i] >= maxDefinitionLevel) {
                // Struct is defined and not empty
                structIsNull.add(false);
            }
        }
    }
    return structIsNull;
}
 

/**
 * Each collection (Array or Map) has four variants of presence:
 * 1) Collection is not defined, because one of it's optional parent fields is null
 * 2) Collection is null
 * 3) Collection is defined but empty
 * 4) Collection is defined and not empty. In this case offset value is increased by the number of elements in that collection
 */
public static void calculateCollectionOffsets(Field field, IntList offsets, BooleanList collectionIsNull, int[] definitionLevels, int[] repetitionLevels)
{
    int maxDefinitionLevel = field.getDefinitionLevel();
    int maxElementRepetitionLevel = field.getRepetitionLevel() + 1;
    boolean required = field.isRequired();
    int offset = 0;
    offsets.add(offset);
    for (int i = 0; i < definitionLevels.length; i = getNextCollectionStartIndex(repetitionLevels, maxElementRepetitionLevel, i)) {
        if (ParquetTypeUtils.isValueNull(required, definitionLevels[i], maxDefinitionLevel)) {
            // Collection is null
            collectionIsNull.add(true);
            offsets.add(offset);
        }
        else if (definitionLevels[i] == maxDefinitionLevel) {
            // Collection is defined but empty
            collectionIsNull.add(false);
            offsets.add(offset);
        }
        else if (definitionLevels[i] > maxDefinitionLevel) {
            // Collection is defined and not empty
            collectionIsNull.add(false);
            offset += getCollectionSize(repetitionLevels, maxElementRepetitionLevel, i + 1);
            offsets.add(offset);
        }
    }
}
 

@Test
public void testWriteMultiple()
        throws IOException
{
    BooleanOutputStream outputStream = createValueOutputStream();
    for (int i = 0; i < 3; i++) {
        outputStream.reset();

        BooleanList expectedValues = new BooleanArrayList(1024);
        outputStream.writeBooleans(32, true);
        expectedValues.addAll(Collections.nCopies(32, true));
        outputStream.writeBooleans(32, false);
        expectedValues.addAll(Collections.nCopies(32, false));

        outputStream.writeBooleans(1, true);
        expectedValues.add(true);
        outputStream.writeBooleans(1, false);
        expectedValues.add(false);

        outputStream.writeBooleans(34, true);
        expectedValues.addAll(Collections.nCopies(34, true));
        outputStream.writeBooleans(34, false);
        expectedValues.addAll(Collections.nCopies(34, false));

        outputStream.writeBoolean(true);
        expectedValues.add(true);
        outputStream.writeBoolean(false);
        expectedValues.add(false);

        outputStream.close();

        DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1000);
        StreamDataOutput streamDataOutput = outputStream.getStreamDataOutput(new OrcColumnId(33));
        streamDataOutput.writeData(sliceOutput);
        Stream stream = streamDataOutput.getStream();
        assertEquals(stream.getStreamKind(), StreamKind.DATA);
        assertEquals(stream.getColumnId(), new OrcColumnId(33));
        assertEquals(stream.getLength(), sliceOutput.size());

        BooleanInputStream valueStream = createValueStream(sliceOutput.slice());
        for (int index = 0; index < expectedValues.size(); index++) {
            boolean expectedValue = expectedValues.getBoolean(index);
            boolean actualValue = readValue(valueStream);
            assertEquals(actualValue, expectedValue);
        }
    }
}
 

public static BooleanListBitVector wrap( final BooleanList list ) {
	return new BooleanListBitVector( list );
}
 

protected BooleanListBitVector( final BooleanList list ) { this.list = list; }