Java Code Examples for io.airlift.units.DataSize#ofBytes()

@BeforeMethod
public void setUp()
{
    memoryPool = new MemoryPool(GENERAL_POOL, DataSize.ofBytes(10));

    TaskExecutor taskExecutor = new TaskExecutor(8, 16, 3, 4, Ticker.systemTicker());
    taskExecutor.start();

    // Must be single threaded
    executor = newScheduledThreadPool(1, threadsNamed("task-notification-%s"));
    scheduledExecutor = newScheduledThreadPool(2, threadsNamed("task-notification-%s"));

    LocalExecutionPlanner planner = createTestingPlanner();

    sqlTaskExecutionFactory = new SqlTaskExecutionFactory(
            executor,
            taskExecutor,
            planner,
            createTestSplitMonitor(),
            new TaskManagerConfig());

    allOperatorContexts = null;
}
 

private static DataSize writeRcFileColumnNew(File outputFile, Format format, Compression compression, Type type, Iterator<?> values, Map<String, String> metadata)
        throws Exception
{
    OutputStreamSliceOutput output = new OutputStreamSliceOutput(new FileOutputStream(outputFile));
    AircompressorCodecFactory codecFactory = new AircompressorCodecFactory(new HadoopCodecFactory(RcFileTester.class.getClassLoader()));
    RcFileWriter writer = new RcFileWriter(
            output,
            ImmutableList.of(type),
            format.getVectorEncoding(),
            compression.getCodecName(),
            codecFactory,
            metadata,
            DataSize.of(100, KILOBYTE),   // use a smaller size to create more row groups
            DataSize.of(200, KILOBYTE),
            true);
    BlockBuilder blockBuilder = type.createBlockBuilder(null, 1024);
    while (values.hasNext()) {
        Object value = values.next();
        writeValue(type, blockBuilder, value);
    }

    writer.write(new Page(blockBuilder.build()));
    writer.close();

    writer.validate(new FileRcFileDataSource(outputFile));

    return DataSize.ofBytes(output.size());
}
 

public SourceOperator newScanFilterAndProjectOperator(DriverContext driverContext)
{
    ConnectorPageSource pageSource = newPageSource();
    ImmutableList.Builder<RowExpression> projectionsBuilder = ImmutableList.builder();
    for (int i = 0; i < types.size(); i++) {
        projectionsBuilder.add(field(i, types.get(i)));
    }
    Supplier<CursorProcessor> cursorProcessor = EXPRESSION_COMPILER.compileCursorProcessor(Optional.empty(), projectionsBuilder.build(), "key");
    Supplier<PageProcessor> pageProcessor = EXPRESSION_COMPILER.compilePageProcessor(Optional.empty(), projectionsBuilder.build());
    SourceOperatorFactory sourceOperatorFactory = new ScanFilterAndProjectOperatorFactory(
            0,
            new PlanNodeId("test"),
            new PlanNodeId("0"),
            (session, split, table, columnHandles, dynamicFilter) -> pageSource,
            cursorProcessor,
            pageProcessor,
            TEST_TABLE_HANDLE,
            columns.stream().map(columnHandle -> (ColumnHandle) columnHandle).collect(toList()),
            TupleDomain::all,
            types,
            DataSize.ofBytes(0),
            0);
    SourceOperator operator = sourceOperatorFactory.createOperator(driverContext);
    operator.addSplit(new Split(new CatalogName("test"), TestingSplit.createLocalSplit(), Lifespan.taskWide()));
    operator.noMoreSplits();
    return operator;
}
 

@Override
public void fail(Throwable cause)
{
    memoryUsage = DataSize.ofBytes(0);
    state = FAILED;
    failureCause = cause;
    fireStateChange();
}
 

@Test
public void testPageSourceLazyLoad()
{
    Block inputBlock = BlockAssertions.createLongSequenceBlock(0, 100);
    // If column 1 is loaded, test will fail
    Page input = new Page(100, inputBlock, new LazyBlock(100, () -> {
        throw new AssertionError("Lazy block should not be loaded");
    }));
    DriverContext driverContext = newDriverContext();

    List<RowExpression> projections = ImmutableList.of(field(0, VARCHAR));
    Supplier<CursorProcessor> cursorProcessor = expressionCompiler.compileCursorProcessor(Optional.empty(), projections, "key");
    PageProcessor pageProcessor = new PageProcessor(Optional.of(new SelectAllFilter()), ImmutableList.of(new LazyPagePageProjection()));

    ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory factory = new ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory(
            0,
            new PlanNodeId("test"),
            new PlanNodeId("0"),
            (session, split, table, columns, dynamicFilter) -> new SinglePagePageSource(input),
            cursorProcessor,
            () -> pageProcessor,
            TEST_TABLE_HANDLE,
            ImmutableList.of(),
            TupleDomain::all,
            ImmutableList.of(BIGINT),
            DataSize.ofBytes(0),
            0);

    SourceOperator operator = factory.createOperator(driverContext);
    operator.addSplit(new Split(new CatalogName("test"), TestingSplit.createLocalSplit(), Lifespan.taskWide()));
    operator.noMoreSplits();

    MaterializedResult expected = toMaterializedResult(driverContext.getSession(), ImmutableList.of(BIGINT), ImmutableList.of(new Page(inputBlock)));
    MaterializedResult actual = toMaterializedResult(driverContext.getSession(), ImmutableList.of(BIGINT), toPages(operator));

    assertEquals(actual.getRowCount(), expected.getRowCount());
    assertEquals(actual, expected);
}
 

@Override
public TaskInfo getTaskInfo()
{
    TaskState state = taskStateMachine.getState();
    List<ExecutionFailureInfo> failures = ImmutableList.of();
    if (state == TaskState.FAILED) {
        failures = toFailures(taskStateMachine.getFailureCauses());
    }

    return new TaskInfo(
            new TaskStatus(
                    taskStateMachine.getTaskId(),
                    TASK_INSTANCE_ID,
                    nextTaskInfoVersion.getAndIncrement(),
                    state,
                    location,
                    nodeId,
                    ImmutableSet.of(),
                    failures,
                    0,
                    0,
                    false,
                    DataSize.ofBytes(0),
                    DataSize.ofBytes(0),
                    DataSize.ofBytes(0),
                    DataSize.ofBytes(0),
                    0,
                    new Duration(0, MILLISECONDS),
                    ImmutableMap.of()),
            DateTime.now(),
            outputBuffer.getInfo(),
            ImmutableSet.of(),
            taskContext.getTaskStats(),
            true);
}
 

public ExchangeClientFactory(
        NodeInfo nodeInfo,
        DataIntegrityVerification dataIntegrityVerification,
        DataSize maxBufferedBytes,
        DataSize maxResponseSize,
        int concurrentRequestMultiplier,
        Duration maxErrorDuration,
        boolean acknowledgePages,
        int pageBufferClientMaxCallbackThreads,
        HttpClient httpClient,
        ScheduledExecutorService scheduler)
{
    this.nodeInfo = requireNonNull(nodeInfo, "nodeInfo is null");
    this.dataIntegrityVerification = requireNonNull(dataIntegrityVerification, "dataIntegrityVerification is null");
    this.maxBufferedBytes = requireNonNull(maxBufferedBytes, "maxBufferedBytes is null");
    this.concurrentRequestMultiplier = concurrentRequestMultiplier;
    this.maxErrorDuration = requireNonNull(maxErrorDuration, "maxErrorDuration is null");
    this.acknowledgePages = acknowledgePages;
    this.httpClient = requireNonNull(httpClient, "httpClient is null");

    // Use only 0.75 of the maxResponseSize to leave room for additional bytes from the encoding
    // TODO figure out a better way to compute the size of data that will be transferred over the network
    requireNonNull(maxResponseSize, "maxResponseSize is null");
    long maxResponseSizeBytes = (long) (Math.min(httpClient.getMaxContentLength(), maxResponseSize.toBytes()) * 0.75);
    this.maxResponseSize = DataSize.ofBytes(maxResponseSizeBytes);

    this.scheduler = requireNonNull(scheduler, "scheduler is null");

    this.pageBufferClientCallbackExecutor = newFixedThreadPool(pageBufferClientMaxCallbackThreads, daemonThreadsNamed("page-buffer-client-callback-%s"));
    this.executorMBean = new ThreadPoolExecutorMBean((ThreadPoolExecutor) pageBufferClientCallbackExecutor);

    checkArgument(maxBufferedBytes.toBytes() > 0, "maxBufferSize must be at least 1 byte: %s", maxBufferedBytes);
    checkArgument(maxResponseSize.toBytes() > 0, "maxResponseSize must be at least 1 byte: %s", maxResponseSize);
    checkArgument(concurrentRequestMultiplier > 0, "concurrentRequestMultiplier must be at least 1: %s", concurrentRequestMultiplier);
}
 

@Test(dataProvider = "executionStrategy")
public void writeUnblockWhenAllReadersFinishAndPagesConsumed(PipelineExecutionStrategy executionStrategy)
{
    LocalExchangeFactory localExchangeFactory = new LocalExchangeFactory(
            FIXED_BROADCAST_DISTRIBUTION,
            2,
            TYPES,
            ImmutableList.of(),
            Optional.empty(),
            executionStrategy,
            DataSize.ofBytes(1));
    LocalExchangeSinkFactoryId localExchangeSinkFactoryId = localExchangeFactory.newSinkFactoryId();
    localExchangeFactory.noMoreSinkFactories();

    run(localExchangeFactory, executionStrategy, exchange -> {
        assertEquals(exchange.getBufferCount(), 2);
        assertExchangeTotalBufferedBytes(exchange, 0);

        LocalExchangeSinkFactory sinkFactory = exchange.getSinkFactory(localExchangeSinkFactoryId);
        LocalExchangeSink sinkA = sinkFactory.createSink();
        assertSinkCanWrite(sinkA);
        LocalExchangeSink sinkB = sinkFactory.createSink();
        assertSinkCanWrite(sinkB);
        sinkFactory.close();
        sinkFactory.noMoreSinkFactories();

        LocalExchangeSource sourceA = exchange.getSource(0);
        assertSource(sourceA, 0);

        LocalExchangeSource sourceB = exchange.getSource(1);
        assertSource(sourceB, 0);

        sinkA.addPage(createPage(0));
        ListenableFuture<?> sinkAFuture = assertSinkWriteBlocked(sinkA);
        ListenableFuture<?> sinkBFuture = assertSinkWriteBlocked(sinkB);

        assertSource(sourceA, 1);
        assertSource(sourceB, 1);
        assertExchangeTotalBufferedBytes(exchange, 1);

        sourceA.finish();
        assertSource(sourceA, 1);
        assertRemovePage(sourceA, createPage(0));
        assertSourceFinished(sourceA);
        assertExchangeTotalBufferedBytes(exchange, 1);

        assertSource(sourceB, 1);
        assertSinkWriteBlocked(sinkA);
        assertSinkWriteBlocked(sinkB);

        sourceB.finish();
        assertSource(sourceB, 1);
        assertRemovePage(sourceB, createPage(0));
        assertSourceFinished(sourceB);
        assertExchangeTotalBufferedBytes(exchange, 0);

        assertTrue(sinkAFuture.isDone());
        assertTrue(sinkBFuture.isDone());

        assertSinkFinished(sinkA);
        assertSinkFinished(sinkB);
    });
}
 

@Test
public void testBufferLimit()
{
    DataSize maxResponseSize = DataSize.ofBytes(1);
    MockExchangeRequestProcessor processor = new MockExchangeRequestProcessor(maxResponseSize);

    URI location = URI.create("http://localhost:8080");

    // add a pages
    processor.addPage(location, createPage(1));
    processor.addPage(location, createPage(2));
    processor.addPage(location, createPage(3));
    processor.setComplete(location);

    @SuppressWarnings("resource")
    ExchangeClient exchangeClient = new ExchangeClient(
            "localhost",
            DataIntegrityVerification.ABORT,
            DataSize.ofBytes(1),
            maxResponseSize,
            1,
            new Duration(1, TimeUnit.MINUTES),
            true,
            new TestingHttpClient(processor, newCachedThreadPool(daemonThreadsNamed("test-%s"))),
            scheduler,
            new SimpleLocalMemoryContext(newSimpleAggregatedMemoryContext(), "test"),
            pageBufferClientCallbackExecutor);

    exchangeClient.addLocation(location);
    exchangeClient.noMoreLocations();
    assertEquals(exchangeClient.isClosed(), false);

    long start = System.nanoTime();

    // start fetching pages
    exchangeClient.scheduleRequestIfNecessary();
    // wait for a page to be fetched
    do {
        // there is no thread coordination here, so sleep is the best we can do
        assertLessThan(Duration.nanosSince(start), new Duration(5, TimeUnit.SECONDS));
        sleepUninterruptibly(100, MILLISECONDS);
    }
    while (exchangeClient.getStatus().getBufferedPages() == 0);

    // client should have sent a single request for a single page
    assertEquals(exchangeClient.getStatus().getBufferedPages(), 1);
    assertTrue(exchangeClient.getStatus().getBufferedBytes() > 0);
    assertStatus(exchangeClient.getStatus().getPageBufferClientStatuses().get(0), location, "queued", 1, 1, 1, "not scheduled");

    // remove the page and wait for the client to fetch another page
    assertPageEquals(exchangeClient.pollPage(), createPage(1));
    do {
        assertLessThan(Duration.nanosSince(start), new Duration(5, TimeUnit.SECONDS));
        sleepUninterruptibly(100, MILLISECONDS);
    }
    while (exchangeClient.getStatus().getBufferedPages() == 0);

    // client should have sent a single request for a single page
    assertStatus(exchangeClient.getStatus().getPageBufferClientStatuses().get(0), location, "queued", 2, 2, 2, "not scheduled");
    assertEquals(exchangeClient.getStatus().getBufferedPages(), 1);
    assertTrue(exchangeClient.getStatus().getBufferedBytes() > 0);

    // remove the page and wait for the client to fetch another page
    assertPageEquals(exchangeClient.pollPage(), createPage(2));
    do {
        assertLessThan(Duration.nanosSince(start), new Duration(5, TimeUnit.SECONDS));
        sleepUninterruptibly(100, MILLISECONDS);
    }
    while (exchangeClient.getStatus().getBufferedPages() == 0);

    // client should have sent a single request for a single page
    assertStatus(exchangeClient.getStatus().getPageBufferClientStatuses().get(0), location, "queued", 3, 3, 3, "not scheduled");
    assertEquals(exchangeClient.getStatus().getBufferedPages(), 1);
    assertTrue(exchangeClient.getStatus().getBufferedBytes() > 0);

    // remove last page
    assertPageEquals(getNextPage(exchangeClient), createPage(3));

    //  wait for client to decide there are no more pages
    assertNull(getNextPage(exchangeClient));
    assertEquals(exchangeClient.getStatus().getBufferedPages(), 0);
    assertTrue(exchangeClient.getStatus().getBufferedBytes() == 0);
    assertEquals(exchangeClient.isClosed(), true);
    assertStatus(exchangeClient.getStatus().getPageBufferClientStatuses().get(0), location, "closed", 3, 5, 5, "not scheduled");
}
 

@Override
public DataSize getUserMemoryReservation()
{
    return DataSize.ofBytes(0);
}
 

default DataSize getPhysicalInputDataSize()
{
    return DataSize.ofBytes(0);
}
 

@Override
public DataSize getInputDataSize()
{
    return DataSize.ofBytes(processedBytes);
}
 

private TaskStatus createTaskStatus(TaskHolder taskHolder)
{
    // Always return a new TaskInfo with a larger version number;
    // otherwise a client will not accept the update
    long versionNumber = nextTaskInfoVersion.getAndIncrement();

    TaskState state = taskStateMachine.getState();
    List<ExecutionFailureInfo> failures = ImmutableList.of();
    if (state == FAILED) {
        failures = toFailures(taskStateMachine.getFailureCauses());
    }

    int queuedPartitionedDrivers = 0;
    int runningPartitionedDrivers = 0;
    DataSize physicalWrittenDataSize = DataSize.ofBytes(0);
    DataSize userMemoryReservation = DataSize.ofBytes(0);
    DataSize systemMemoryReservation = DataSize.ofBytes(0);
    DataSize revocableMemoryReservation = DataSize.ofBytes(0);
    // TODO: add a mechanism to avoid sending the whole completedDriverGroups set over the wire for every task status reply
    Set<Lifespan> completedDriverGroups = ImmutableSet.of();
    long fullGcCount = 0;
    Duration fullGcTime = new Duration(0, MILLISECONDS);
    Map<DynamicFilterId, Domain> dynamicTupleDomains = ImmutableMap.of();
    if (taskHolder.getFinalTaskInfo() != null) {
        TaskInfo taskInfo = taskHolder.getFinalTaskInfo();
        TaskStats taskStats = taskInfo.getStats();
        queuedPartitionedDrivers = taskStats.getQueuedPartitionedDrivers();
        runningPartitionedDrivers = taskStats.getRunningPartitionedDrivers();
        physicalWrittenDataSize = taskStats.getPhysicalWrittenDataSize();
        userMemoryReservation = taskStats.getUserMemoryReservation();
        systemMemoryReservation = taskStats.getSystemMemoryReservation();
        revocableMemoryReservation = taskStats.getRevocableMemoryReservation();
        fullGcCount = taskStats.getFullGcCount();
        fullGcTime = taskStats.getFullGcTime();
        dynamicTupleDomains = taskInfo.getTaskStatus().getDynamicFilterDomains();
    }
    else if (taskHolder.getTaskExecution() != null) {
        long physicalWrittenBytes = 0;
        TaskContext taskContext = taskHolder.getTaskExecution().getTaskContext();
        for (PipelineContext pipelineContext : taskContext.getPipelineContexts()) {
            PipelineStatus pipelineStatus = pipelineContext.getPipelineStatus();
            queuedPartitionedDrivers += pipelineStatus.getQueuedPartitionedDrivers();
            runningPartitionedDrivers += pipelineStatus.getRunningPartitionedDrivers();
            physicalWrittenBytes += pipelineContext.getPhysicalWrittenDataSize();
        }
        physicalWrittenDataSize = succinctBytes(physicalWrittenBytes);
        userMemoryReservation = taskContext.getMemoryReservation();
        systemMemoryReservation = taskContext.getSystemMemoryReservation();
        revocableMemoryReservation = taskContext.getRevocableMemoryReservation();
        completedDriverGroups = taskContext.getCompletedDriverGroups();
        fullGcCount = taskContext.getFullGcCount();
        fullGcTime = taskContext.getFullGcTime();
        dynamicTupleDomains = taskContext.getDynamicTupleDomains();
    }
    // Compact TupleDomain before reporting dynamic filters to coordinator to avoid bloating QueryInfo
    Map<DynamicFilterId, Domain> compactDynamicTupleDomains = dynamicTupleDomains.entrySet().stream()
            .collect(toImmutableMap(Map.Entry::getKey, entry -> entry.getValue().simplify()));

    return new TaskStatus(taskStateMachine.getTaskId(),
            taskInstanceId,
            versionNumber,
            state,
            location,
            nodeId,
            completedDriverGroups,
            failures,
            queuedPartitionedDrivers,
            runningPartitionedDrivers,
            isOutputBufferOverutilized(),
            physicalWrittenDataSize,
            userMemoryReservation,
            systemMemoryReservation,
            revocableMemoryReservation,
            fullGcCount,
            fullGcTime,
            compactDynamicTupleDomains);
}
 

static DataSize sizeOfPages(int count)
{
    return DataSize.ofBytes(BUFFERED_PAGE_SIZE.toBytes() * count);
}
 

@Test(dataProvider = "executionStrategy")
public void testGatherSingleWriter(PipelineExecutionStrategy executionStrategy)
{
    LocalExchangeFactory localExchangeFactory = new LocalExchangeFactory(
            SINGLE_DISTRIBUTION,
            8,
            TYPES,
            ImmutableList.of(),
            Optional.empty(),
            executionStrategy,
            DataSize.ofBytes(retainedSizeOfPages(99)));
    LocalExchangeSinkFactoryId localExchangeSinkFactoryId = localExchangeFactory.newSinkFactoryId();
    localExchangeFactory.noMoreSinkFactories();

    run(localExchangeFactory, executionStrategy, exchange -> {
        assertEquals(exchange.getBufferCount(), 1);
        assertExchangeTotalBufferedBytes(exchange, 0);

        LocalExchangeSinkFactory sinkFactory = exchange.getSinkFactory(localExchangeSinkFactoryId);

        LocalExchangeSource source = exchange.getSource(0);
        assertSource(source, 0);

        LocalExchangeSink sink = sinkFactory.createSink();
        sinkFactory.close();
        sinkFactory.noMoreSinkFactories();

        assertSinkCanWrite(sink);
        assertSource(source, 0);

        // add the first page which should cause the reader to unblock
        ListenableFuture<?> readFuture = source.waitForReading();
        assertFalse(readFuture.isDone());
        sink.addPage(createPage(0));
        assertTrue(readFuture.isDone());
        assertExchangeTotalBufferedBytes(exchange, 1);

        assertSource(source, 1);

        sink.addPage(createPage(1));
        assertSource(source, 2);
        assertExchangeTotalBufferedBytes(exchange, 2);

        assertRemovePage(source, createPage(0));
        assertSource(source, 1);
        assertExchangeTotalBufferedBytes(exchange, 1);

        assertRemovePage(source, createPage(1));
        assertSource(source, 0);
        assertExchangeTotalBufferedBytes(exchange, 0);

        sink.addPage(createPage(2));
        sink.addPage(createPage(3));
        assertSource(source, 2);
        assertExchangeTotalBufferedBytes(exchange, 2);

        sink.finish();
        assertSinkFinished(sink);

        assertSource(source, 2);

        assertRemovePage(source, createPage(2));
        assertSource(source, 1);
        assertSinkFinished(sink);
        assertExchangeTotalBufferedBytes(exchange, 1);

        assertRemovePage(source, createPage(3));
        assertSourceFinished(source);
        assertExchangeTotalBufferedBytes(exchange, 0);
    });
}
 

@Test(dataProvider = "executionStrategy")
public void testPassthrough(PipelineExecutionStrategy executionStrategy)
{
    LocalExchangeFactory localExchangeFactory = new LocalExchangeFactory(
            FIXED_PASSTHROUGH_DISTRIBUTION,
            2,
            TYPES,
            ImmutableList.of(),
            Optional.empty(),
            executionStrategy,
            DataSize.ofBytes(retainedSizeOfPages(1)));

    LocalExchangeSinkFactoryId localExchangeSinkFactoryId = localExchangeFactory.newSinkFactoryId();
    localExchangeFactory.noMoreSinkFactories();

    run(localExchangeFactory, executionStrategy, exchange -> {
        assertEquals(exchange.getBufferCount(), 2);
        assertExchangeTotalBufferedBytes(exchange, 0);

        LocalExchangeSinkFactory sinkFactory = exchange.getSinkFactory(localExchangeSinkFactoryId);
        LocalExchangeSink sinkA = sinkFactory.createSink();
        LocalExchangeSink sinkB = sinkFactory.createSink();
        assertSinkCanWrite(sinkA);
        assertSinkCanWrite(sinkB);
        sinkFactory.close();
        sinkFactory.noMoreSinkFactories();

        LocalExchangeSource sourceA = exchange.getSource(0);
        assertSource(sourceA, 0);

        LocalExchangeSource sourceB = exchange.getSource(1);
        assertSource(sourceB, 0);

        sinkA.addPage(createPage(0));
        assertSource(sourceA, 1);
        assertSource(sourceB, 0);
        assertSinkWriteBlocked(sinkA);

        assertSinkCanWrite(sinkB);
        sinkB.addPage(createPage(1));
        assertSource(sourceA, 1);
        assertSource(sourceB, 1);
        assertSinkWriteBlocked(sinkA);

        assertExchangeTotalBufferedBytes(exchange, 2);

        assertRemovePage(sourceA, createPage(0));
        assertSource(sourceA, 0);
        assertSinkCanWrite(sinkA);
        assertSinkWriteBlocked(sinkB);
        assertExchangeTotalBufferedBytes(exchange, 1);

        sinkA.finish();
        assertSinkFinished(sinkA);
        assertSource(sourceB, 1);

        sourceA.finish();
        sourceB.finish();
        assertRemovePage(sourceB, createPage(1));
        assertSourceFinished(sourceA);
        assertSourceFinished(sourceB);

        assertSinkFinished(sinkB);
        assertExchangeTotalBufferedBytes(exchange, 0);
    });
}
 

public DataSize getEstimatedSize()
{
    return DataSize.ofBytes(estimatedSize);
}
 

@Override
public QueryInfo getFullQueryInfo()
{
    return new QueryInfo(
            new QueryId("test"),
            session.toSessionRepresentation(),
            state,
            new MemoryPoolId("test"),
            !state.isDone(),
            URI.create("http://test"),
            ImmutableList.of(),
            "SELECT 1",
            Optional.empty(),
            new QueryStats(
                    new DateTime(1),
                    new DateTime(2),
                    new DateTime(3),
                    new DateTime(4),
                    new Duration(6, NANOSECONDS),
                    new Duration(5, NANOSECONDS),
                    new Duration(31, NANOSECONDS),
                    new Duration(41, NANOSECONDS),
                    new Duration(7, NANOSECONDS),
                    new Duration(8, NANOSECONDS),

                    new Duration(100, NANOSECONDS),
                    new Duration(200, NANOSECONDS),

                    9,
                    10,
                    11,

                    12,
                    13,
                    15,
                    30,
                    16,

                    17.0,
                    DataSize.ofBytes(18),
                    DataSize.ofBytes(19),
                    DataSize.ofBytes(20),
                    DataSize.ofBytes(21),
                    DataSize.ofBytes(22),
                    DataSize.ofBytes(30),
                    DataSize.ofBytes(23),
                    DataSize.ofBytes(24),
                    DataSize.ofBytes(25),
                    DataSize.ofBytes(26),

                    true,
                    new Duration(20, NANOSECONDS),
                    new Duration(21, NANOSECONDS),
                    new Duration(23, NANOSECONDS),
                    false,
                    ImmutableSet.of(),

                    DataSize.ofBytes(241),
                    251,
                    new Duration(24, NANOSECONDS),

                    DataSize.ofBytes(242),
                    252,

                    DataSize.ofBytes(25),
                    26,

                    DataSize.ofBytes(27),
                    28,

                    DataSize.ofBytes(29),
                    30,

                    DataSize.ofBytes(31),

                    ImmutableList.of(),
                    ImmutableList.of()),
            Optional.empty(),
            Optional.empty(),
            Optional.empty(),
            ImmutableMap.of(),
            ImmutableSet.of(),
            ImmutableMap.of(),
            ImmutableMap.of(),
            ImmutableSet.of(),
            Optional.empty(),
            false,
            "",
            Optional.empty(),
            null,
            null,
            ImmutableList.of(),
            ImmutableSet.of(),
            Optional.empty(),
            ImmutableList.of(),
            ImmutableList.of(),
            state.isDone(),
            Optional.empty());
}
 

@Override
public DataSize getUserMemoryReservation()
{
    return DataSize.ofBytes(0);
}
 

@Test
public void testMergeWithMemorySpill()
{
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(BIGINT);

    int smallPagesSpillThresholdSize = 150000;

    List<Page> input = rowPagesBuilder
            .addSequencePage(smallPagesSpillThresholdSize, 0)
            .addSequencePage(10, smallPagesSpillThresholdSize)
            .build();

    HashAggregationOperatorFactory operatorFactory = new HashAggregationOperatorFactory(
            0,
            new PlanNodeId("test"),
            ImmutableList.of(BIGINT),
            ImmutableList.of(0),
            ImmutableList.of(),
            Step.SINGLE,
            false,
            ImmutableList.of(LONG_MIN.bind(ImmutableList.of(0), Optional.empty())),
            rowPagesBuilder.getHashChannel(),
            Optional.empty(),
            1,
            Optional.of(DataSize.of(16, MEGABYTE)),
            true,
            DataSize.ofBytes(smallPagesSpillThresholdSize),
            succinctBytes(Integer.MAX_VALUE),
            spillerFactory,
            joinCompiler,
            false);

    DriverContext driverContext = createDriverContext(smallPagesSpillThresholdSize);

    MaterializedResult.Builder resultBuilder = resultBuilder(driverContext.getSession(), BIGINT, BIGINT);
    for (int i = 0; i < smallPagesSpillThresholdSize + 10; ++i) {
        resultBuilder.row((long) i, (long) i);
    }

    assertOperatorEqualsIgnoreOrder(operatorFactory, driverContext, input, resultBuilder.build());
}