io.airlift.stats.CounterStat Java Examples

private SqlTask newSqlTask()
{
    TaskId taskId = new TaskId("query", 0, idGeneator.incrementAndGet());
    URI location = URI.create("fake://task/" + taskId);

    return createSqlTask(
            taskId,
            location,
            "fake",
            new QueryContext(new QueryId("query"),
                    DataSize.of(1, MEGABYTE),
                    DataSize.of(2, MEGABYTE),
                    memoryPool,
                    new TestingGcMonitor(),
                    executor,
                    scheduledExecutor,
                    DataSize.of(1, GIGABYTE),
                    spillSpaceTracker),
            sqlTaskExecutionFactory,
            executor,
            Functions.identity(),
            DataSize.of(32, MEGABYTE),
            new CounterStat());
}
 

public MultilevelSplitQueue(double levelTimeMultiplier)
{
    this.levelMinPriority = new AtomicLong[LEVEL_THRESHOLD_SECONDS.length];
    this.levelWaitingSplits = new ArrayList<>(LEVEL_THRESHOLD_SECONDS.length);
    ImmutableList.Builder<CounterStat> counters = ImmutableList.builder();

    for (int i = 0; i < LEVEL_THRESHOLD_SECONDS.length; i++) {
        levelScheduledTime[i] = new AtomicLong();
        levelMinPriority[i] = new AtomicLong(-1);
        levelWaitingSplits.add(new PriorityQueue<>());
        counters.add(new CounterStat());
    }

    this.selectedLevelCounters = counters.build();

    this.levelTimeMultiplier = levelTimeMultiplier;
}
 

private HiveSplitSource(
        ConnectorSession session,
        String databaseName,
        String tableName,
        PerBucket queues,
        int maxInitialSplits,
        DataSize maxOutstandingSplitsSize,
        HiveSplitLoader splitLoader,
        AtomicReference<State> stateReference,
        CounterStat highMemorySplitSourceCounter)
{
    requireNonNull(session, "session is null");
    this.queryId = session.getQueryId();
    this.databaseName = requireNonNull(databaseName, "databaseName is null");
    this.tableName = requireNonNull(tableName, "tableName is null");
    this.queues = requireNonNull(queues, "queues is null");
    this.maxOutstandingSplitsBytes = requireNonNull(maxOutstandingSplitsSize, "maxOutstandingSplitsSize is null").toBytes();
    this.splitLoader = requireNonNull(splitLoader, "splitLoader is null");
    this.stateReference = requireNonNull(stateReference, "stateReference is null");
    this.highMemorySplitSourceCounter = requireNonNull(highMemorySplitSourceCounter, "highMemorySplitSourceCounter is null");

    this.maxSplitSize = getMaxSplitSize(session);
    this.maxInitialSplitSize = getMaxInitialSplitSize(session);
    this.remainingInitialSplits = new AtomicInteger(maxInitialSplits);
}
 

PrioritizedSplitRunner(
        TaskHandle taskHandle,
        SplitRunner split,
        Ticker ticker,
        CounterStat globalCpuTimeMicros,
        CounterStat globalScheduledTimeMicros,
        TimeStat blockedQuantaWallTime,
        TimeStat unblockedQuantaWallTime)
{
    this.taskHandle = taskHandle;
    this.splitId = taskHandle.getNextSplitId();
    this.split = split;
    this.ticker = ticker;
    this.workerId = NEXT_WORKER_ID.getAndIncrement();
    this.globalCpuTimeMicros = globalCpuTimeMicros;
    this.globalScheduledTimeMicros = globalScheduledTimeMicros;
    this.blockedQuantaWallTime = blockedQuantaWallTime;
    this.unblockedQuantaWallTime = unblockedQuantaWallTime;

    this.updateLevelPriority();
}
 

public TopologyAwareNodeSelector(
        InternalNodeManager nodeManager,
        NodeTaskMap nodeTaskMap,
        boolean includeCoordinator,
        Supplier<NodeMap> nodeMap,
        int minCandidates,
        int maxSplitsPerNode,
        int maxPendingSplitsPerTask,
        List<CounterStat> topologicalSplitCounters,
        NetworkTopology networkTopology)
{
    this.nodeManager = requireNonNull(nodeManager, "nodeManager is null");
    this.nodeTaskMap = requireNonNull(nodeTaskMap, "nodeTaskMap is null");
    this.includeCoordinator = includeCoordinator;
    this.nodeMap = new AtomicReference<>(nodeMap);
    this.minCandidates = minCandidates;
    this.maxSplitsPerNode = maxSplitsPerNode;
    this.maxPendingSplitsPerTask = maxPendingSplitsPerTask;
    this.topologicalSplitCounters = requireNonNull(topologicalSplitCounters, "topologicalSplitCounters is null");
    this.networkTopology = requireNonNull(networkTopology, "networkTopology is null");
}
 

public HiveSplitManager(
        Function<HiveTransactionHandle, SemiTransactionalHiveMetastore> metastoreProvider,
        HivePartitionManager partitionManager,
        NamenodeStats namenodeStats,
        HdfsEnvironment hdfsEnvironment,
        DirectoryLister directoryLister,
        Executor executor,
        CoercionPolicy coercionPolicy,
        CounterStat highMemorySplitSourceCounter,
        int maxOutstandingSplits,
        DataSize maxOutstandingSplitsSize,
        int minPartitionBatchSize,
        int maxPartitionBatchSize,
        int maxInitialSplits,
        int splitLoaderConcurrency,
        @Nullable Integer maxSplitsPerSecond,
        boolean recursiveDfsWalkerEnabled,
        TypeManager typeManager)
{
    this.metastoreProvider = requireNonNull(metastoreProvider, "metastore is null");
    this.partitionManager = requireNonNull(partitionManager, "partitionManager is null");
    this.namenodeStats = requireNonNull(namenodeStats, "namenodeStats is null");
    this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null");
    this.directoryLister = requireNonNull(directoryLister, "directoryLister is null");
    this.executor = new ErrorCodedExecutor(executor);
    this.coercionPolicy = requireNonNull(coercionPolicy, "coercionPolicy is null");
    this.highMemorySplitSourceCounter = requireNonNull(highMemorySplitSourceCounter, "highMemorySplitSourceCounter is null");
    checkArgument(maxOutstandingSplits >= 1, "maxOutstandingSplits must be at least 1");
    this.maxOutstandingSplits = maxOutstandingSplits;
    this.maxOutstandingSplitsSize = maxOutstandingSplitsSize;
    this.minPartitionBatchSize = minPartitionBatchSize;
    this.maxPartitionBatchSize = maxPartitionBatchSize;
    this.maxInitialSplits = maxInitialSplits;
    this.splitLoaderConcurrency = splitLoaderConcurrency;
    this.maxSplitsPerSecond = firstNonNull(maxSplitsPerSecond, Integer.MAX_VALUE);
    this.recursiveDfsWalkerEnabled = recursiveDfsWalkerEnabled;
    this.typeManager = requireNonNull(typeManager, "typeManager is null");
}
 

public CounterStat getProcessedInputDataSize()
{
    CounterStat stat = new CounterStat();
    for (PipelineContext pipelineContext : pipelineContexts) {
        if (pipelineContext.isInputPipeline()) {
            stat.merge(pipelineContext.getProcessedInputDataSize());
        }
    }
    return stat;
}
 

private SqlTask createInitialTask()
{
    TaskId taskId = new TaskId("query", 0, nextTaskId.incrementAndGet());
    URI location = URI.create("fake://task/" + taskId);

    QueryContext queryContext = new QueryContext(new QueryId("query"),
            DataSize.of(1, MEGABYTE),
            DataSize.of(2, MEGABYTE),
            new MemoryPool(new MemoryPoolId("test"), DataSize.of(1, GIGABYTE)),
            new TestingGcMonitor(),
            taskNotificationExecutor,
            driverYieldExecutor,
            DataSize.of(1, MEGABYTE),
            new SpillSpaceTracker(DataSize.of(1, GIGABYTE)));

    queryContext.addTaskContext(new TaskStateMachine(taskId, taskNotificationExecutor), testSessionBuilder().build(), false, false, OptionalInt.empty());

    return createSqlTask(
            taskId,
            location,
            "fake",
            queryContext,
            sqlTaskExecutionFactory,
            taskNotificationExecutor,
            Functions.identity(),
            DataSize.of(32, MEGABYTE),
            new CounterStat());
}
 

public CounterStat getOutputPositions()
{
    CounterStat stat = new CounterStat();
    stat.merge(outputPositions);
    for (DriverContext driver : drivers) {
        stat.merge(driver.getOutputPositions());
    }
    return stat;
}
 

@Test
public void testOutstandingSplitCount()
{
    HiveSplitSource hiveSplitSource = HiveSplitSource.allAtOnce(
            SESSION,
            "database",
            "table",
            10,
            10,
            DataSize.of(1, MEGABYTE),
            Integer.MAX_VALUE,
            new TestingHiveSplitLoader(),
            Executors.newFixedThreadPool(5),
            new CounterStat());

    // add 10 splits
    for (int i = 0; i < 10; i++) {
        hiveSplitSource.addToQueue(new TestSplit(i));
        assertEquals(hiveSplitSource.getBufferedInternalSplitCount(), i + 1);
    }

    // remove 1 split
    assertEquals(getSplits(hiveSplitSource, 1).size(), 1);
    assertEquals(hiveSplitSource.getBufferedInternalSplitCount(), 9);

    // remove 4 splits
    assertEquals(getSplits(hiveSplitSource, 4).size(), 4);
    assertEquals(hiveSplitSource.getBufferedInternalSplitCount(), 5);

    // try to remove 20 splits, and verify we only got 5
    assertEquals(getSplits(hiveSplitSource, 20).size(), 5);
    assertEquals(hiveSplitSource.getBufferedInternalSplitCount(), 0);
}
 

public CounterStat getInputPositions()
{
    CounterStat stat = new CounterStat();
    for (PipelineContext pipelineContext : pipelineContexts) {
        if (pipelineContext.isInputPipeline()) {
            stat.merge(pipelineContext.getInputPositions());
        }
    }
    return stat;
}
 

public CounterStat getOutputPositions()
{
    OperatorContext inputOperator = getLast(operatorContexts, null);
    if (inputOperator != null) {
        return inputOperator.getOutputPositions();
    }
    else {
        return new CounterStat();
    }
}
 

public CounterStat getOutputDataSize()
{
    OperatorContext inputOperator = getLast(operatorContexts, null);
    if (inputOperator != null) {
        return inputOperator.getOutputDataSize();
    }
    else {
        return new CounterStat();
    }
}
 

@Inject
public TopologyAwareNodeSelectorFactory(
        NetworkTopology networkTopology,
        InternalNodeManager nodeManager,
        NodeSchedulerConfig schedulerConfig,
        NodeTaskMap nodeTaskMap,
        TopologyAwareNodeSelectorConfig topologyConfig)
{
    requireNonNull(networkTopology, "networkTopology is null");
    requireNonNull(nodeManager, "nodeManager is null");
    requireNonNull(schedulerConfig, "schedulerConfig is null");
    requireNonNull(nodeTaskMap, "nodeTaskMap is null");
    requireNonNull(topologyConfig, "topologyConfig is null");

    this.networkTopology = networkTopology;
    this.nodeManager = nodeManager;
    this.minCandidates = schedulerConfig.getMinCandidates();
    this.includeCoordinator = schedulerConfig.isIncludeCoordinator();
    this.maxSplitsPerNode = schedulerConfig.getMaxSplitsPerNode();
    this.maxPendingSplitsPerTask = schedulerConfig.getMaxPendingSplitsPerTask();
    this.nodeTaskMap = requireNonNull(nodeTaskMap, "nodeTaskMap is null");
    checkArgument(maxSplitsPerNode >= maxPendingSplitsPerTask, "maxSplitsPerNode must be > maxPendingSplitsPerTask");

    Builder<CounterStat> placementCounters = ImmutableList.builder();
    ImmutableMap.Builder<String, CounterStat> placementCountersByName = ImmutableMap.builder();

    // always add a counter for "all" locations, which is the default global segment
    CounterStat allCounter = new CounterStat();
    placementCounters.add(allCounter);
    placementCountersByName.put("all", allCounter);

    for (String segmentName : ImmutableList.copyOf(topologyConfig.getLocationSegmentNames())) {
        CounterStat segmentCounter = new CounterStat();
        placementCounters.add(segmentCounter);
        placementCountersByName.put(segmentName, segmentCounter);
    }

    this.placementCounters = placementCounters.build();
    this.placementCountersByName = placementCountersByName.build();
}
 

public static SqlTask createSqlTask(
        TaskId taskId,
        URI location,
        String nodeId,
        QueryContext queryContext,
        SqlTaskExecutionFactory sqlTaskExecutionFactory,
        ExecutorService taskNotificationExecutor,
        Function<SqlTask, ?> onDone,
        DataSize maxBufferSize,
        CounterStat failedTasks)
{
    SqlTask sqlTask = new SqlTask(taskId, location, nodeId, queryContext, sqlTaskExecutionFactory, taskNotificationExecutor, maxBufferSize);
    sqlTask.initialize(onDone, failedTasks);
    return sqlTask;
}
 

private static HiveSplitSource hiveSplitSource(HiveSplitLoader hiveSplitLoader)
{
    return HiveSplitSource.allAtOnce(
            SESSION,
            SIMPLE_TABLE.getDatabaseName(),
            SIMPLE_TABLE.getTableName(),
            1,
            1,
            DataSize.of(32, MEGABYTE),
            Integer.MAX_VALUE,
            hiveSplitLoader,
            EXECUTOR,
            new CounterStat());
}
 

public CounterStat getOutputDataSize()
{
    CounterStat stat = new CounterStat();
    for (PipelineContext pipelineContext : pipelineContexts) {
        if (pipelineContext.isOutputPipeline()) {
            stat.merge(pipelineContext.getOutputDataSize());
        }
    }
    return stat;
}
 

@Managed
@Nested
public CounterStat getFailures()
{
    return failures;
}
 

@Managed
@Nested
public CounterStat getSelectedCountLevel0()
{
    return selectedLevelCounters.get(0);
}
 

@Managed
@Nested
public CounterStat getBackupSuccess()
{
    return backupSuccess;
}
 

@Managed
@Nested
public CounterStat getDbLoadFailures()
{
    return dbLoadFailures;
}
 

@Managed
@Nested
public CounterStat getAwsThrottleExceptions()
{
    return awsThrottleExceptions;
}
 

@Managed
@Nested
public CounterStat getRetries()
{
    return retries;
}
 

@Managed
@Nested
public CounterStat getThriftExceptions()
{
    return thriftExceptions;
}
 

@Managed
@Nested
public CounterStat getInsufficientResourcesFailures()
{
    return insufficientResourcesFailures;
}
 

@Managed
@Nested
public CounterStat getExternalFailures()
{
    return externalFailures;
}
 

@Managed
@Nested
public CounterStat getCanceledQueries()
{
    return canceledQueries;
}
 

@Managed
@Nested
public CounterStat getOutputShards()
{
    return outputShards;
}
 

@Managed
@Nested
public CounterStat getUserErrorFailures()
{
    return userErrorFailures;
}
 

@Managed
@Nested
public CounterStat getStartedQueries()
{
    return startedQueries;
}