org.apache.kylin.query.enumerator.OLAPQuery Java Examples

The following examples show how to use org.apache.kylin.query.enumerator.OLAPQuery. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BasicService.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
public DataSource getOLAPDataSource(String project) {

        project = ProjectInstance.getNormalizedProjectName(project);

        DataSource ret = olapDataSources.get(project);
        if (ret == null) {
            logger.debug("Creating a new data source");
            logger.debug("OLAP data source pointing to " + getConfig());

            File modelJson = OLAPSchemaFactory.createTempOLAPJson(project, getConfig());

            try {
                List<String> text = Files.readLines(modelJson, Charset.defaultCharset());
                logger.debug("The new temp olap json is :");
                for (String line : text)
                    logger.debug(line);
            } catch (IOException e) {
                e.printStackTrace(); // logging failure is not critical
            }

            DriverManagerDataSource ds = new DriverManagerDataSource();
            Properties props = new Properties();
            props.setProperty(OLAPQuery.PROP_SCAN_THRESHOLD, String.valueOf(KylinConfig.getInstanceFromEnv().getScanThreshold()));
            ds.setConnectionProperties(props);
            ds.setDriverClassName("net.hydromatic.optiq.jdbc.Driver");
            ds.setUrl("jdbc:calcite:model=" + modelJson.getAbsolutePath());

            ret = olapDataSources.putIfAbsent(project, ds);
            if (ret == null) {
                ret = ds;
            }
        }
        return ret;
    }
Example #2
Source File: KylinQueryTest.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private static void setUpCubeConn() throws SQLException {
    // Cube Connection
    File olapTmp = OLAPSchemaFactory.createTempOLAPJson(ProjectInstance.DEFAULT_PROJECT_NAME, config);
    Properties props = new Properties();
    props.setProperty(OLAPQuery.PROP_SCAN_THRESHOLD, "10000");
    cubeConnection = DriverManager.getConnection("jdbc:calcite:model=" + olapTmp.getAbsolutePath(), props);
}
Example #3
Source File: OLAPTable.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public <T> Queryable<T> asQueryable(QueryProvider queryProvider, SchemaPlus schema, String tableName) {
    return new AbstractTableQueryable<T>(queryProvider, schema, this, tableName) {
        @SuppressWarnings("unchecked")
        public Enumerator<T> enumerator() {
            final OLAPQuery query = new OLAPQuery(EnumeratorTypeEnum.OLAP, 0);
            return (Enumerator<T>) query.enumerator();
        }
    };
}
Example #4
Source File: OLAPTable.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public <T> Queryable<T> asQueryable(QueryProvider queryProvider, SchemaPlus schema, String tableName) {
    return new AbstractTableQueryable<T>(queryProvider, schema, this, tableName) {
        @SuppressWarnings("unchecked")
        public Enumerator<T> enumerator() {
            final OLAPQuery query = new OLAPQuery(EnumeratorTypeEnum.INDEX, 0);
            return (Enumerator<T>) query.enumerator();
        }
    };
}
Example #5
Source File: OLAPTable.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Override
public <T> Queryable<T> asQueryable(QueryProvider queryProvider, SchemaPlus schema, String tableName) {
    return new AbstractTableQueryable<T>(queryProvider, schema, this, tableName) {
        @SuppressWarnings("unchecked")
        public Enumerator<T> enumerator() {
            final OLAPQuery query = new OLAPQuery(EnumeratorTypeEnum.OLAP, 0);
            return (Enumerator<T>) query.enumerator();
        }
    };
}
Example #6
Source File: OLAPTable.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public Enumerable<Object[]> executeColumnDictionaryQuery(DataContext optiqContext, int ctxSeq) {
    return new OLAPQuery(optiqContext, EnumeratorTypeEnum.COL_DICT, ctxSeq);
}
Example #7
Source File: OLAPTable.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
public Enumerable<Object[]> executeHiveQuery(DataContext optiqContext, int ctxSeq) {
    return new OLAPQuery(optiqContext, EnumeratorTypeEnum.HIVE, ctxSeq);
}
Example #8
Source File: OLAPTable.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
public Enumerable<Object[]> executeLookupTableQuery(DataContext optiqContext, int ctxSeq) {
    return new OLAPQuery(optiqContext, EnumeratorTypeEnum.LOOKUP_TABLE, ctxSeq);
}
Example #9
Source File: OLAPTable.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
public Enumerable<Object[]> executeIndexQuery(DataContext optiqContext, int ctxSeq) {
    return new OLAPQuery(optiqContext, EnumeratorTypeEnum.INDEX, ctxSeq);
}
Example #10
Source File: QueryMetricsFacade.java    From kylin with Apache License 2.0 4 votes vote down vote up 
/**
 * report query related metrics
 */
private static void updateMetricsToReservoir(SQLRequest sqlRequest, SQLResponse sqlResponse) {
    if (!KylinConfig.getInstanceFromEnv().isKylinMetricsReporterForQueryEnabled()) {
        return;
    }
    String user = SecurityContextHolder.getContext().getAuthentication().getName();
    if (user == null) {
        user = "unknown";
    }
    for (QueryContext.RPCStatistics entry : QueryContextFacade.current().getRpcStatisticsList()) {
        RecordEvent rpcMetricsEvent = new TimedRecordEvent(
                KylinConfig.getInstanceFromEnv().getKylinMetricsSubjectQueryRpcCall());
        setRPCWrapper(rpcMetricsEvent, //
                norm(sqlRequest.getProject()), entry.getRealizationName(), entry.getRpcServer(),
                entry.getException());
        setRPCStats(rpcMetricsEvent, //
                entry.getCallTimeMs(), entry.getSkippedRows(), entry.getScannedRows(), entry.getReturnedRows(),
                entry.getAggregatedRows());
        //For update rpc level related metrics
        MetricsManager.getInstance().update(rpcMetricsEvent);
    }
    for (QueryContext.CubeSegmentStatisticsResult contextEntry : sqlResponse.getCubeSegmentStatisticsList()) {
        RecordEvent queryMetricsEvent = new TimedRecordEvent(
                KylinConfig.getInstanceFromEnv().getKylinMetricsSubjectQuery());
        setQueryWrapper(queryMetricsEvent, //
                user, sqlRequest.getSql(), sqlResponse.isStorageCacheUsed() ? "CACHE" : contextEntry.getQueryType(),
                norm(sqlRequest.getProject()), contextEntry.getRealization(), contextEntry.getRealizationType(),
                sqlResponse.getThrowable());

        long totalStorageReturnCount = 0L;
        if (contextEntry.getQueryType().equalsIgnoreCase(OLAPQuery.EnumeratorTypeEnum.OLAP.name())) {
            for (Map<String, QueryContext.CubeSegmentStatistics> cubeEntry : contextEntry.getCubeSegmentStatisticsMap()
                    .values()) {
                for (QueryContext.CubeSegmentStatistics segmentEntry : cubeEntry.values()) {
                    RecordEvent cubeSegmentMetricsEvent = new TimedRecordEvent(
                            KylinConfig.getInstanceFromEnv().getKylinMetricsSubjectQueryCube());

                    setCubeWrapper(cubeSegmentMetricsEvent, //
                            norm(sqlRequest.getProject()), segmentEntry.getCubeName(), segmentEntry.getSegmentName(),
                            segmentEntry.getSourceCuboidId(), segmentEntry.getTargetCuboidId(),
                            segmentEntry.getFilterMask());

                    setCubeStats(cubeSegmentMetricsEvent, //
                            segmentEntry.getCallCount(), segmentEntry.getCallTimeSum(), segmentEntry.getCallTimeMax(),
                            segmentEntry.getStorageSkippedRows(), segmentEntry.getStorageScannedRows(),
                            segmentEntry.getStorageReturnedRows(), segmentEntry.getStorageAggregatedRows(),
                            segmentEntry.isIfSuccess(), 1.0 / cubeEntry.size());

                    totalStorageReturnCount += segmentEntry.getStorageReturnedRows();
                    //For update cube segment level related query metrics
                    MetricsManager.getInstance().update(cubeSegmentMetricsEvent);
                }
            }
        } else {
            if (!sqlResponse.getIsException()) {
                totalStorageReturnCount = sqlResponse.getResults().size();
            }
        }
        setQueryStats(queryMetricsEvent, //
                sqlResponse.getDuration(), sqlResponse.getResults() == null ? 0 : sqlResponse.getResults().size(),
                totalStorageReturnCount);
        //For update query level metrics
        MetricsManager.getInstance().update(queryMetricsEvent);
    }
}
Example #11
Source File: OLAPTable.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public Enumerable<Object[]> executeHiveQuery(DataContext optiqContext, int ctxSeq) {
    return new OLAPQuery(optiqContext, EnumeratorTypeEnum.HIVE, ctxSeq);
}
Example #12
Source File: OLAPTable.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public Enumerable<Object[]> executeLookupTableQuery(DataContext optiqContext, int ctxSeq) {
    return new OLAPQuery(optiqContext, EnumeratorTypeEnum.LOOKUP_TABLE, ctxSeq);
}
Example #13
Source File: OLAPTable.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public Enumerable<Object[]> executeOLAPQuery(DataContext optiqContext, int ctxSeq) {
    return new OLAPQuery(optiqContext, EnumeratorTypeEnum.OLAP, ctxSeq);
}
Example #14
Source File: QueryMetricsFacade.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
/**
 * report query related metrics
 */
private static void updateMetricsToReservoir(SQLRequest sqlRequest, SQLResponse sqlResponse) {
    if (!KylinConfig.getInstanceFromEnv().isKylinMetricsReporterForQueryEnabled()) {
        return;
    }
    String user = SecurityContextHolder.getContext().getAuthentication().getName();
    if (user == null) {
        user = "unknown";
    }
    for (QueryContext.RPCStatistics entry : QueryContextFacade.current().getRpcStatisticsList()) {
        RecordEvent rpcMetricsEvent = new TimedRecordEvent(
                KylinConfig.getInstanceFromEnv().getKylinMetricsSubjectQueryRpcCall());
        setRPCWrapper(rpcMetricsEvent, //
                norm(sqlRequest.getProject()), entry.getRealizationName(), entry.getRpcServer(),
                entry.getException());
        setRPCStats(rpcMetricsEvent, //
                entry.getCallTimeMs(), entry.getSkippedRows(), entry.getScannedRows(), entry.getReturnedRows(),
                entry.getAggregatedRows());
        //For update rpc level related metrics
        MetricsManager.getInstance().update(rpcMetricsEvent);
    }
    for (QueryContext.CubeSegmentStatisticsResult contextEntry : sqlResponse.getCubeSegmentStatisticsList()) {
        RecordEvent queryMetricsEvent = new TimedRecordEvent(
                KylinConfig.getInstanceFromEnv().getKylinMetricsSubjectQuery());
        setQueryWrapper(queryMetricsEvent, //
                user, sqlRequest.getSql(), sqlResponse.isStorageCacheUsed() ? "CACHE" : contextEntry.getQueryType(),
                norm(sqlRequest.getProject()), contextEntry.getRealization(), contextEntry.getRealizationType(),
                sqlResponse.getThrowable());

        long totalStorageReturnCount = 0L;
        if (contextEntry.getQueryType().equalsIgnoreCase(OLAPQuery.EnumeratorTypeEnum.OLAP.name())) {
            for (Map<String, QueryContext.CubeSegmentStatistics> cubeEntry : contextEntry.getCubeSegmentStatisticsMap()
                    .values()) {
                for (QueryContext.CubeSegmentStatistics segmentEntry : cubeEntry.values()) {
                    RecordEvent cubeSegmentMetricsEvent = new TimedRecordEvent(
                            KylinConfig.getInstanceFromEnv().getKylinMetricsSubjectQueryCube());

                    setCubeWrapper(cubeSegmentMetricsEvent, //
                            norm(sqlRequest.getProject()), segmentEntry.getCubeName(), segmentEntry.getSegmentName(),
                            segmentEntry.getSourceCuboidId(), segmentEntry.getTargetCuboidId(),
                            segmentEntry.getFilterMask());

                    setCubeStats(cubeSegmentMetricsEvent, //
                            segmentEntry.getCallCount(), segmentEntry.getCallTimeSum(), segmentEntry.getCallTimeMax(),
                            segmentEntry.getStorageSkippedRows(), segmentEntry.getStorageScannedRows(),
                            segmentEntry.getStorageReturnedRows(), segmentEntry.getStorageAggregatedRows(),
                            segmentEntry.isIfSuccess(), 1.0 / cubeEntry.size());

                    totalStorageReturnCount += segmentEntry.getStorageReturnedRows();
                    //For update cube segment level related query metrics
                    MetricsManager.getInstance().update(cubeSegmentMetricsEvent);
                }
            }
        } else {
            if (!sqlResponse.getIsException()) {
                totalStorageReturnCount = sqlResponse.getResults().size();
            }
        }
        setQueryStats(queryMetricsEvent, //
                sqlResponse.getDuration(), sqlResponse.getResults() == null ? 0 : sqlResponse.getResults().size(),
                totalStorageReturnCount);
        //For update query level metrics
        MetricsManager.getInstance().update(queryMetricsEvent);
    }
}
Example #15
Source File: OLAPTable.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public Enumerable<Object[]> executeHiveQuery(DataContext optiqContext, int ctxSeq) {
    return new OLAPQuery(optiqContext, EnumeratorTypeEnum.HIVE, ctxSeq);
}
Example #16
Source File: OLAPTable.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public Enumerable<Object[]> executeColumnDictionaryQuery(DataContext optiqContext, int ctxSeq) {
    return new OLAPQuery(optiqContext, EnumeratorTypeEnum.COL_DICT, ctxSeq);
}
Example #17
Source File: OLAPTable.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public Enumerable<Object[]> executeLookupTableQuery(DataContext optiqContext, int ctxSeq) {
    return new OLAPQuery(optiqContext, EnumeratorTypeEnum.LOOKUP_TABLE, ctxSeq);
}
Example #18
Source File: OLAPTable.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public Enumerable<Object[]> executeOLAPQuery(DataContext optiqContext, int ctxSeq) {
    return new OLAPQuery(optiqContext, EnumeratorTypeEnum.OLAP, ctxSeq);
}