org.apache.kylin.rest.response.SQLResponse Java Exaples

Source File: RealizationSetCalculator.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

@Override
public String calculateSignature(KylinConfig config, SQLResponse sqlResponse, ProjectInstance project) {
    Set<String> realizations = getRealizations(config, sqlResponse.getCube(), project);
    if (realizations == null) {
        return null;
    }
    Set<RealizationSignature> signatureSet = Sets.newTreeSet();
    for (String realization : realizations) {
        RealizationSignature realizationSignature = getRealizationSignature(config, realization);
        if (realizationSignature != null) {
            signatureSet.add(realizationSignature);
        }
    }
    if (signatureSet.isEmpty()) {
        return null;
    }
    try {
        MessageDigest md = MessageDigest.getInstance("MD5");
        byte[] signature = md.digest(signatureSet.toString().getBytes("UTF-8"));
        return new String(Base64.encodeBase64(signature), "UTF-8");
    } catch (Exception e) {
        logger.warn("Failed to calculate signature due to " + e);
        return null;
    }
}

Source File: QueryMetrics2Facade.java From kylin with Apache License 2.0

6 votes

public static void updateMetrics(SQLRequest sqlRequest, SQLResponse sqlResponse) {
    if (!enabled) {
        return;
    }
    if (metrics == null) {
        metrics = MetricsFactory.getInstance();
    }
    String projectName = sqlRequest.getProject();
    String cube = sqlResponse.getCube();
    if (cube == null) {
        return;
    }
    String cubeName = cube.replace("=", "->");

    //        update(getQueryMetrics("Server_Total"), sqlResponse);
    update(buildCubeMetricPrefix(TOTAL), sqlResponse);
    update(buildCubeMetricPrefix(projectName), sqlResponse);
    String cubeMetricName = buildCubeMetricPrefix(projectName, cubeName);
    update(cubeMetricName, sqlResponse);
}

Source File: QueryServiceTest.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

@Test
public void testBasics() throws JobException, IOException, SQLException {
    Assert.assertNotNull(queryService.getConfig());
    Assert.assertNotNull(queryService.getConfig());
    Assert.assertNotNull(queryService.getDataModelManager());
    Assert.assertNotNull(QueryConnection.getConnection(ProjectInstance.DEFAULT_PROJECT_NAME));

    //        Assert.assertTrue(queryService.getQueries("ADMIN").size() == 0);
    //
    //        queryService.saveQuery("test", "test", "select * from test_table", "test");
    //        Assert.assertTrue(queryService.getQueries("ADMIN").size() == 1);
    //
    //        queryService.removeQuery(queryService.getQueries("ADMIN").get(0).getProperty("id"));
    //        Assert.assertTrue(queryService.getQueries("ADMIN").size() == 0);

    SQLRequest request = new SQLRequest();
    request.setSql("select * from test_table");
    request.setAcceptPartial(true);
    QueryContext queryContext = QueryContextFacade.current();
    SQLResponse response = new SQLResponse();
    response.setHitExceptionCache(true);
    queryService.logQuery(queryContext.getQueryId(), request, response);
}

Source File: QueryServiceTest.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

@Test
public void testCreateTableToWith() {
    String create_table1 = " create table tableId as select * from some_table1;";
    String create_table2 = "CREATE TABLE tableId2 AS select * FROM some_table2;";
    String select_table = "select * from tableId join tableId2 on tableId.a = tableId2.b;";

    KylinConfig config = KylinConfig.getInstanceFromEnv();
    config.setProperty("kylin.query.convert-create-table-to-with", "true");
    try (SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(config)) {

        SQLRequest request = new SQLRequest();
        request.setProject("default");
        request.setSql(create_table1);
        queryService.doQueryWithCache(request);

        request.setSql(create_table2);
        queryService.doQueryWithCache(request);

        request.setSql(select_table);
        SQLResponse response = queryService.doQueryWithCache(request, true);

        Assert.assertEquals(
                "WITH tableId as (select * from some_table1) , tableId2 AS (select * FROM some_table2) select * from tableId join tableId2 on tableId.a = tableId2.b;",
                response.getExceptionMessage());
    }
}

Source File: QueryMetrics2Facade.java From kylin with Apache License 2.0

6 votes

private static void update(String name, SQLResponse sqlResponse) {
    try {
        incrQueryCount(name, sqlResponse);
        incrCacheHitCount(name, sqlResponse);
        if (!sqlResponse.getIsException()) {
            metrics.updateTimer(MetricsNameBuilder.buildMetricName(name, MetricsConstant.QUERY_DURATION),
                    sqlResponse.getDuration(), TimeUnit.MILLISECONDS);
            metrics.updateHistogram(MetricsNameBuilder.buildMetricName(name, MetricsConstant.QUERY_RESULT_ROWCOUNT),
                    sqlResponse.getResults().size());
            metrics.updateHistogram(MetricsNameBuilder.buildMetricName(name, MetricsConstant.QUERY_SCAN_ROWCOUNT),
                    sqlResponse.getTotalScanCount());
        }
    } catch (Exception e) {
        logger.error(e.getMessage());
    }

}

Source File: DashboardController.java From kylin with Apache License 2.0

6 votes

@RequestMapping(value = "/metric/query", method = RequestMethod.GET)
@ResponseBody
public MetricsResponse getQueryMetrics(@RequestParam(value = "projectName", required = false) String projectName,
        @RequestParam(value = "cubeName", required = false) String cubeName,
        @RequestParam(value = "startTime") String startTime, @RequestParam(value = "endTime") String endTime) {
    checkAuthorization(projectName);
    MetricsResponse queryMetrics = new MetricsResponse();
    PrepareSqlRequest sqlRequest = dashboardService.getQueryMetricsSQLRequest(startTime, endTime, projectName,
            cubeName);
    SQLResponse sqlResponse = queryService.doQueryWithCache(sqlRequest);
    if (!sqlResponse.getIsException()) {
        queryMetrics.increase("queryCount",
                dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(0)));
        queryMetrics.increase("avgQueryLatency",
                dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(1)));
        queryMetrics.increase("maxQueryLatency",
                dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(2)));
        queryMetrics.increase("minQueryLatency",
                dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(3)));
    }
    return queryMetrics;
}

Source File: QueryMetrics2Facade.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

public static void updateMetrics(SQLRequest sqlRequest, SQLResponse sqlResponse) {
    if (!enabled) {
        return;
    }
    if (metrics == null) {
        metrics = MetricsFactory.getInstance();
    }
    String projectName = sqlRequest.getProject();
    String cube = sqlResponse.getCube();
    if (cube == null) {
        return;
    }
    String cubeName = cube.replace("=", "->");

    //        update(getQueryMetrics("Server_Total"), sqlResponse);
    update(buildCubeMetricPrefix(TOTAL), sqlResponse);
    update(buildCubeMetricPrefix(projectName), sqlResponse);
    String cubeMetricName = buildCubeMetricPrefix(projectName, cubeName);
    update(cubeMetricName, sqlResponse);
}

Source File: QueryServiceTest.java From Kylin with Apache License 2.0

6 votes

@Test
public void testBasics() throws JobException, IOException, SQLException {
    Assert.assertNotNull(queryService.getConfig());
    Assert.assertNotNull(queryService.getKylinConfig());
    Assert.assertNotNull(queryService.getMetadataManager());
    Assert.assertNotNull(queryService.getOLAPDataSource(ProjectInstance.DEFAULT_PROJECT_NAME));

    //        Assert.assertTrue(queryService.getQueries("ADMIN").size() == 0);
    //
    //        queryService.saveQuery("test", "test", "select * from test_table", "test");
    //        Assert.assertTrue(queryService.getQueries("ADMIN").size() == 1);
    //
    //        queryService.removeQuery(queryService.getQueries("ADMIN").get(0).getProperty("id"));
    //        Assert.assertTrue(queryService.getQueries("ADMIN").size() == 0);

    SQLRequest request = new SQLRequest();
    request.setSql("select * from test_table");
    request.setAcceptPartial(true);
    SQLResponse response = new SQLResponse();
    response.setHitCache(true);
    queryService.logQuery(request, response, new Date(), new Date());
}

Source File: QueryMetrics2Facade.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

private static void update(String name, SQLResponse sqlResponse) {
    try {
        incrQueryCount(name, sqlResponse);
        incrCacheHitCount(name, sqlResponse);
        if (!sqlResponse.getIsException()) {
            metrics.updateTimer(MetricsNameBuilder.buildMetricName(name, MetricsConstant.QUERY_DURATION),
                    sqlResponse.getDuration(), TimeUnit.MILLISECONDS);
            metrics.updateHistogram(MetricsNameBuilder.buildMetricName(name, MetricsConstant.QUERY_RESULT_ROWCOUNT),
                    sqlResponse.getResults().size());
            metrics.updateHistogram(MetricsNameBuilder.buildMetricName(name, MetricsConstant.QUERY_SCAN_ROWCOUNT),
                    sqlResponse.getTotalScanCount());
        }
    } catch (Exception e) {
        logger.error(e.getMessage());
    }

}

Source File: QueryServiceTest.java From kylin with Apache License 2.0

6 votes

@Test
public void testCreateTableToWith() {
    String create_table1 = " create table tableId as select * from some_table1;";
    String create_table2 = "CREATE TABLE tableId2 AS select * FROM some_table2;";
    String select_table = "select * from tableId join tableId2 on tableId.a = tableId2.b;";

    KylinConfig config = KylinConfig.getInstanceFromEnv();
    config.setProperty("kylin.query.convert-create-table-to-with", "true");
    try (SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(config)) {

        SQLRequest request = new SQLRequest();
        request.setProject("default");
        request.setSql(create_table1);
        queryService.doQueryWithCache(request);

        request.setSql(create_table2);
        queryService.doQueryWithCache(request);

        request.setSql(select_table);
        SQLResponse response = queryService.doQueryWithCache(request, true);

        Assert.assertEquals(
                "WITH tableId as (select * from some_table1) , tableId2 AS (select * FROM some_table2) select * from tableId join tableId2 on tableId.a = tableId2.b;",
                response.getExceptionMessage());
    }
}

Source File: QueryServiceTest.java From kylin with Apache License 2.0

6 votes

@Test
public void testBasics() throws JobException, IOException, SQLException {
    Assert.assertNotNull(queryService.getConfig());
    Assert.assertNotNull(queryService.getConfig());
    Assert.assertNotNull(queryService.getDataModelManager());
    Assert.assertNotNull(QueryConnection.getConnection(ProjectInstance.DEFAULT_PROJECT_NAME));

    //        Assert.assertTrue(queryService.getQueries("ADMIN").size() == 0);
    //
    //        queryService.saveQuery("test", "test", "select * from test_table", "test");
    //        Assert.assertTrue(queryService.getQueries("ADMIN").size() == 1);
    //
    //        queryService.removeQuery(queryService.getQueries("ADMIN").get(0).getProperty("id"));
    //        Assert.assertTrue(queryService.getQueries("ADMIN").size() == 0);

    SQLRequest request = new SQLRequest();
    request.setSql("select * from test_table");
    request.setAcceptPartial(true);
    QueryContext queryContext = QueryContextFacade.current();
    SQLResponse response = new SQLResponse();
    response.setHitExceptionCache(true);
    queryService.logQuery(queryContext.getQueryId(), request, response);
}

Source File: DashboardController.java From kylin with Apache License 2.0

6 votes

@RequestMapping(value = "/metric/job", method = RequestMethod.GET)
@ResponseBody
public MetricsResponse getJobMetrics(@RequestParam(value = "projectName", required = false) String projectName,
        @RequestParam(value = "cubeName", required = false) String cubeName,
        @RequestParam(value = "startTime") String startTime, @RequestParam(value = "endTime") String endTime) {
    checkAuthorization(projectName);
    MetricsResponse jobMetrics = new MetricsResponse();
    PrepareSqlRequest sqlRequest = dashboardService.getJobMetricsSQLRequest(startTime, endTime, projectName,
            cubeName);
    SQLResponse sqlResponse = queryService.doQueryWithCache(sqlRequest);
    if (!sqlResponse.getIsException()) {
        jobMetrics.increase("jobCount", dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(0)));
        jobMetrics.increase("avgJobBuildTime",
                dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(1)));
        jobMetrics.increase("maxJobBuildTime",
                dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(2)));
        jobMetrics.increase("minJobBuildTime",
                dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(3)));
    }
    return jobMetrics;
}

Source File: DashboardController.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

@RequestMapping(value = "/metric/query", method = RequestMethod.GET)
@ResponseBody
public MetricsResponse getQueryMetrics(@RequestParam(value = "projectName", required = false) String projectName,
        @RequestParam(value = "cubeName", required = false) String cubeName,
        @RequestParam(value = "startTime") String startTime, @RequestParam(value = "endTime") String endTime) {
    checkAuthorization(projectName);
    MetricsResponse queryMetrics = new MetricsResponse();
    PrepareSqlRequest sqlRequest = dashboardService.getQueryMetricsSQLRequest(startTime, endTime, projectName,
            cubeName);
    SQLResponse sqlResponse = queryService.doQueryWithCache(sqlRequest);
    if (!sqlResponse.getIsException()) {
        queryMetrics.increase("queryCount",
                dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(0)));
        queryMetrics.increase("avgQueryLatency",
                dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(1)));
        queryMetrics.increase("maxQueryLatency",
                dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(2)));
        queryMetrics.increase("minQueryLatency",
                dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(3)));
    }
    return queryMetrics;
}

Source File: DashboardController.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

@RequestMapping(value = "/metric/job", method = RequestMethod.GET)
@ResponseBody
public MetricsResponse getJobMetrics(@RequestParam(value = "projectName", required = false) String projectName,
        @RequestParam(value = "cubeName", required = false) String cubeName,
        @RequestParam(value = "startTime") String startTime, @RequestParam(value = "endTime") String endTime) {
    checkAuthorization(projectName);
    MetricsResponse jobMetrics = new MetricsResponse();
    PrepareSqlRequest sqlRequest = dashboardService.getJobMetricsSQLRequest(startTime, endTime, projectName,
            cubeName);
    SQLResponse sqlResponse = queryService.doQueryWithCache(sqlRequest);
    if (!sqlResponse.getIsException()) {
        jobMetrics.increase("jobCount", dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(0)));
        jobMetrics.increase("avgJobBuildTime",
                dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(1)));
        jobMetrics.increase("maxJobBuildTime",
                dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(2)));
        jobMetrics.increase("minJobBuildTime",
                dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(3)));
    }
    return jobMetrics;
}

Source File: QueryService.java From Kylin with Apache License 2.0

6 votes

protected SQLResponse executeQuery(String sql, SQLRequest sqlRequest) throws Exception {
    sql = sql.trim().replace(";", "");

    int limit = sqlRequest.getLimit();
    if (limit > 0 && !sql.toLowerCase().contains("limit")) {
        sql += (" LIMIT " + limit);
    }

    int offset = sqlRequest.getOffset();
    if (offset > 0 && !sql.toLowerCase().contains("offset")) {
        sql += (" OFFSET " + offset);
    }

    // add extra parameters into olap context, like acceptPartial
    Map<String, String> parameters = new HashMap<String, String>();
    parameters.put(OLAPContext.PRM_ACCEPT_PARTIAL_RESULT, String.valueOf(sqlRequest.isAcceptPartial()));
    OLAPContext.setParameters(parameters);

    return execute(sql, sqlRequest);
}

Source File: RealizationSetCalculator.java From kylin with Apache License 2.0

6 votes

@Override
public String calculateSignature(KylinConfig config, SQLResponse sqlResponse, ProjectInstance project) {
    Set<String> realizations = getRealizations(config, sqlResponse.getCube(), project);
    if (realizations == null) {
        return null;
    }
    Set<RealizationSignature> signatureSet = Sets.newTreeSet();
    for (String realization : realizations) {
        RealizationSignature realizationSignature = getRealizationSignature(config, realization);
        if (realizationSignature != null) {
            signatureSet.add(realizationSignature);
        }
    }
    if (signatureSet.isEmpty()) {
        return null;
    }
    try {
        MessageDigest md = MessageDigest.getInstance("MD5");
        byte[] signature = md.digest(signatureSet.toString().getBytes("UTF-8"));
        return new String(Base64.encodeBase64(signature), "UTF-8");
    } catch (Exception e) {
        logger.warn("Failed to calculate signature due to " + e);
        return null;
    }
}

Source File: QueryService.java From kylin with Apache License 2.0

6 votes

public SQLResponse update(SQLRequest sqlRequest) throws Exception {
    // non select operations, only supported when enable pushdown
    logger.debug("Query pushdown enabled, redirect the non-select query to pushdown engine.");
    Connection conn = null;
    try {
        conn = QueryConnection.getConnection(sqlRequest.getProject());
        Pair<List<List<String>>, List<SelectedColumnMeta>> r = PushDownUtil.tryPushDownNonSelectQuery(
                sqlRequest.getProject(), sqlRequest.getSql(), conn.getSchema(), BackdoorToggles.getPrepareOnly());

        List<SelectedColumnMeta> columnMetas = Lists.newArrayList();
        columnMetas.add(new SelectedColumnMeta(false, false, false, false, 1, false, Integer.MAX_VALUE, "c0", "c0",
                null, null, null, Integer.MAX_VALUE, 128, 1, "char", false, false, false));

        return buildSqlResponse(sqlRequest.getProject(), true, r.getFirst(), columnMetas);

    } catch (Exception e) {
        logger.info("pushdown engine failed to finish current non-select query");
        throw e;
    } finally {
        close(null, null, conn);
    }
}

Source File: QueryService.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

/**
 * @param correctedSql
 * @param sqlRequest
 * @return
 * @throws Exception
 */
private SQLResponse executeRequest(String correctedSql, SQLRequest sqlRequest, Connection conn) throws Exception {
    Statement stat = null;
    ResultSet resultSet = null;
    boolean isPushDown = false;

    Pair<List<List<String>>, List<SelectedColumnMeta>> r = null;
    try {
        stat = conn.createStatement();
        processStatementAttr(stat, sqlRequest);
        resultSet = stat.executeQuery(correctedSql);

        r = createResponseFromResultSet(resultSet);

    } catch (SQLException sqlException) {
        r = pushDownQuery(sqlRequest, correctedSql, conn, sqlException);
        if (r == null)
            throw sqlException;

        isPushDown = true;
    } finally {
        close(resultSet, stat, null); //conn is passed in, not my duty to close
    }

    return buildSqlResponse(sqlRequest.getProject(), isPushDown, r.getFirst(), r.getSecond());
}

Source File: QueryMetricsFacade.java From kylin with Apache License 2.0

5 votes

private static void update(QueryMetrics queryMetrics, SQLResponse sqlResponse) {
    try {
        incrQueryCount(queryMetrics, sqlResponse);
        incrCacheHitCount(queryMetrics, sqlResponse);

        if (!sqlResponse.getIsException()) {
            queryMetrics.addQueryLatency(sqlResponse.getDuration());
            queryMetrics.addScanRowCount(sqlResponse.getTotalScanCount());
            queryMetrics.addResultRowCount(sqlResponse.getResults().size());
        }
    } catch (Exception e) {
        logger.error(e.getMessage());
    }

}

Source File: BeanTest.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

@Test
public void test() {
    try {
        BeanValidator.validateAccssor(ColumnMeta.class, new String[0]);
        BeanValidator.validateAccssor(TableMeta.class, new String[0]);
        BeanValidator.validateAccssor(SelectedColumnMeta.class, new String[0]);
        BeanValidator.validateAccssor(AccessRequest.class, new String[0]);
        BeanValidator.validateAccssor(CubeRequest.class, new String[0]);
        BeanValidator.validateAccssor(JobListRequest.class, new String[0]);
        BeanValidator.validateAccssor(SQLRequest.class, new String[0]);
        BeanValidator.validateAccssor(AccessEntryResponse.class, new String[0]);
        BeanValidator.validateAccssor(SQLResponse.class, new String[0]);
    } catch (IntrospectionException e) {
    }

    new SQLResponse(null, null, 0, true, null);

    SelectedColumnMeta coulmnMeta = new SelectedColumnMeta(false, false, false, false, 0, false, 0, null, null,
            null, null, null, 0, 0, 0, null, false, false, false);
    Assert.assertTrue(!coulmnMeta.isAutoIncrement());
    Assert.assertTrue(!coulmnMeta.isCaseSensitive());
    Assert.assertTrue(!coulmnMeta.isSearchable());
    Assert.assertTrue(!coulmnMeta.isCurrency());
    Assert.assertTrue(coulmnMeta.getIsNullable() == 0);
    Assert.assertTrue(!coulmnMeta.isSigned());

    Assert.assertEquals(Constant.ACCESS_HAS_ROLE_ADMIN, "hasRole('ROLE_ADMIN')");
    Assert.assertEquals(Constant.ACCESS_POST_FILTER_READ,
            "hasRole('ROLE_ADMIN') " + " or hasPermission(filterObject, 'ADMINISTRATION')"
                    + " or hasPermission(filterObject, 'MANAGEMENT')"
                    + " or hasPermission(filterObject, 'OPERATION')" + " or hasPermission(filterObject, 'READ')");
    Assert.assertEquals(Constant.FakeCatalogName, "defaultCatalog");
    Assert.assertEquals(Constant.FakeSchemaName, "defaultSchema");
    Assert.assertEquals(Constant.IDENTITY_ROLE, "role");
    Assert.assertEquals(Constant.IDENTITY_USER, "user");
}

Source File: QueryControllerTest.java From kylin with Apache License 2.0

5 votes

@Test(expected = Exception.class)
public void testQueryException() throws Exception {
    PrepareSqlRequest sqlRequest = new PrepareSqlRequest();
    sqlRequest.setSql("select * from not_exist_table");
    sqlRequest.setProject("default");
    SQLResponse response1 = queryController.query(sqlRequest);
    Assert.assertEquals(false, response1.getIsException());

    SQLResponse response2 = queryController.query(sqlRequest);
    Assert.assertEquals(false, response2.getIsException());
}

Source File: QueryMetricsFacade.java From kylin with Apache License 2.0

5 votes

private static void updateMetricsToLocal(SQLRequest sqlRequest, SQLResponse sqlResponse) {
    if (!enabled)
        return;

    String projectName = sqlRequest.getProject();
    update(getQueryMetrics("Server_Total"), sqlResponse);
    update(getQueryMetrics(projectName), sqlResponse);

    String cube = sqlResponse.getCube();
    String cubeName = cube.replace("=", "->");
    String cubeMetricName = projectName + ",sub=" + cubeName;
    update(getQueryMetrics(cubeMetricName), sqlResponse);
}

Source File: QueryMetricsFacade.java From kylin with Apache License 2.0

5 votes

private static void incrQueryCount(QueryMetrics queryMetrics, SQLResponse sqlResponse) {
    if (!sqlResponse.isHitExceptionCache() && !sqlResponse.getIsException()) {
        queryMetrics.incrQuerySuccessCount();
    } else {
        queryMetrics.incrQueryFailCount();
    }
    queryMetrics.incrQueryCount();
}

Source File: MigrationRuleSet.java From kylin with Apache License 2.0

5 votes

private long executeQueries(final List<String> queries, final Context ctx) throws Exception {
    int maxThreads = KylinConfig.getInstanceFromEnv().getMigrationRuleQueryLatencyMaxThreads();
    int threadNum = Math.min(maxThreads, queries.size());
    ExecutorService threadPool = Executors.newFixedThreadPool(threadNum);
    CompletionService<Long> completionService = new ExecutorCompletionService<Long>(threadPool);
    final Authentication auth = SecurityContextHolder.getContext().getAuthentication();
    long start = System.currentTimeMillis();
    for (final String query : queries) {
        completionService.submit(new Callable<Long>() {
            @Override
            public Long call() throws Exception {
                SecurityContextHolder.getContext().setAuthentication(auth);
                SQLRequest sqlRequest = new SQLRequest();
                sqlRequest.setProject(ctx.getSrcProjectName());
                sqlRequest.setSql(query);
                SQLResponse sqlResponse = ctx.getQueryService().doQueryWithCache(sqlRequest, false);
                if (sqlResponse.getIsException()) {
                    throw new RuleValidationException(sqlResponse.getExceptionMessage());
                }
                return sqlResponse.getDuration();
            }

        });
    }
    long timeCostSum = 0L;
    for (int i = 0; i < queries.size(); ++i) {
        try {
            timeCostSum += completionService.take().get();
        } catch (InterruptedException | ExecutionException e) {
            threadPool.shutdownNow();
            throw e;
        }
    }
    long end = System.currentTimeMillis();
    logger.info("Execute" + queries.size() + " queries took " + (end - start) + " ms, query time cost sum "
            + timeCostSum + " ms.");
    return timeCostSum / queries.size();
}

Source File: QueryController.java From kylin with Apache License 2.0

5 votes

@RequestMapping(value = "/query/prestate", method = RequestMethod.POST, produces = "application/json")
@ResponseBody
public SQLResponse prepareQuery(@RequestBody PrepareSqlRequest sqlRequest) {
    Map<String, String> newToggles = Maps.newHashMap();
    if (sqlRequest.getBackdoorToggles() != null)
        newToggles.putAll(sqlRequest.getBackdoorToggles());
    newToggles.put(BackdoorToggles.DEBUG_TOGGLE_PREPARE_ONLY, "true");
    sqlRequest.setBackdoorToggles(newToggles);

    return queryService.doQueryWithCache(sqlRequest);
}

Source File: SQLResponseSignatureUtil.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

public static String createSignature(KylinConfig config, SQLResponse sqlResponse, String projectName) {
    ProjectInstance project = ProjectManager.getInstance(config).getProject(projectName);
    Preconditions.checkNotNull(project);

    SignatureCalculator signatureCalculator;
    try {
        Class signatureClass = getSignatureClass(project.getConfig());
        signatureCalculator = (SignatureCalculator) signatureClass.getConstructor().newInstance();
    } catch (Exception e) {
        logger.warn("Will use default signature since fail to construct signature due to " + e);
        signatureCalculator = new FactTableRealizationSetCalculator();
    }
    return signatureCalculator.calculateSignature(config, sqlResponse, project);
}

Source File: SQLResponseSignatureUtil.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

public static boolean checkSignature(KylinConfig config, SQLResponse sqlResponse, String projectName) {
    String old = sqlResponse.getSignature();
    if (old == null) {
        return false;
    }
    String latest = createSignature(config, sqlResponse, projectName);
    return old.equals(latest);
}

Source File: TableauInterceptor.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

public static SQLResponse tableauIntercept(String sql) {

        String[] tokens = sql.split("[\r\n\t \\(\\)]");
        for (int i = 0; i < tableauTestQueries.length; ++i) {
            if (isTokenWiseEqual(tokens, tableauTestQueriesInToken.get(i))) {
                logger.info("Hit fake response " + i);
                return fakeResponses[i];
            }
        }

        return null;
    }

Source File: QueryService.java From kylin with Apache License 2.0

5 votes

private SQLResponse executePrepareRequest(String correctedSql, PrepareSqlRequest sqlRequest,
        PreparedContext preparedContext) throws Exception {
    ResultSet resultSet = null;
    boolean isPushDown = false;

    Pair<List<List<String>>, List<SelectedColumnMeta>> r = null;
    Connection conn = preparedContext.conn;
    try {
        PreparedStatement preparedStatement = preparedContext.preparedStatement;
        processStatementAttr(preparedStatement, sqlRequest);
        for (int i = 0; i < sqlRequest.getParams().length; i++) {
            setParam(preparedStatement, i + 1, (sqlRequest.getParams()[i]));
        }
        resultSet = preparedStatement.executeQuery();
        r = createResponseFromResultSet(resultSet);
    } catch (SQLException sqlException) {
        r = pushDownQuery(sqlRequest, correctedSql, conn, sqlException);
        if (r == null)
            throw sqlException;

        isPushDown = true;
    } finally {
        DBUtils.closeQuietly(resultSet);
    }

    return buildSqlResponse(sqlRequest.getProject(), isPushDown, r.getFirst(), r.getSecond());
}

Source File: DashboardService.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

public MetricsResponse transformChartData(SQLResponse sqlResponse) {
    if (!sqlResponse.getIsException()) {
        MetricsResponse metrics = new MetricsResponse();
        List<List<String>> results = sqlResponse.getResults();
        for (List<String> result : results) {
            String dimension = result.get(0);
            if (dimension != null && !dimension.isEmpty()) {
                String metric = result.get(1);
                metrics.increase(dimension, getMetricValue(metric));
            }
        }
        return metrics;
    }
    return null;
}

org.apache.kylin.rest.response.SQLResponse Java Examples