org.apache.kylin.query.relnode.OLAPContext Java Examples

private void resetRealizationInContext(OLAPContext olapContext) {
    IRealization realization = olapContext.realization;
    if (realization == null) {
        return;
    }
    KylinConfig config = getConfig();
    HybridInstance hybridInstance = HybridManager.getInstance(config).getHybridInstance(realization.getName());
    if (hybridInstance != null) {
        olapContext.realization = hybridInstance;
        return;
    }
    CubeInstance cubeInstance = CubeManager.getInstance(config).getCube(realization.getName());
    if (cubeInstance != null) {
        olapContext.realization = cubeInstance;
    }
}
 

public LookupTableEnumerator(OLAPContext olapContext) {

        //TODO: assuming LookupTableEnumerator is handled by a cube
        CubeInstance cube = (CubeInstance) olapContext.realization;

        String lookupTableName = olapContext.firstTableScan.getTableName();
        DimensionDesc dim = cube.getDescriptor().findDimensionByTable(lookupTableName);
        if (dim == null)
            throw new IllegalStateException("No dimension with derived columns found for lookup table " + lookupTableName + ", cube desc " + cube.getDescriptor());

        CubeManager cubeMgr = CubeManager.getInstance(cube.getConfig());
        LookupStringTable table = cubeMgr.getLookupTable(cube.getLatestReadySegment(), dim);
        this.allRows = table.getAllRows();

        OLAPTable olapTable = (OLAPTable) olapContext.firstTableScan.getOlapTable();
        this.colDescs = olapTable.getExposedColumns();
        this.current = new Object[colDescs.size()];

        reset();
    }
 

private static void convertAggregationToDimension(OLAPContext olapContext, Collection<FunctionDesc> availableAggregations, String factTableName) {
    Iterator<FunctionDesc> it = olapContext.aggregations.iterator();
    while (it.hasNext()) {
        FunctionDesc functionDesc = it.next();
        if (!availableAggregations.contains(functionDesc)) {
            // try to convert the metric to dimension to see if it works
            TblColRef col = functionDesc.selectTblColRef(olapContext.metricsColumns, factTableName);
            functionDesc.setDimensionAsMetric(true);
            olapContext.rewriteFields.remove(functionDesc.getRewriteFieldName());
            if (col != null) {
                olapContext.metricsColumns.remove(col);
                olapContext.groupByColumns.add(col);
            }
            logger.info("Adjust OLAPContext for " + functionDesc);
        }
    }
}
 

private void resetRealizationInContext(OLAPContext olapContext) {
    IRealization realization = olapContext.realization;
    if (realization == null) {
        return;
    }
    KylinConfig config = getConfig();
    HybridInstance hybridInstance = HybridManager.getInstance(config).getHybridInstance(realization.getName());
    if (hybridInstance != null) {
        olapContext.realization = hybridInstance;
        return;
    }
    CubeInstance cubeInstance = CubeManager.getInstance(config).getCube(realization.getName());
    if (cubeInstance != null) {
        olapContext.realization = cubeInstance;
    }
}
 

private SQLResponse queryWithSqlMassage(SQLRequest sqlRequest) throws Exception {
    SQLResponse fakeResponse = QueryUtil.tableauIntercept(sqlRequest.getSql());
    if (null != fakeResponse) {
        logger.debug("Return fake response, is exception? " + fakeResponse.getIsException());
        return fakeResponse;
    }

    String correctedSql = QueryUtil.massageSql(sqlRequest);
    if (correctedSql.equals(sqlRequest.getSql()) == false)
        logger.info("The corrected query: " + correctedSql);

    // add extra parameters into olap context, like acceptPartial
    Map<String, String> parameters = new HashMap<String, String>();
    parameters.put(OLAPContext.PRM_ACCEPT_PARTIAL_RESULT, String.valueOf(sqlRequest.isAcceptPartial()));
    OLAPContext.setParameters(parameters);

    return execute(correctedSql, sqlRequest);
}
 

@Override
public void apply(List<IRealization> realizations, OLAPContext olapContext) {
    if (realizations.size() > 0) {
        IRealization first = realizations.get(0);

        if (first instanceof CubeInstance) {
            CubeInstance cube = (CubeInstance) first;
            adjustOLAPContextIfNecessary(cube, olapContext);
        }

        if (first instanceof IIInstance) {
            IIInstance ii = (IIInstance) first;
            adjustOLAPContextIfNecessary(ii, olapContext);
        }
    }
}
 

public Enumerator<Object[]> enumerator() {
    OLAPContext olapContext = OLAPContext.getThreadLocalContextById(contextId);
    switch (type) {
    case OLAP:
        return BackdoorToggles.getPrepareOnly() ? new EmptyEnumerator()
                : new OLAPEnumerator(olapContext, optiqContext);
    case LOOKUP_TABLE:
        return BackdoorToggles.getPrepareOnly() ? new EmptyEnumerator() : new LookupTableEnumerator(olapContext);
    case COL_DICT:
        return BackdoorToggles.getPrepareOnly() ? new EmptyEnumerator() : new DictionaryEnumerator(olapContext);
    case HIVE:
        return BackdoorToggles.getPrepareOnly() ? new EmptyEnumerator() : new HiveEnumerator(olapContext);
    default:
        throw new IllegalArgumentException("Wrong type " + type + "!");
    }
}
 

public Enumerator<Object[]> enumerator() {
    OLAPContext olapContext = OLAPContext.getThreadLocalContextById(contextId);
    switch (type) {
    case OLAP:
        return BackdoorToggles.getPrepareOnly() ? new EmptyEnumerator()
                : new OLAPEnumerator(olapContext, optiqContext);
    case LOOKUP_TABLE:
        return BackdoorToggles.getPrepareOnly() ? new EmptyEnumerator() : new LookupTableEnumerator(olapContext);
    case COL_DICT:
        return BackdoorToggles.getPrepareOnly() ? new EmptyEnumerator() : new DictionaryEnumerator(olapContext);
    case HIVE:
        return BackdoorToggles.getPrepareOnly() ? new EmptyEnumerator() : new HiveEnumerator(olapContext);
    default:
        throw new IllegalArgumentException("Wrong type " + type + "!");
    }
}
 

public static IRealization selectRealization(OLAPContext olapContext) throws NoRealizationFoundException {

        ProjectManager prjMgr = ProjectManager.getInstance(olapContext.olapSchema.getConfig());
        String factTableName = olapContext.firstTableScan.getTableName();
        String projectName = olapContext.olapSchema.getProjectName();
        List<IRealization> realizations = Lists.newArrayList(prjMgr.getRealizationsByTable(projectName, factTableName));
        logger.info("Find candidates by table " + factTableName + " and project=" + projectName + " : " + StringUtils.join(realizations, ","));

        //rule based realization selection, rules might reorder realizations or remove specific realization
        RoutingRule.applyRules(realizations, olapContext);

        if (realizations.size() == 0) {
            throw new NoRealizationFoundException("Can't find any realization. Please confirm with providers. SQL digest: " + olapContext.getSQLDigest().toString());
        }

        logger.info("The realizations remaining: ");
        logger.info(RoutingRule.getPrintableText(realizations));
        logger.info("The realization being chosen: " + realizations.get(0).getName());

        return realizations.get(0);
    }
 

protected SQLResponse executeQuery(String sql, SQLRequest sqlRequest) throws Exception {
    sql = sql.trim().replace(";", "");

    int limit = sqlRequest.getLimit();
    if (limit > 0 && !sql.toLowerCase().contains("limit")) {
        sql += (" LIMIT " + limit);
    }

    int offset = sqlRequest.getOffset();
    if (offset > 0 && !sql.toLowerCase().contains("offset")) {
        sql += (" OFFSET " + offset);
    }

    // add extra parameters into olap context, like acceptPartial
    Map<String, String> parameters = new HashMap<String, String>();
    parameters.put(OLAPContext.PRM_ACCEPT_PARTIAL_RESULT, String.valueOf(sqlRequest.isAcceptPartial()));
    OLAPContext.setParameters(parameters);

    return execute(sql, sqlRequest);
}
 

private SQLResponse queryWithSqlMassage(SQLRequest sqlRequest) throws Exception {
    SQLResponse fakeResponse = QueryUtil.tableauIntercept(sqlRequest.getSql());
    if (null != fakeResponse) {
        logger.debug("Return fake response, is exception? " + fakeResponse.getIsException());
        return fakeResponse;
    }

    String correctedSql = QueryUtil.massageSql(sqlRequest);
    if (correctedSql.equals(sqlRequest.getSql()) == false)
        logger.info("The corrected query: " + correctedSql);

    // add extra parameters into olap context, like acceptPartial
    Map<String, String> parameters = new HashMap<String, String>();
    parameters.put(OLAPContext.PRM_ACCEPT_PARTIAL_RESULT, String.valueOf(sqlRequest.isAcceptPartial()));
    OLAPContext.setParameters(parameters);

    return execute(correctedSql, sqlRequest);
}
 

public static Set<String> getTableWithSchema(List<OLAPContext> contexts) {
    // all tables with DB, Like DB.TABLE
    Set<String> tableWithSchema = new HashSet<>();
    for (OLAPContext context : contexts) {
        Set<TblColRef> allColumns = context.allColumns;
        for (TblColRef tblColRef : allColumns) {
            tableWithSchema.add(tblColRef.getTableWithSchema());
        }
    }
    return tableWithSchema;
}
 

private Set<String> getAllTblsWithSchema(List<OLAPContext> contexts) {
    // all tables with DB, Like DB.TABLE
    Set<String> tableWithSchema = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
    for (OLAPContext context : contexts) {
        for (OLAPTableScan tableScan : context.allTableScans) {
            tableWithSchema.add(tableScan.getTableRef().getTableIdentity());
        }
    }
    return tableWithSchema;
}
 

private static void collectIncapableReason(OLAPContext olapContext, List<Candidate> candidates) {
    for (Candidate candidate : candidates) {
        if (!candidate.getCapability().capable) {
            RealizationCheck.IncapableReason reason = RealizationCheck.IncapableReason
                    .create(candidate.getCapability().incapableCause);
            if (reason != null)
                olapContext.realizationCheck.addIncapableCube(candidate.getRealization(), reason);
        } else {
            olapContext.realizationCheck.addCapableCube(candidate.getRealization());
        }
    }
}
 

@Override
protected Set<String> getIdentifierBlackList(List<OLAPContext> contexts) {
    String project = getProject(contexts);
    String username = getUser(contexts);

    return TableACLManager
            .getInstance(KylinConfig.getInstanceFromEnv())
            .getTableACLByCache(project)
            .getTableBlackList(username, AclPermissionUtil.getCurrentUserGroups());
}
 

public static void selectRealization(List<OLAPContext> contexts) {
    // try different model for different context

    for (OLAPContext ctx : contexts) {
        ctx.realizationCheck = new RealizationCheck();
        attemptSelectRealization(ctx);
        Preconditions.checkNotNull(ctx.realization);
    }
}
 

private static Map<String, String> matches(DataModelDesc model, OLAPContext ctx) {
    Map<String, String> result = Maps.newHashMap();

    TableRef firstTable = ctx.firstTableScan.getTableRef();

    Map<String, String> matchUp = null;

    if (ctx.joins.isEmpty() && model.isLookupTable(firstTable.getTableIdentity())) {
        // one lookup table
        String modelAlias = model.findFirstTable(firstTable.getTableIdentity()).getAlias();
        matchUp = ImmutableMap.of(firstTable.getAlias(), modelAlias);
    } else if (ctx.joins.size() != ctx.allTableScans.size() - 1) {
        // has hanging tables
        ctx.realizationCheck.addModelIncapableReason(model,
                RealizationCheck.IncapableReason.create(RealizationCheck.IncapableType.MODEL_BAD_JOIN_SEQUENCE));
        throw new IllegalStateException("Please adjust the sequence of join tables. " + toErrorMsg(ctx));
    } else {
        // normal big joins
        if (ctx.joinsTree == null) {
            ctx.joinsTree = new JoinsTree(firstTable, ctx.joins);
        }
        matchUp = ctx.joinsTree.matches(model.getJoinsTree(), result);
    }

    if (matchUp == null) {
        ctx.realizationCheck.addModelIncapableReason(model,
                RealizationCheck.IncapableReason.create(RealizationCheck.IncapableType.MODEL_UNMATCHED_JOIN));
        return null;
    }

    result.putAll(matchUp);

    ctx.realizationCheck.addCapableModel(model, result);
    return result;
}
 

public void apply(List<IRealization> realizations, OLAPContext olapContext) {

        Collections.sort(realizations, new Comparator<IRealization>() {
            @Override
            public int compare(IRealization o1, IRealization o2) {
                int i1 = priorities.get(o1.getType());
                int i2 = priorities.get(o2.getType());
                return i1 - i2;
            }
        });
    }
 

private Set<String> getAllTblsWithSchema(List<OLAPContext> contexts) {
    // all tables with DB, Like DB.TABLE
    Set<String> tableWithSchema = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
    for (OLAPContext context : contexts) {
        for (OLAPTableScan tableScan : context.allTableScans) {
            tableWithSchema.add(tableScan.getTableRef().getTableIdentity());
        }
    }
    return tableWithSchema;
}
 

public static void tableFilter(List<OLAPContext> contexts, List<String> tableBlackList) {
    Set<String> tableWithSchema = getTableWithSchema(contexts);
    for (String tbl : tableBlackList) {
        if (tableWithSchema.contains(tbl.toUpperCase(Locale.ROOT))) {
            //                throw new kylin.AccessDeniedException("table:" + tbl);
            System.out.println("Access table:" + tbl + " denied");
        }
    }
}
 

@Override
public void apply(List<IRealization> realizations, OLAPContext olapContext) {

    // sort cube candidates, 0) the cost indicator, 1) the lesser header
    // columns the better, 2) the lesser body columns the better
    List<Integer> items = super.findRealizationsOf(realizations, RealizationType.CUBE);
    PartialSorter.partialSort(realizations, items, new Comparator<IRealization>() {
        @Override
        public int compare(IRealization o1, IRealization o2) {
            CubeInstance c1 = (CubeInstance) o1;
            CubeInstance c2 = (CubeInstance) o2;
            int comp = 0;
            comp = c1.getCost() - c2.getCost();
            if (comp != 0) {
                return comp;
            }

            CubeDesc schema1 = c1.getDescriptor();
            CubeDesc schema2 = c2.getDescriptor();

            comp = schema1.listDimensionColumnsIncludingDerived().size() - schema2.listDimensionColumnsIncludingDerived().size();
            if (comp != 0)
                return comp;

            comp = schema1.getMeasures().size() - schema2.getMeasures().size();
            return comp;
        }
    });

}
 

@Override
public void apply(List<IRealization> realizations, OLAPContext olapContext) {
    for (Iterator<IRealization> iterator = realizations.iterator(); iterator.hasNext();) {
        IRealization realization = iterator.next();
        if (!realization.isCapable(olapContext.getSQLDigest())) {
            iterator.remove();
        }
    }
}
 

private static PreparedContext createPreparedContext(String project, String sql) throws Exception {
    Connection conn = QueryConnection.getConnection(project);
    PreparedStatement preparedStatement = conn.prepareStatement(sql);
    Collection<OLAPContext> olapContexts = OLAPContext.getThreadLocalContexts();
    // If the preparedContext is first initialized, then set the borrowed tag to false
    for (OLAPContext olapContext : olapContexts) {
        olapContext.isBorrowedContext = false;
    }
    return new PreparedContext(conn, preparedStatement, olapContexts);
}
 

private void intercept(List<OLAPContext> contexts, Collection<String> blackList) {
    if (blackList.isEmpty()) {
        return;
    }

    Collection<String> queryCols = getQueryIdentifiers(contexts);
    for (String id : blackList) {
        if (queryCols.contains(id.toUpperCase(Locale.ROOT))) {
            throw new AccessDeniedException(getIdentifierType() + ":" + id);
        }
    }
}
 

public void intercept(List<OLAPContext> contexts) {
    if (!isEnabled()) {
        return;
    }
    Collection<String> userIdentifierBlackList = getIdentifierBlackList(contexts);
    intercept(contexts, userIdentifierBlackList);
}
 

public void intercept(List<OLAPContext> contexts) {
    if (!isEnabled()) {
        return;
    }
    Collection<String> userIdentifierBlackList = getIdentifierBlackList(contexts);
    intercept(contexts, userIdentifierBlackList);
}
 

private static PreparedContext createPreparedContext(String project, String sql) throws Exception {
    Connection conn = QueryConnection.getConnection(project);
    PreparedStatement preparedStatement = conn.prepareStatement(sql);
    Collection<OLAPContext> olapContexts = OLAPContext.getThreadLocalContexts();
    // If the preparedContext is first initialized, then set the borrowed tag to false
    for (OLAPContext olapContext : olapContexts) {
        olapContext.isBorrowedContext = false;
    }
    return new PreparedContext(conn, preparedStatement, olapContexts);
}
 

private static Dataset<Row> queryWithKap(String prj, String joinType, Pair<String, String> pair,
                                         Map<String, CompareEntity> compareEntityMap) {

    compareEntityMap.putIfAbsent(pair.getFirst(), new CompareEntity());
    final CompareEntity entity = compareEntityMap.get(pair.getFirst());
    entity.setSql(pair.getFirst());
    Dataset<Row> rowDataset = queryFromCube(prj, changeJoinType(pair.getSecond(), joinType));
    entity.setOlapContexts(OLAPContext.getThreadLocalContexts());
    OLAPContext.clearThreadLocalContexts();
    return rowDataset;
}
 

private static void adjustOLAPContextIfNecessary(CubeInstance cube, OLAPContext olapContext) {
    if (CubeCapabilityChecker.check(cube, olapContext.getSQLDigest(), false))
        return;

    CubeDesc cubeDesc = cube.getDescriptor();
    Collection<FunctionDesc> cubeFuncs = cubeDesc.listAllFunctions();
    convertAggregationToDimension(olapContext, cubeFuncs, cubeDesc.getFactTable());
}
 

public static void columnFilter(List<OLAPContext> contexts, List<String> columnBlackList) {
    List<String> allColWithTblAndSchema = getAllColWithTblAndSchema(contexts);
    for (String tbl : columnBlackList) {
        if (allColWithTblAndSchema.contains(tbl.toUpperCase(Locale.ROOT))) {
            //                throw new kylin.AccessDeniedException("table:" + tbl);
            System.out.println("Access table:" + tbl + " denied");
        }
    }
}