Java Code Examples for org.apache.kylin.metadata.realization.IRealization#isReady()

The following examples show how to use org.apache.kylin.metadata.realization.IRealization#isReady() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ProjectL2Cache.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
public List<MeasureDesc> listEffectiveRewriteMeasures(String project, String table, boolean onlyRewriteMeasure) {
    Set<IRealization> realizations = getRealizationsByTable(project, table);
    List<MeasureDesc> result = Lists.newArrayList();
    for (IRealization r : realizations) {
        if (!r.isReady())
            continue;

        for (MeasureDesc m : r.getMeasures()) {
            FunctionDesc func = m.getFunction();
            if (belongToTable(func, table, r.getModel())) {
                if (!onlyRewriteMeasure || func.needRewrite()) {
                    result.add(m);
                }
            }
        }
    }
    return result;
}
Example 2
Source File: ProjectL2Cache.java    From kylin with Apache License 2.0 6 votes vote down vote up 
public List<MeasureDesc> listEffectiveRewriteMeasures(String project, String table, boolean onlyRewriteMeasure) {
    Set<IRealization> realizations = getRealizationsByTable(project, table);
    List<MeasureDesc> result = Lists.newArrayList();
    for (IRealization r : realizations) {
        if (!r.isReady())
            continue;

        for (MeasureDesc m : r.getMeasures()) {
            FunctionDesc func = m.getFunction();
            if (belongToTable(func, table, r.getModel())) {
                if (!onlyRewriteMeasure || func.needRewrite()) {
                    result.add(m);
                }
            }
        }
    }
    return result;
}
Example 3
Source File: ProjectL2Cache.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private void markExposedTablesAndColumns(ProjectCache prjCache, IRealization realization) {
    if (!realization.isReady()) {
        return;
    }

    for (TblColRef col : realization.getAllColumns()) {
        TableCache tableCache = prjCache.tables.get(col.getTable());
        prjCache.exposedTables.add(tableCache.tableDesc);
        tableCache.exposed = true;
        tableCache.exposedColumns.add(col.getColumnDesc());
    }
}
Example 4
Source File: CubeManager.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
/**
 * To keep "select * from LOOKUP_TABLE" has consistent and latest result, we manually choose
 * CubeInstance here to answer such query.
 */
public CubeInstance findLatestSnapshot(List<RealizationEntry> realizationEntries, String lookupTableName,
        CubeInstance cubeInstance) {
    CubeInstance cube = null;
    try {
        if (!realizationEntries.isEmpty()) {
            long maxBuildTime = Long.MIN_VALUE;
            RealizationRegistry registry = RealizationRegistry.getInstance(config);
            for (RealizationEntry entry : realizationEntries) {
                IRealization realization = registry.getRealization(entry.getType(), entry.getRealization());
                if (realization != null && realization.isReady() && realization instanceof CubeInstance) {
                    CubeInstance current = (CubeInstance) realization;
                    if (checkMeetSnapshotTable(current, lookupTableName)) {
                        CubeSegment segment = current.getLatestReadySegment();
                        if (segment != null) {
                            long latestBuildTime = segment.getLastBuildTime();
                            if (latestBuildTime > maxBuildTime) {
                                maxBuildTime = latestBuildTime;
                                cube = current;
                            }
                        }
                    }
                }
            }
        }
    } catch (Exception e) {
        logger.info("Unexpected error.", e);
    }
    if (!cubeInstance.equals(cube)) {
        logger.debug("Picked cube {} over {} as it provides a more recent snapshot of the lookup table {}", cube,
                cubeInstance, lookupTableName);
    }
    return cube == null ? cubeInstance : cube;
}
Example 5
Source File: ProjectL2Cache.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private void markExposedTablesAndColumns(ProjectCache prjCache, IRealization realization) {
    if (!realization.isReady()) {
        return;
    }

    for (TblColRef col : realization.getAllColumns()) {
        TableCache tableCache = prjCache.tables.get(col.getTable());
        prjCache.exposedTables.add(tableCache.tableDesc);
        tableCache.exposed = true;
        tableCache.exposedColumns.add(col.getColumnDesc());
    }
}
Example 6
Source File: CubeManager.java    From kylin with Apache License 2.0 5 votes vote down vote up 
/**
 * To keep "select * from LOOKUP_TABLE" has consistent and latest result, we manually choose
 * CubeInstance here to answer such query.
 */
public CubeInstance findLatestSnapshot(List<RealizationEntry> realizationEntries, String lookupTableName,
        CubeInstance cubeInstance) {
    CubeInstance cube = null;
    try {
        if (!realizationEntries.isEmpty()) {
            long maxBuildTime = Long.MIN_VALUE;
            RealizationRegistry registry = RealizationRegistry.getInstance(config);
            for (RealizationEntry entry : realizationEntries) {
                IRealization realization = registry.getRealization(entry.getType(), entry.getRealization());
                if (realization != null && realization.isReady() && realization instanceof CubeInstance) {
                    CubeInstance current = (CubeInstance) realization;
                    if (current.getDescriptor().findDimensionByTable(lookupTableName) != null) {
                        CubeSegment segment = current.getLatestReadySegment();
                        if (segment != null) {
                            long latestBuildTime = segment.getLastBuildTime();
                            if (latestBuildTime > maxBuildTime) {
                                maxBuildTime = latestBuildTime;
                                cube = current;
                            }
                        }
                    }
                }
            }
        }
    } catch (Exception e) {
        logger.info("Unexpected error.", e);
        throw e;
    }
    if (!cubeInstance.equals(cube)) {
        logger.debug("Picked cube {} over {} as it provides a more recent snapshot of the lookup table {}", cube,
                cubeInstance, lookupTableName);
    }
    return cube == null ? cubeInstance : cube;
}
Example 7
Source File: ProjectL2Cache.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
public List<IRealization> getOnlineRealizationByFactTable(String project, String factTable) {
    Set<IRealization> realizations = getRealizationsByTable(project, factTable);
    List<IRealization> result = Lists.newArrayListWithCapacity(realizations.size());
    for (IRealization r : realizations) {
        if (r.getFactTable().equalsIgnoreCase(factTable) && r.isReady()) {
            result.add(r);
        }
    }
    return result;
}
Example 8
Source File: ProjectL2Cache.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
public List<MeasureDesc> listEffectiveRewriteMeasures(String project, String factTable) {
    Set<IRealization> realizations = getRealizationsByTable(project, factTable);
    List<MeasureDesc> result = Lists.newArrayList();
    for (IRealization r : realizations) {
        if (r.getFactTable().equalsIgnoreCase(factTable) && r.isReady()) {
            for (MeasureDesc m : r.getMeasures()) {
                FunctionDesc func = m.getFunction();
                if (func.needRewrite())
                    result.add(m);
            }
        }
    }
    return result;
}
Example 9
Source File: ProjectL2Cache.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private void markExposedTablesAndColumns(ProjectCache prjCache, IRealization realization) {
    if (!realization.isReady()) {
        return;
    }

    for (TblColRef col : realization.getAllColumns()) {
        TableCache tableCache = prjCache.tables.get(col.getTable());
        prjCache.exposedTables.add(tableCache.tableDesc);
        tableCache.exposed = true;
        tableCache.exposedColumns.add(col.getColumn());
    }
}
Example 10
Source File: RealizationChooser.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
private static Map<DataModelDesc, Set<IRealization>> makeOrderedModelMap(OLAPContext context) {
    OLAPContext first = context;
    KylinConfig kylinConfig = first.olapSchema.getConfig();
    String projectName = first.olapSchema.getProjectName();
    String factTableName = first.firstTableScan.getOlapTable().getTableName();
    Set<IRealization> realizations = ProjectManager.getInstance(kylinConfig).getRealizationsByTable(projectName,
            factTableName);

    final Map<DataModelDesc, Set<IRealization>> models = Maps.newHashMap();
    final Map<DataModelDesc, RealizationCost> costs = Maps.newHashMap();

    for (IRealization real : realizations) {
        if (real.isReady() == false) {
            context.realizationCheck.addIncapableCube(real,
                    RealizationCheck.IncapableReason.create(RealizationCheck.IncapableType.CUBE_NOT_READY));
            continue;
        }
        if (containsAll(real.getAllColumnDescs(), first.allColumns) == false) {
            context.realizationCheck.addIncapableCube(real, RealizationCheck.IncapableReason
                    .notContainAllColumn(notContain(real.getAllColumnDescs(), first.allColumns)));
            continue;
        }
        if (RemoveBlackoutRealizationsRule.accept(real) == false) {
            context.realizationCheck.addIncapableCube(real, RealizationCheck.IncapableReason
                    .create(RealizationCheck.IncapableType.CUBE_BLACK_OUT_REALIZATION));
            continue;
        }

        RealizationCost cost = new RealizationCost(real);
        DataModelDesc m = real.getModel();
        Set<IRealization> set = models.get(m);
        if (set == null) {
            set = Sets.newHashSet();
            set.add(real);
            models.put(m, set);
            costs.put(m, cost);
        } else {
            set.add(real);
            RealizationCost curCost = costs.get(m);
            if (cost.compareTo(curCost) < 0)
                costs.put(m, cost);
        }
    }

    // order model by cheapest realization cost
    TreeMap<DataModelDesc, Set<IRealization>> result = Maps.newTreeMap(new Comparator<DataModelDesc>() {
        @Override
        public int compare(DataModelDesc o1, DataModelDesc o2) {
            RealizationCost c1 = costs.get(o1);
            RealizationCost c2 = costs.get(o2);
            int comp = c1.compareTo(c2);
            if (comp == 0)
                comp = o1.getName().compareTo(o2.getName());
            return comp;
        }
    });
    result.putAll(models);

    return result;
}
Example 11
Source File: QueryRouter.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public static IRealization selectRealization(OLAPContext olapContext, Set<IRealization> realizations)
        throws NoRealizationFoundException {

    String factTableName = olapContext.firstTableScan.getTableName();
    String projectName = olapContext.olapSchema.getProjectName();
    SQLDigest sqlDigest = olapContext.getSQLDigest();

    List<Candidate> candidates = Lists.newArrayListWithCapacity(realizations.size());
    for (IRealization real : realizations) {
        if (real.isReady())
            candidates.add(new Candidate(real, sqlDigest));
        if (BackdoorToggles.getForceHitCube() != null && BackdoorToggles.getForceHitCube().equalsIgnoreCase(real.getName())) {
            logger.info("Force choose {} as selected cube for specific purpose.", real.getName());
            return real;
        }
    }

    logger.info("Find candidates by table " + factTableName + " and project=" + projectName + " : "
            + StringUtils.join(candidates, ","));

    List<Candidate> originCandidates = Lists.newArrayList(candidates);

    // rule based realization selection, rules might reorder realizations or remove specific realization
    RoutingRule.applyRules(candidates);

    collectIncapableReason(olapContext, originCandidates);

    if (candidates.size() == 0) {
        return null;
    }

    Candidate chosen = candidates.get(0);
    adjustForDimensionAsMeasure(chosen, olapContext);

    logger.info("The realizations remaining: " + RoutingRule.getPrintableText(candidates)
            + ",and the final chosen one for current olap context " + olapContext.id + " is "
            + chosen.realization.getCanonicalName());

    for (CapabilityInfluence influence : chosen.getCapability().influences) {
        if (influence.getInvolvedMeasure() != null) {
            olapContext.involvedMeasure.add(influence.getInvolvedMeasure());
        }
    }

    return chosen.realization;
}
Example 12
Source File: RealizationChooser.java    From kylin with Apache License 2.0 4 votes vote down vote up 
private static Map<DataModelDesc, Set<IRealization>> makeOrderedModelMap(OLAPContext context) {
    OLAPContext first = context;
    KylinConfig kylinConfig = first.olapSchema.getConfig();
    String projectName = first.olapSchema.getProjectName();
    String factTableName = first.firstTableScan.getOlapTable().getTableName();
    Set<IRealization> realizations = ProjectManager.getInstance(kylinConfig).getRealizationsByTable(projectName,
            factTableName);

    final Map<DataModelDesc, Set<IRealization>> models = Maps.newHashMap();
    final Map<DataModelDesc, RealizationCost> costs = Maps.newHashMap();

    for (IRealization real : realizations) {
        if (real.isReady() == false) {
            context.realizationCheck.addIncapableCube(real,
                    RealizationCheck.IncapableReason.create(RealizationCheck.IncapableType.CUBE_NOT_READY));
            continue;
        }
        if (containsAll(real.getAllColumnDescs(), first.allColumns) == false) {
            context.realizationCheck.addIncapableCube(real, RealizationCheck.IncapableReason
                    .notContainAllColumn(notContain(real.getAllColumnDescs(), first.allColumns)));
            continue;
        }
        if (RemoveBlackoutRealizationsRule.accept(real) == false) {
            context.realizationCheck.addIncapableCube(real, RealizationCheck.IncapableReason
                    .create(RealizationCheck.IncapableType.CUBE_BLACK_OUT_REALIZATION));
            continue;
        }

        RealizationCost cost = new RealizationCost(real);
        DataModelDesc m = real.getModel();
        Set<IRealization> set = models.get(m);
        if (set == null) {
            set = Sets.newHashSet();
            set.add(real);
            models.put(m, set);
            costs.put(m, cost);
        } else {
            set.add(real);
            RealizationCost curCost = costs.get(m);
            if (cost.compareTo(curCost) < 0)
                costs.put(m, cost);
        }
    }

    // order model by cheapest realization cost
    TreeMap<DataModelDesc, Set<IRealization>> result = Maps.newTreeMap(new Comparator<DataModelDesc>() {
        @Override
        public int compare(DataModelDesc o1, DataModelDesc o2) {
            RealizationCost c1 = costs.get(o1);
            RealizationCost c2 = costs.get(o2);
            int comp = c1.compareTo(c2);
            if (comp == 0)
                comp = o1.getName().compareTo(o2.getName());
            return comp;
        }
    });
    result.putAll(models);

    return result;
}
Example 13
Source File: QueryRouter.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public static IRealization selectRealization(OLAPContext olapContext, Set<IRealization> realizations)
        throws NoRealizationFoundException {

    String factTableName = olapContext.firstTableScan.getTableName();
    String projectName = olapContext.olapSchema.getProjectName();
    SQLDigest sqlDigest = olapContext.getSQLDigest();

    List<Candidate> candidates = Lists.newArrayListWithCapacity(realizations.size());
    for (IRealization real : realizations) {
        if (real.isReady())
            candidates.add(new Candidate(real, sqlDigest));
        if (BackdoorToggles.getForceHitCube() != null && BackdoorToggles.getForceHitCube().equalsIgnoreCase(real.getName())) {
            logger.info("Force choose {} as selected cube for specific purpose.", real.getName());
            return real;
        }
    }

    logger.info("Find candidates by table " + factTableName + " and project=" + projectName + " : "
            + StringUtils.join(candidates, ","));

    List<Candidate> originCandidates = Lists.newArrayList(candidates);

    // rule based realization selection, rules might reorder realizations or remove specific realization
    RoutingRule.applyRules(candidates);

    collectIncapableReason(olapContext, originCandidates);

    if (candidates.size() == 0) {
        return null;
    }

    Candidate chosen = candidates.get(0);
    adjustForDimensionAsMeasure(chosen, olapContext);

    logger.info("The realizations remaining: " + RoutingRule.getPrintableText(candidates)
            + ",and the final chosen one for current olap context " + olapContext.id + " is "
            + chosen.realization.getCanonicalName());

    for (CapabilityInfluence influence : chosen.getCapability().influences) {
        if (influence.getInvolvedMeasure() != null) {
            olapContext.involvedMeasure.add(influence.getInvolvedMeasure());
        }
    }

    return chosen.realization;
}