org.apache.kylin.metadata.realization.RealizationType Java Examples

public CubeInstance createCube(String cubeName, String projectName, CubeDesc desc, String owner)
        throws IOException {
    try (AutoLock lock = cubeMapLock.lockForWrite()) {
        logger.info("Creating cube '{}-->{}' from desc '{}'", projectName, cubeName, desc.getName());

        // save cube resource
        CubeInstance cube = CubeInstance.create(cubeName, desc);
        cube.setOwner(owner);
        updateCubeWithRetry(new CubeUpdate(cube), 0);

        ProjectManager.getInstance(config).moveRealizationToProject(RealizationType.CUBE, cubeName, projectName,
                owner);

        return cube;
    }
}
 

private static List<String> filterByProjects(List<String> allTableNames, List<String> projectNames) {
    ProjectManager projectManager = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
    CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());

    List<String> result = Lists.newArrayList();
    for (String p : projectNames) {
        p = p.trim();
        if (p.endsWith(",")) {
            p = p.substring(0, p.length() - 1);
        }

        ProjectInstance projectInstance = projectManager.getProject(p);
        List<RealizationEntry> cubeList = projectInstance.getRealizationEntries(RealizationType.CUBE);
        for (RealizationEntry cube : cubeList) {
            CubeInstance cubeInstance = cubeManager.getCube(cube.getRealization());
            for (CubeSegment segment : cubeInstance.getSegments()) {
                String tableName = segment.getStorageLocationIdentifier();
                if (allTableNames.contains(tableName)) {
                    result.add(tableName);
                }
            }
        }
    }
    return result;
}
 

public CubeInstance dropCube(String cubeName, boolean deleteDesc) throws IOException {
    try (AutoLock lock = cubeMapLock.lockForWrite()) {
        logger.info("Dropping cube '{}'", cubeName);
        // load projects before remove cube from project

        // delete cube instance and cube desc
        CubeInstance cube = getCube(cubeName);

        // remove cube and update cache
        crud.delete(cube);
        Cuboid.clearCache(cube);

        if (deleteDesc && cube.getDescriptor() != null) {
            CubeDescManager.getInstance(config).removeCubeDesc(cube.getDescriptor());
        }

        // delete cube from project
        ProjectManager.getInstance(config).removeRealizationsFromProjects(RealizationType.CUBE, cubeName);

        return cube;
    }
}
 

protected boolean isCubeInProject(String projectName, CubeInstance target) {
    ProjectManager projectManager = getProjectManager();
    ProjectInstance project = projectManager.getProject(projectName);
    if (project == null) {
        return false;
    }
    for (RealizationEntry projectDataModel : project.getRealizationEntries()) {
        if (projectDataModel.getType() == RealizationType.CUBE) {
            CubeInstance cube = getCubeManager().getCube(projectDataModel.getRealization());
            if (cube == null) {
                logger.error("Project " + projectName + " contains realization " + projectDataModel.getRealization()
                        + " which is not found by CubeManager");
                continue;
            }
            if (cube.equals(target)) {
                return true;
            }
        }
    }
    return false;
}
 

@Override
public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey)
        throws IOException {
    if ("hybrid".equals(entity)) {
        String hybridName = cacheKey;

        try (AutoLock l = lock.lockForWrite()) {
            if (event == Event.DROP)
                hybridMap.removeLocal(hybridName);
            else
                crud.reloadQuietly(hybridName);
        }

        for (ProjectInstance prj : ProjectManager.getInstance(config).findProjects(RealizationType.HYBRID,
                hybridName)) {
            broadcaster.notifyProjectSchemaUpdate(prj.getName());
        }
    } else if ("cube".equals(entity)) {
        String cubeName = cacheKey;
        try (AutoLock l = lock.lockForWrite()) {
            for (HybridInstance hybrid : getHybridInstancesByChild(RealizationType.CUBE, cubeName)) {
                crud.reloadQuietly(hybrid.getName());
            }
        }
    }
}
 

static HybridSignature getHybridSignature(KylinConfig config, String realizationName) {
    HybridInstance hybridInstance = HybridManager.getInstance(config).getHybridInstance(realizationName);
    if (hybridInstance == null) {
        return null;
    }
    IRealization[] realizations = hybridInstance.getRealizations();
    Set<RealizationSignature> realizationSignatureSet = Sets.newHashSetWithExpectedSize(realizations.length);
    for (IRealization realization : realizations) {
        RealizationSignature realizationSignature = null;
        if (realization.getType() == RealizationType.CUBE) {
            realizationSignature = CubeSignature.getCubeSignature(config, realization.getName());
        } else if (realization.getType() == RealizationType.HYBRID) {
            realizationSignature = getHybridSignature(config, realization.getName());
        }
        if (realizationSignature != null) {
            realizationSignatureSet.add(realizationSignature);
        }
    }
    return new HybridSignature(realizationName, realizationSignatureSet);
}
 

@Before
public void setup() throws SchedulerException {
    this.createTestMetadata("../../examples/test_case_data/file_prunning");
    System.setProperty("kylin.env", "UT");
    System.setProperty("kylin.query.enable-dynamic-column", "false");
    Map<RealizationType, Integer> priorities = Maps.newHashMap();
    priorities.put(RealizationType.HYBRID, 0);
    priorities.put(RealizationType.CUBE, 0);
    Candidate.setPriorities(priorities);
    overwriteSystemProp("kylin.job.scheduler.poll-interval-second", "1");
    overwriteSystemProp("calcite.keep-in-clause", "true");
    overwriteSystemProp("kylin.metadata.distributed-lock-impl", "org.apache.kylin.engine.spark.utils.MockedDistributedLock$MockedFactory");
    DefaultScheduler scheduler = DefaultScheduler.getInstance();
    scheduler.init(new JobEngineConfig(KylinConfig.getInstanceFromEnv()), new MockJobLock());
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
    config = KylinConfig.getInstanceFromEnv();
    cubeMgr = CubeManager.getInstance(config);
    execMgr = ExecutableManager.getInstance(config);
}
 

private static List<String> filterByProjects(List<String> allTableNames, List<String> projectNames) {
    ProjectManager projectManager = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
    CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());

    List<String> result = Lists.newArrayList();
    for (String p : projectNames) {
        p = p.trim();
        if (p.endsWith(",")) {
            p = p.substring(0, p.length() - 1);
        }

        ProjectInstance projectInstance = projectManager.getProject(p);
        List<RealizationEntry> cubeList = projectInstance.getRealizationEntries(RealizationType.CUBE);
        for (RealizationEntry cube : cubeList) {
            CubeInstance cubeInstance = cubeManager.getCube(cube.getRealization());
            for (CubeSegment segment : cubeInstance.getSegments()) {
                String tableName = segment.getStorageLocationIdentifier();
                if (allTableNames.contains(tableName)) {
                    result.add(tableName);
                }
            }
        }
    }
    return result;
}
 

private ProjectInstance addRealizationToProject(RealizationType type, String realizationName, String project,
        String user) throws IOException {
    if (StringUtils.isEmpty(project)) {
        throw new IllegalArgumentException("Project name should not be empty.");
    }
    ProjectInstance newProject = getProject(project);
    if (newProject == null) {
        newProject = this.createProject(project, user,
                "This is a project automatically added when adding realization " + realizationName + "(" + type
                        + ")",
                null);
    }
    newProject.addRealizationEntry(type, realizationName);
    save(newProject);

    return newProject;
}
 

public List<CubeInstance> listAllCubes(String projectName) {
    ProjectManager projectManager = getProjectManager();
    ProjectInstance project = projectManager.getProject(projectName);
    if (project == null) {
        return Collections.emptyList();
    }
    ArrayList<CubeInstance> result = new ArrayList<CubeInstance>();
    for (RealizationEntry projectDataModel : project.getRealizationEntries()) {
        if (projectDataModel.getType() == RealizationType.CUBE) {
            CubeInstance cube = getCubeManager().getCube(projectDataModel.getRealization());
            if (cube != null)
                result.add(cube);
            else
                logger.error("Cube instance " + projectDataModel.getRealization() + " is failed to load");
        }
    }
    return result;
}
 

private List<JobInstance> listJobInstances(String project, String cube, long startTime, long endTime) {
    final List<JobInstance> result = Lists.newArrayList();
    final List<AbstractExecutable> executables = executableManager.getAllExecutables(startTime, endTime);
    final Map<String, Output> allOutputs = executableManager.getAllOutputs();
    for (AbstractExecutable executable : executables) {
        if (executable instanceof CubingJob) {
            String cubeName = CubingExecutableUtil.getCubeName(executable.getParams());
            boolean shouldExtract = false;
            if (cube == null || cube.equalsIgnoreCase(cubeName)) {
                if (project == null) {
                    shouldExtract = true;
                } else {
                    ProjectInstance projectInstance = projectManager.getProject(project);
                    if (projectInstance != null && projectInstance.containsRealization(RealizationType.CUBE, cubeName)) {
                        shouldExtract = true;
                    }
                }
            }

            if (shouldExtract) {
                result.add(parseToJobInstance((CubingJob) executable, allOutputs));
            }
        }
    }
    return result;
}
 

@Override
public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey)
        throws IOException {
    if ("hybrid".equals(entity)) {
        String hybridName = cacheKey;

        try (AutoLock l = lock.lockForWrite()) {
            if (event == Event.DROP)
                hybridMap.removeLocal(hybridName);
            else
                crud.reloadQuietly(hybridName);
        }

        for (ProjectInstance prj : ProjectManager.getInstance(config).findProjects(RealizationType.HYBRID,
                hybridName)) {
            broadcaster.notifyProjectSchemaUpdate(prj.getName());
        }
    } else if ("cube".equals(entity)) {
        String cubeName = cacheKey;
        try (AutoLock l = lock.lockForWrite()) {
            for (HybridInstance hybrid : getHybridInstancesByChild(RealizationType.CUBE, cubeName)) {
                crud.reloadQuietly(hybrid.getName());
            }
        }
    }
}
 

private void update(HybridInstance hybridInstance, List<RealizationEntry> realizationEntries, String projectName, String owner, boolean checkCubeSize) throws IOException {
    if (checkCubeSize)
        checkSegmentOffset(realizationEntries);
    hybridInstance.setRealizationEntries(realizationEntries);
    store.checkAndPutResource(hybridInstance.getResourcePath(), hybridInstance, HybridManager.HYBRID_SERIALIZER);
    ProjectManager.getInstance(kylinConfig).moveRealizationToProject(RealizationType.HYBRID, hybridInstance.getName(), projectName, owner);
    hybridManager.reloadHybridInstance(hybridInstance.getName());
    logger.info("HybridInstance was updated at: " + hybridInstance.getResourcePath());
}
 

public MetricsResponse getCubeMetrics(String projectName, String cubeName) {
    MetricsResponse cubeMetrics = new MetricsResponse();
    Float totalCubeSize = 0f;
    long totalRecoadSize = 0;
    List<CubeInstance> cubeInstances = cubeService.listAllCubes(cubeName, projectName, null, true);
    Integer totalCube = cubeInstances.size();
    if (projectName == null) {
        totalCube += getHybridManager().listHybridInstances().size();
    } else {
        ProjectInstance project = getProjectManager().getProject(projectName);
        totalCube += project.getRealizationCount(RealizationType.HYBRID);
    }
    Float minCubeExpansion = Float.POSITIVE_INFINITY;
    Float maxCubeExpansion = Float.NEGATIVE_INFINITY;
    cubeMetrics.increase("totalCube", totalCube.floatValue());
    for (CubeInstance cubeInstance : cubeInstances) {
        if (cubeInstance.getInputRecordSizeBytes() > 0) {
            totalCubeSize += cubeInstance.getSizeKB();
            totalRecoadSize += cubeInstance.getInputRecordSizeBytes();
            Float cubeExpansion = new Float(cubeInstance.getSizeKB()) * 1024
                    / cubeInstance.getInputRecordSizeBytes();
            if (cubeExpansion > maxCubeExpansion) {
                maxCubeExpansion = cubeExpansion;
            }
            if (cubeExpansion < minCubeExpansion) {
                minCubeExpansion = cubeExpansion;
            }
        }
    }
    Float avgCubeExpansion = 0f;
    if (totalRecoadSize != 0) {
        avgCubeExpansion = totalCubeSize * 1024 / totalRecoadSize;
    }
    cubeMetrics.increase("avgCubeExpansion", avgCubeExpansion);
    cubeMetrics.increase("maxCubeExpansion", maxCubeExpansion == Float.NEGATIVE_INFINITY ? 0 : maxCubeExpansion);
    cubeMetrics.increase("minCubeExpansion", minCubeExpansion == Float.POSITIVE_INFINITY ? 0 : minCubeExpansion);
    return cubeMetrics;
}
 

public String getProjectOfCube(String cubeName) {
    for (ProjectInstance p : getProjectManager().listAllProjects()) {
        if (p.containsRealization(RealizationType.CUBE, cubeName))
            return p.getName();
    }
    return null;
}
 

@Test
public void testHappyIngest() {
    String srcPath = Thread.currentThread().getContextClassLoader().getResource("cloned_cube_and_model.zip").getPath();
    CubeMetaIngester.main(new String[] { "-project", "default", "-srcPath", srcPath });

    ProjectInstance project = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).getProject("default");
    Assert.assertEquals(1, Collections.frequency(project.getTables(), "DEFAULT.TEST_KYLIN_FACT"));
    Assert.assertTrue(project.getModels().contains("cloned_model"));
    Assert.assertTrue(project.getRealizationEntries().contains(RealizationEntry.create(RealizationType.CUBE, "cloned_cube")));

    getTestConfig().clearManagers();
    CubeInstance instance = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube("cloned_cube");
    Assert.assertTrue(instance != null);
}
 

public boolean containsRealization(final RealizationType type, final String realization) {
    return Iterables.any(this.realizationEntries, new Predicate<RealizationEntry>() {
        @Override
        public boolean apply(RealizationEntry input) {
            return input.getType() == type && input.getRealization().equalsIgnoreCase(realization);
        }
    });
}
 

public List<ProjectInstance> findProjects(RealizationType type, String realizationName) {
    try (AutoLock lock = prjMapLock.lockForWrite()) {
        List<ProjectInstance> result = Lists.newArrayList();
        for (ProjectInstance prj : projectMap.values()) {
            for (RealizationEntry entry : prj.getRealizationEntries()) {
                if (entry.getType().equals(type) && entry.getRealization().equals(realizationName)) {
                    result.add(prj);
                    break;
                }
            }
        }
        return result;
    }
}
 

private boolean containsRealization(Set<IRealization> realizations, RealizationType type, String name) {
    for (IRealization realization : realizations) {
        if (realization.getType() == type && realization.getName().equals(name)) {
            return true;
        }
    }
    return false;
}
 

protected List<Integer> findRealizationsOf(List<IRealization> realizations, RealizationType type) {
    List<Integer> itemIndexes = Lists.newArrayList();
    for (int i = 0; i < realizations.size(); ++i) {
        if (realizations.get(i).getType() == type) {
            itemIndexes.add(i);
        }
    }
    return itemIndexes;
}
 

@Override
public void setup() throws SchedulerException {
    super.setup();
    overwriteSystemProp("kylin.env", "UT");
    overwriteSystemProp("isDeveloperMode", "true");
    overwriteSystemProp("kylin.query.enable-dynamic-column", "false");
    Map<RealizationType, Integer> priorities = Maps.newHashMap();
    priorities.put(RealizationType.HYBRID, 0);
    priorities.put(RealizationType.CUBE, 0);
    Candidate.setPriorities(priorities);
    config = KylinConfig.getInstanceFromEnv();
    cubeMgr = CubeManager.getInstance(config);
    execMgr = ExecutableManager.getInstance(config);
}
 

@Override
public IStorageQuery createQuery(IRealization realization) {

    if (realization.getType() == RealizationType.CUBE) {

        CubeInstance cubeInstance = (CubeInstance) realization;
        String cubeStorageQuery;
        if (cubeInstance.getStorageType() == IStorageAware.ID_HBASE) {//v2 query engine cannot go with v1 storage now
            throw new IllegalStateException(
                    "Storage Engine (id=" + IStorageAware.ID_HBASE + ") is not supported any more");
        } else {
            cubeStorageQuery = v2CubeStorageQuery;//by default use v2
        }

        IStorageQuery ret;
        try {
            ret = (IStorageQuery) Class.forName(cubeStorageQuery).getConstructor(CubeInstance.class)
                    .newInstance((CubeInstance) realization);
        } catch (Exception e) {
            throw new RuntimeException("Failed to initialize storage query for " + cubeStorageQuery, e);
        }

        return ret;
    } else {
        throw new IllegalArgumentException("Unknown realization type " + realization.getType());
    }
}
 

public void removeRealization(final RealizationType type, final String realization) {
    Iterables.removeIf(this.realizationEntries, new Predicate<RealizationEntry>() {
        @Override
        public boolean apply(RealizationEntry input) {
            return input.getType() == type && input.getRealization().equalsIgnoreCase(realization);
        }
    });
}
 

@Override
public IStorageQuery createQuery(IRealization realization) {
    if (realization.getType() == RealizationType.CUBE) {
        CubeInstance cubeInstance = (CubeInstance) realization;
        return new StreamStorageQuery(cubeInstance, getStreamingDataSearchClient());
    } else {
        throw new IllegalArgumentException("Unknown realization type " + realization.getType());
    }
}
 

public List<RealizationEntry> getRealizationEntries(final RealizationType type) {
    if (type == null)
        return getRealizationEntries();

    return ImmutableList.copyOf(Iterables.filter(realizationEntries, new Predicate<RealizationEntry>() {
        @Override
        public boolean apply(@Nullable RealizationEntry input) {
            return input.getType() == type;
        }
    }));
}
 

public CubeDesc updateCubeAndDesc(CubeInstance cube, CubeDesc desc, String newProjectName, boolean forceUpdate)
        throws IOException {
    aclEvaluate.checkProjectWritePermission(cube);
    Message msg = MsgPicker.getMsg();

    final List<CubingJob> cubingJobs = jobService.listJobsByRealizationName(cube.getName(), null,
            EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
    if (!cubingJobs.isEmpty()) {
        throw new BadRequestException(String.format(Locale.ROOT, msg.getDISCARD_JOB_FIRST(), cube.getName()));
    }

    //double check again
    if (!forceUpdate && !cube.getDescriptor().consistentWith(desc)) {
        throw new BadRequestException(String.format(Locale.ROOT, msg.getINCONSISTENT_CUBE_DESC(), desc.getName()));
    }

    CubeDesc updatedCubeDesc = getCubeDescManager().updateCubeDesc(desc);
    int cuboidCount = CuboidCLI.simulateCuboidGeneration(updatedCubeDesc, false);
    logger.info("Updated cube " + cube.getName() + " has " + cuboidCount + " cuboids");

    ProjectManager projectManager = getProjectManager();
    if (!isCubeInProject(newProjectName, cube)) {
        String owner = SecurityContextHolder.getContext().getAuthentication().getName();
        ProjectInstance newProject = projectManager.moveRealizationToProject(RealizationType.CUBE, cube.getName(),
                newProjectName, owner);
    }

    return updatedCubeDesc;
}
 

public void removeRealization(final RealizationType type, final String realization) {
    Iterables.removeIf(this.realizationEntries, new Predicate<RealizationEntry>() {
        @Override
        public boolean apply(RealizationEntry input) {
            return input.getType() == type && input.getRealization().equalsIgnoreCase(realization);
        }
    });
}
 

public ProjectInstance moveRealizationToProject(RealizationType type, String realizationName, String newProjectName,
        String owner) throws IOException {
    try (AutoLock lock = prjMapLock.lockForWrite()) {
        removeRealizationsFromProjects(type, realizationName);
        return addRealizationToProject(type, realizationName, newProjectName, owner);
    }
}
 

public CubeInstance createCube(CubeInstance cube, String projectName, String owner) throws IOException {
    try (AutoLock lock = cubeMapLock.lockForWrite()) {
        logger.info("Creating cube '{}-->{}' from instance object. '", projectName, cube.getName());

        // save cube resource
        cube.setOwner(owner);
        updateCubeWithRetry(new CubeUpdate(cube), 0);

        ProjectManager.getInstance(config).moveRealizationToProject(RealizationType.CUBE, cube.getName(),
                projectName, owner);

        return cube;
    }
}
 

@Override
public void apply(List<IRealization> realizations, OLAPContext olapContext) {

    // sort cube candidates, 0) the cost indicator, 1) the lesser header
    // columns the better, 2) the lesser body columns the better
    List<Integer> items = super.findRealizationsOf(realizations, RealizationType.CUBE);
    PartialSorter.partialSort(realizations, items, new Comparator<IRealization>() {
        @Override
        public int compare(IRealization o1, IRealization o2) {
            CubeInstance c1 = (CubeInstance) o1;
            CubeInstance c2 = (CubeInstance) o2;
            int comp = 0;
            comp = c1.getCost() - c2.getCost();
            if (comp != 0) {
                return comp;
            }

            CubeDesc schema1 = c1.getDescriptor();
            CubeDesc schema2 = c2.getDescriptor();

            comp = schema1.listDimensionColumnsIncludingDerived().size() - schema2.listDimensionColumnsIncludingDerived().size();
            if (comp != 0)
                return comp;

            comp = schema1.getMeasures().size() - schema2.getMeasures().size();
            return comp;
        }
    });

}