org.apache.kylin.cube.CubeManager#getInstance

Source File: ColumnarSplitReader.java From kylin with Apache License 2.0

6 votes

@Override
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
    if (!(split instanceof FileSplit)) {
        throw new IllegalArgumentException("Only compatible with FileSplits.");
    } else {
        logger.debug("CFG_Cube_Name: " + BatchConstants.CFG_CUBE_NAME);
        cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase(Locale.ROOT);
        segmentName = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_NAME).toUpperCase(Locale.ROOT);

        KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();
        CubeManager cubeManager = CubeManager.getInstance(config);
        cube = cubeManager.getCube(cubeName);
        cubeDesc = cube.getDescriptor();
        cubeSegment = cube.getSegment(segmentName, SegmentStatusEnum.NEW);
    }
}

Source File: JobControllerTest.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

@Before
public void setup() throws Exception {
    super.setup();

    jobSchedulerController = new JobController();
    jobSchedulerController.setJobService(jobService);
    cubeController = new CubeController();
    cubeController.setJobService(jobService);
    cubeController.setCubeService(cubeService);

    KylinConfig testConfig = getTestConfig();
    cubeManager = CubeManager.getInstance(testConfig);
    cubeDescManager = CubeDescManager.getInstance(testConfig);
    executableDAO = ExecutableDao.getInstance(testConfig);

}

Source File: DeployCoprocessorCLI.java From kylin with Apache License 2.0

6 votes

private static List<String> filterByCubes(List<String> allTableNames, List<String> cubeNames) {
    CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
    List<String> result = Lists.newArrayList();
    for (String c : cubeNames) {
        c = c.trim();
        if (c.endsWith(","))
            c = c.substring(0, c.length() - 1);

        CubeInstance cubeInstance = cubeManager.getCube(c);
        for (CubeSegment segment : cubeInstance.getSegments()) {
            String tableName = segment.getStorageLocationIdentifier();
            if (allTableNames.contains(tableName)) {
                result.add(tableName);
            }
        }
    }
    return result;
}

Source File: SCCreatorTest.java From kylin with Apache License 2.0

6 votes

@Test
public void testExecute() throws Exception {
    String metadataPath = tempMetadataDir.getPath();

    String inputPath = "src/main/resources/SCSinkTools.json";

    SCCreator cli = new SCCreator();
    cli.execute("ADMIN", metadataPath, inputPath);
    Assert.assertTrue(tempMetadataDir.isDirectory());

    KylinConfig local = KylinConfig.createKylinConfig(KylinConfig.getInstanceFromEnv());
    local.setMetadataUrl(metadataPath);

    CubeManager cubeManager = CubeManager.getInstance(local);
    List<CubeInstance> cubeList = cubeManager.listAllCubes();
    System.out.println("System cubes: " + cubeList);
    assertEquals(cubeList.size(), 10);

    for (CubeInstance cube : cubeList) {
        Assert.assertTrue(cube.getStatus() != RealizationStatusEnum.DESCBROKEN);
    }
}

Source File: CubeStorageEngine.java From Kylin with Apache License 2.0

6 votes

private TupleFilter translateDerivedInCompare(CompareTupleFilter compf, Set<TblColRef> collector) {
    if (compf.getColumn() == null || compf.getValues().isEmpty())
        return compf;

    TblColRef derived = compf.getColumn();
    if (cubeDesc.isDerived(derived) == false)
        return compf;

    DeriveInfo hostInfo = cubeDesc.getHostInfo(derived);
    CubeManager cubeMgr = CubeManager.getInstance(this.cubeInstance.getConfig());
    CubeSegment seg = cubeInstance.getLatestReadySegment();
    LookupStringTable lookup = cubeMgr.getLookupTable(seg, hostInfo.dimension);
    Pair<TupleFilter, Boolean> translated = DerivedFilterTranslator.translate(lookup, hostInfo, compf);
    TupleFilter translatedFilter = translated.getFirst();
    boolean loosened = translated.getSecond();
    if (loosened) {
        collectColumnsRecursively(compf, collector);
    }
    return translatedFilter;
}

Source File: CubeSizeEstimationCLI.java From Kylin with Apache License 2.0

6 votes

public static long estimatedCubeSize(String cubeName, long[] cardinality) {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    CubeManager cubeManager = CubeManager.getInstance(config);
    CubeInstance cubeInstance = cubeManager.getCube(cubeName);
    CubeDesc cubeDesc = cubeInstance.getDescriptor();

    CuboidScheduler scheduler = new CuboidScheduler(cubeDesc);
    long baseCuboid = Cuboid.getBaseCuboidId(cubeDesc);
    LinkedList<Long> cuboidQueue = new LinkedList<Long>();
    cuboidQueue.push(baseCuboid);

    long totalSpace = 0;

    while (!cuboidQueue.isEmpty()) {
        long cuboidID = cuboidQueue.pop();
        Collection<Long> spanningCuboid = scheduler.getSpanningCuboid(cuboidID);
        for (Long sc : spanningCuboid) {
            cuboidQueue.push(sc);
        }

        totalSpace += estimateCuboidSpace(cuboidID, cardinality, cubeDesc);
    }
    return totalSpace;
}

Source File: UpdateHTableHostCLI.java From kylin with Apache License 2.0

6 votes

private static List<String> getHTableNames(KylinConfig config) {
    CubeManager cubeMgr = CubeManager.getInstance(config);

    ArrayList<String> result = new ArrayList<>();
    for (CubeInstance cube : cubeMgr.listAllCubes()) {
        for (CubeSegment seg : cube.getSegments(SegmentStatusEnum.READY)) {
            String tableName = seg.getStorageLocationIdentifier();
            if (!StringUtils.isBlank(tableName)) {
                result.add(tableName);
                logger.info("added new table: {}", tableName);
            }
        }
    }

    return result;
}

Source File: MergeCuboidMapper.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

@Override
protected void doSetup(Context context) throws IOException, InterruptedException {
    super.bindCurrentConfiguration(context.getConfiguration());

    String cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME);
    String segmentID = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_ID);

    KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();

    CubeManager cubeManager = CubeManager.getInstance(config);
    CubeInstance cube = cubeManager.getCube(cubeName);
    CubeDesc cubeDesc = cube.getDescriptor();
    CubeSegment mergedCubeSegment = cube.getSegmentById(segmentID);

    // decide which source segment
    FileSplit fileSplit = (FileSplit) context.getInputSplit();
    IMROutput2.IMRMergeOutputFormat outputFormat = MRUtil.getBatchMergeOutputSide2(mergedCubeSegment)
            .getOutputFormat();
    CubeSegment sourceCubeSegment = outputFormat.findSourceSegment(fileSplit, cube);
    reEncoder = new SegmentReEncoder(cubeDesc, sourceCubeSegment, mergedCubeSegment, config);
}

Source File: UpdateHTableHostCLI.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

private static List<String> filterByCubes(List<String> allTableNames, List<String> cubeNames) {
    CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
    List<String> result = Lists.newArrayList();
    for (String c : cubeNames) {
        c = c.trim();
        if (c.endsWith(","))
            c = c.substring(0, c.length() - 1);

        CubeInstance cubeInstance = cubeManager.getCube(c);
        for (CubeSegment segment : cubeInstance.getSegments()) {
            String tableName = segment.getStorageLocationIdentifier();
            if (allTableNames.contains(tableName)) {
                result.add(tableName);
            }
        }
    }
    return result;
}

Source File: KylinLogExtractor.java From kylin with Apache License 2.0

5 votes

private void beforeExtract() {
    // reload metadata before extract diagnosis info
    logger.info("Start to reload metadata from diagnosis.");
    
    config.clearManagers();

    CubeManager.getInstance(config);
    CubeDescManager.getInstance(config);
    DataModelManager.getInstance(config);
    ProjectManager.getInstance(config);
}

Source File: CubeMigrationCheckCLI.java From kylin with Apache License 2.0

5 votes

public void checkAll() {
    List<String> segFullNameList = Lists.newArrayList();

    CubeManager cubeMgr = CubeManager.getInstance(dstCfg);
    for (CubeInstance cube : cubeMgr.listAllCubes()) {
        addHTableNamesForCube(cube, segFullNameList);
    }

    check(segFullNameList);
}

Source File: ExtendCubeToHybridCLI.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

private void verify() {
    kylinConfig.clearManagers();
    
    CubeDescManager.getInstance(kylinConfig);
    CubeManager.getInstance(kylinConfig);
    ProjectManager.getInstance(kylinConfig);
    HybridManager.getInstance(kylinConfig);
}

Source File: ProjectManagerTest.java From Kylin with Apache License 2.0

5 votes

@Test
public void testExistingProject() throws Exception {
    ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
    CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
    CubeDescManager cubeDescMgr = CubeDescManager.getInstance(getTestConfig());
    
    int originalProjectCount = prjMgr.listAllProjects().size();
    int originalCubeCount = cubeMgr.listAllCubes().size();
    ResourceStore store = getStore();

    // clean legacy in case last run failed
    store.deleteResource("/cube/new_cube_in_default.json");

    CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc");
    CubeInstance createdCube = cubeMgr.createCube("new_cube_in_default", ProjectInstance.DEFAULT_PROJECT_NAME, desc, null);
    assertTrue(createdCube == cubeMgr.getCube("new_cube_in_default"));

    System.out.println(JsonUtil.writeValueAsIndentString(createdCube));

    assertTrue(prjMgr.listAllProjects().size() == originalProjectCount);
    assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount + 1);

    CubeInstance droppedCube = cubeMgr.dropCube("new_cube_in_default", true);

    assertTrue(createdCube == droppedCube);
    assertNull(cubeMgr.getCube("new_cube_in_default"));
    assertTrue(prjMgr.listAllProjects().size() == originalProjectCount);
    assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount);
}

Source File: HybridCubeCLI.java From kylin with Apache License 2.0

5 votes

public HybridCubeCLI() {
    options = new Options();
    options.addOption(OPTION_ACTION);
    options.addOption(OPTION_HYBRID_NAME);
    options.addOption(OPTION_PROJECT);
    options.addOption(OPTION_MODEL);
    options.addOption(OPTION_CUBES);
    options.addOption(OPTION_CHECK);

    this.kylinConfig = KylinConfig.getInstanceFromEnv();
    this.store = ResourceStore.getStore(kylinConfig);
    this.cubeManager = CubeManager.getInstance(kylinConfig);
    this.hybridManager = HybridManager.getInstance(kylinConfig);
    this.metadataManager = DataModelManager.getInstance(kylinConfig);
}

Source File: UHCDictionaryJob.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

@Override
public int run(String[] args) throws Exception {
    Options options = new Options();

    try {
        options.addOption(OPTION_JOB_NAME);
        options.addOption(OPTION_CUBE_NAME);
        options.addOption(OPTION_CUBING_JOB_ID);
        options.addOption(OPTION_OUTPUT_PATH);
        options.addOption(OPTION_INPUT_PATH);
        parseOptions(options, args);

        job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME));
        String job_id = getOptionValue(OPTION_CUBING_JOB_ID);
        String cubeName = getOptionValue(OPTION_CUBE_NAME);
        Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
        Path input = new Path(getOptionValue(OPTION_INPUT_PATH));

        //add metadata to distributed cache
        CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
        CubeInstance cube = cubeMgr.getCube(cubeName);
        attachCubeMetadata(cube, job.getConfiguration());

        List<TblColRef> uhcColumns = cube.getDescriptor().getAllUHCColumns();
        int reducerCount = uhcColumns.size();

        //Note! handle uhc columns is null.
        boolean hasUHCValue = false;
        for (TblColRef tblColRef : uhcColumns) {
            Path path = new Path(input.toString() + "/" + tblColRef.getIdentity());
            if (HadoopUtil.getFileSystem(path).exists(path)) {
                FileInputFormat.addInputPath(job, path);
                FileInputFormat.setInputPathFilter(job, UHCDictPathFilter.class);
                hasUHCValue = true;
            }
        }

        if (!hasUHCValue) {
            isSkipped = true;
            return 0;
        }

        setJobClasspath(job, cube.getConfig());
        setupMapper();
        setupReducer(output, reducerCount);

        job.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
        job.getConfiguration().set(BatchConstants.ARG_CUBING_JOB_ID, job_id);
        job.getConfiguration().set(BatchConstants.CFG_GLOBAL_DICT_BASE_DIR, KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory());
        job.getConfiguration().set(BatchConstants.CFG_MAPRED_OUTPUT_COMPRESS, "false");

        //8G memory is enough for all global dict, because the input is sequential and we handle global dict slice by slice
        job.getConfiguration().set("mapreduce.reduce.memory.mb", "8500");
        job.getConfiguration().set("mapred.reduce.child.java.opts", "-Xmx8g");
        //Copying global dict to working dir in GlobalDictHDFSStore maybe elapsed a long time (Maybe we could improve it)
        //Waiting the global dict lock maybe also take a long time.
        //So we set 8 hours here
        job.getConfiguration().set("mapreduce.task.timeout", "28800000");

        //allow user specially set config for uhc step
        for (Map.Entry<String, String> entry : cube.getConfig().getUHCMRConfigOverride().entrySet()) {
            job.getConfiguration().set(entry.getKey(), entry.getValue());
        }

        return waitForCompletion(job);
    } finally {
        if (job != null)
            cleanupTempConfFile(job.getConfiguration());
    }
}

Source File: ColumnarSegmentStoreTest.java From kylin with Apache License 2.0

4 votes

public CubeManager getCubeManager() {
    return CubeManager.getInstance(getTestConfig());
}

Source File: UHCDictionaryJob.java From kylin with Apache License 2.0

4 votes

@Override
public int run(String[] args) throws Exception {
    Options options = new Options();

    try {
        options.addOption(OPTION_JOB_NAME);
        options.addOption(OPTION_CUBE_NAME);
        options.addOption(OPTION_CUBING_JOB_ID);
        options.addOption(OPTION_OUTPUT_PATH);
        options.addOption(OPTION_INPUT_PATH);
        parseOptions(options, args);

        job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME));
        String job_id = getOptionValue(OPTION_CUBING_JOB_ID);
        String cubeName = getOptionValue(OPTION_CUBE_NAME);
        Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
        Path input = new Path(getOptionValue(OPTION_INPUT_PATH));

        //add metadata to distributed cache
        CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
        CubeInstance cube = cubeMgr.getCube(cubeName);
        attachCubeMetadata(cube, job.getConfiguration());

        List<TblColRef> uhcColumns = cube.getDescriptor().getAllUHCColumns();
        int reducerCount = uhcColumns.size();

        //Note! handle uhc columns is null.
        boolean hasUHCValue = false;
        for (TblColRef tblColRef : uhcColumns) {
            Path path = new Path(input.toString() + "/" + tblColRef.getIdentity());
            if (HadoopUtil.getFileSystem(path).exists(path)) {
                FileInputFormat.addInputPath(job, path);
                FileInputFormat.setInputPathFilter(job, UHCDictPathFilter.class);
                hasUHCValue = true;
            }
        }

        if (!hasUHCValue) {
            isSkipped = true;
            return 0;
        }

        setJobClasspath(job, cube.getConfig());
        setupMapper();
        setupReducer(output, reducerCount);

        job.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
        job.getConfiguration().set(BatchConstants.ARG_CUBING_JOB_ID, job_id);
        job.getConfiguration().set(BatchConstants.CFG_GLOBAL_DICT_BASE_DIR, KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory());
        job.getConfiguration().set(BatchConstants.CFG_MAPRED_OUTPUT_COMPRESS, "false");

        //8G memory is enough for all global dict, because the input is sequential and we handle global dict slice by slice
        job.getConfiguration().set("mapreduce.reduce.memory.mb", "8500");
        job.getConfiguration().set("mapred.reduce.child.java.opts", "-Xmx8g");
        //Copying global dict to working dir in GlobalDictHDFSStore maybe elapsed a long time (Maybe we could improve it)
        //Waiting the global dict lock maybe also take a long time.
        //So we set 8 hours here
        job.getConfiguration().set("mapreduce.task.timeout", "28800000");

        //allow user specially set config for uhc step
        for (Map.Entry<String, String> entry : cube.getConfig().getUHCMRConfigOverride().entrySet()) {
            job.getConfiguration().set(entry.getKey(), entry.getValue());
        }

        return waitForCompletion(job);
    } finally {
        if (job != null)
            cleanupTempConfFile(job.getConfiguration());
    }
}

Source File: CacheServiceTest.java From kylin with Apache License 2.0

4 votes

private static CubeManager getCubeManager(KylinConfig config) throws Exception {
    return CubeManager.getInstance(config);
}

Source File: BasicService.java From kylin with Apache License 2.0

4 votes

public CubeManager getCubeManager() {
    return CubeManager.getInstance(getConfig());
}

Source File: StreamingCoordinator.java From kylin with Apache License 2.0

4 votes

public CubeManager getCubeManager() {
    return CubeManager.getInstance(getConfig());
}

Java Code Examples for org.apache.kylin.cube.CubeManager#getInstance()