org.apache.kylin.common.util.HBaseMetadataTestCase Java Examples

The following examples show how to use org.apache.kylin.common.util.HBaseMetadataTestCase. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BuildCubeWithEngineTest.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
@Before
public void before() throws Exception {
    HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);

    DeployUtil.initCliWorkDir();
    DeployUtil.deployMetadata();
    DeployUtil.overrideJobJarLocations();


    final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    jobService = ExecutableManager.getInstance(kylinConfig);
    scheduler = DefaultScheduler.getInstance();
    scheduler.init(new JobEngineConfig(kylinConfig));
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
    cubeManager = CubeManager.getInstance(kylinConfig);
    jobEngineConfig = new JobEngineConfig(kylinConfig);
    for (String jobId : jobService.getAllJobIds()) {
        if(jobService.getJob(jobId) instanceof CubingJob){
            jobService.deleteJob(jobId);
        }
    }

}
Example #2
Source File: SandboxMetastoreCLI.java    From kylin with Apache License 2.0 6 votes vote down vote up 
public static void main(String[] args) throws Exception {
    System.setProperty("HADOOP_USER_NAME", "root");
    logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    System.setProperty(KylinConfig.KYLIN_CONF, HBaseMetadataTestCase.SANDBOX_TEST_DATA);
    if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
        throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.4.0.0-169");
    }

    if (args.length < 1) {
        printUsage();
        return;
    }

    if ("reset".equalsIgnoreCase(args[0])) {
        ResourceTool.main(new String[] { "reset" });
    } else if ("download".equalsIgnoreCase(args[0])) {
        ResourceTool.main(new String[] { "download", args[1] });
    } else if ("fetch".equalsIgnoreCase(args[0])) {
        ResourceTool.main(new String[] { "fetch", args[1], args[2] });
    } else if ("upload".equalsIgnoreCase(args[0])) {
        ResourceTool.main(new String[] { "upload", args[1] });
    } else {
        printUsage();
    }
}
Example #3
Source File: KylinTestBase.java    From kylin with Apache License 2.0 6 votes vote down vote up 
protected static void setupAll() throws Exception {
    //setup env
    HBaseMetadataTestCase.staticCreateTestMetadata();
    config = KylinConfig.getInstanceFromEnv();

    //setup cube conn
    String project = ProjectInstance.DEFAULT_PROJECT_NAME;
    cubeConnection = QueryConnection.getConnection(project);

    //setup h2
    h2Connection = DriverManager.getConnection("jdbc:h2:mem:db" + (h2InstanceCount++) + ";CACHE_SIZE=32072", "sa",
            "");
    // Load H2 Tables (inner join)
    H2Database h2DB = new H2Database(h2Connection, config, project);
    h2DB.loadAllTables();
}
Example #4
Source File: KylinTestBase.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
protected static void setupAll() throws Exception {
    //setup env
    HBaseMetadataTestCase.staticCreateTestMetadata();
    config = KylinConfig.getInstanceFromEnv();

    //setup cube conn
    String project = ProjectInstance.DEFAULT_PROJECT_NAME;
    cubeConnection = QueryConnection.getConnection(project);

    //setup h2
    h2Connection = DriverManager.getConnection("jdbc:h2:mem:db" + (h2InstanceCount++) + ";CACHE_SIZE=32072", "sa",
            "");
    // Load H2 Tables (inner join)
    H2Database h2DB = new H2Database(h2Connection, config, project);
    h2DB.loadAllTables();
}
Example #5
Source File: SandboxMetastoreCLI.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
public static void main(String[] args) throws Exception {
    System.setProperty("HADOOP_USER_NAME", "root");
    logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    System.setProperty(KylinConfig.KYLIN_CONF, HBaseMetadataTestCase.SANDBOX_TEST_DATA);
    if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
        throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.4.0.0-169");
    }

    if (args.length < 1) {
        printUsage();
        return;
    }

    if ("reset".equalsIgnoreCase(args[0])) {
        ResourceTool.main(new String[] { "reset" });
    } else if ("download".equalsIgnoreCase(args[0])) {
        ResourceTool.main(new String[] { "download", args[1] });
    } else if ("fetch".equalsIgnoreCase(args[0])) {
        ResourceTool.main(new String[] { "fetch", args[1], args[2] });
    } else if ("upload".equalsIgnoreCase(args[0])) {
        ResourceTool.main(new String[] { "upload", args[1] });
    } else {
        printUsage();
    }
}
Example #6
Source File: BuildCubeWithStream.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public static void beforeClass() throws Exception {
    logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    System.setProperty(KylinConfig.KYLIN_CONF, HBaseMetadataTestCase.SANDBOX_TEST_DATA);
    if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
        throw new RuntimeException(
                "No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.4.0.0-169");
    }
    HBaseMetadataTestCase.staticCreateTestMetadata(HBaseMetadataTestCase.SANDBOX_TEST_DATA);
}
Example #7
Source File: KylinQueryTest.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
protected static void clean() {
    if (cubeConnection != null)
        closeConnection(cubeConnection);
    if (h2Connection != null)
        closeConnection(h2Connection);

    ObserverEnabler.forceCoprocessorUnset();
    HBaseMetadataTestCase.staticCleanupTestMetadata();
}
Example #8
Source File: BuildIIWithEngineTest.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
@Before
public void before() throws Exception {
    HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);

    DeployUtil.initCliWorkDir();
    //        DeployUtil.deployMetadata();
    DeployUtil.overrideJobJarLocations();

    final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    jobService = ExecutableManager.getInstance(kylinConfig);
    scheduler = DefaultScheduler.getInstance();
    scheduler.init(new JobEngineConfig(kylinConfig));
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
    iiManager = IIManager.getInstance(kylinConfig);
    jobEngineConfig = new JobEngineConfig(kylinConfig);
    for (String jobId : jobService.getAllJobIds()) {
        if(jobService.getJob(jobId) instanceof IIJob){
            jobService.deleteJob(jobId);
        }
    }

    IIInstance ii = iiManager.getII(TEST_II_NAME);
    if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
        ii.setStatus(RealizationStatusEnum.DISABLED);
        iiManager.updateII(ii);
    }
}
Example #9
Source File: BuildCubeWithEngineTest.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
@BeforeClass
public static void beforeClass() throws Exception {
    logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
    System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
}
Example #10
Source File: KylinTestBase.java    From kylin with Apache License 2.0 5 votes vote down vote up 
protected static void clean() {
    if (cubeConnection != null)
        closeConnection(cubeConnection);
    if (h2Connection != null)
        closeConnection(h2Connection);

    HBaseMetadataTestCase.staticCleanupTestMetadata();
    RemoveBlackoutRealizationsRule.blackList.clear();

}
Example #11
Source File: DeployLocalMetaToRemoteTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@BeforeClass
public static void beforeClass() throws Exception {
    logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    System.setProperty(KylinConfig.KYLIN_CONF, HBaseMetadataTestCase.SANDBOX_TEST_DATA);
    if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
        throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.4.0.0-169");
    }
}
Example #12
Source File: DeployLocalMetaToRemoteTest.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Before
public void before() throws Exception {
    HBaseMetadataTestCase.staticCreateTestMetadata(HBaseMetadataTestCase.SANDBOX_TEST_DATA);

    DeployUtil.initCliWorkDir();
    DeployUtil.deployMetadata();
    DeployUtil.overrideJobJarLocations();

}
Example #13
Source File: DeployLocalMetaToRemoteTest.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@BeforeClass
public static void beforeClass() throws Exception {
    logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    System.setProperty(KylinConfig.KYLIN_CONF, HBaseMetadataTestCase.SANDBOX_TEST_DATA);
    if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
        throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.4.0.0-169");
    }
}
Example #14
Source File: DeployLocalMetaToRemoteTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Before
public void before() throws Exception {
    HBaseMetadataTestCase.staticCreateTestMetadata(HBaseMetadataTestCase.SANDBOX_TEST_DATA);

    DeployUtil.initCliWorkDir();
    DeployUtil.deployMetadata();
    DeployUtil.overrideJobJarLocations();

}
Example #15
Source File: KylinTestBase.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
protected static void clean() {
    if (cubeConnection != null)
        closeConnection(cubeConnection);
    if (h2Connection != null)
        closeConnection(h2Connection);

    HBaseMetadataTestCase.staticCleanupTestMetadata();
    RemoveBlackoutRealizationsRule.blackList.clear();

}
Example #16
Source File: BuildCubeWithStream.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public static void beforeClass() throws Exception {
    logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    System.setProperty(KylinConfig.KYLIN_CONF, HBaseMetadataTestCase.SANDBOX_TEST_DATA);
    if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
        throw new RuntimeException(
                "No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.4.0.0-169");
    }
    HBaseMetadataTestCase.staticCreateTestMetadata(HBaseMetadataTestCase.SANDBOX_TEST_DATA);
}
Example #17
Source File: BuildCubeWithStreamV2.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public static void beforeClass() throws Exception {
    beforeClass(HBaseMetadataTestCase.SANDBOX_TEST_DATA);
}
Example #18
Source File: KylinQueryTest.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
private static void setUpEnv() throws Exception {
    HBaseMetadataTestCase.staticCreateTestMetadata();

    config = KylinConfig.getInstanceFromEnv();
}
Example #19
Source File: DeployLocalMetaToRemoteTest.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
@After
public void after() {
    HBaseMetadataTestCase.staticCleanupTestMetadata();
}
Example #20
Source File: BuildIIWithEngineTest.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
@BeforeClass
public static void beforeClass() throws Exception {
    logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
}
Example #21
Source File: BuildCubeWithEngineTest.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
@After
public void after() {
    HBaseMetadataTestCase.staticCleanupTestMetadata();
}
Example #22
Source File: BuildCubeWithStream.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public void cleanup() throws Exception {
    cleanupOldStorage();
    HBaseMetadataTestCase.staticCleanupTestMetadata();
}
Example #23
Source File: BuildCubeWithEngine.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public static void beforeClass() throws Exception {
    beforeClass(HBaseMetadataTestCase.SANDBOX_TEST_DATA);
}
Example #24
Source File: MultipleDictionaryValueEnumeratorTest.java    From kylin with Apache License 2.0 4 votes vote down vote up 
@BeforeClass
public static void beforeClass() throws Exception {
    ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    System.setProperty(KylinConfig.KYLIN_CONF, HBaseMetadataTestCase.SANDBOX_TEST_DATA);
}
Example #25
Source File: BuildCubeWithStreamV2.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public void cleanup() throws Exception {
    cleanupOldStorage();
    HBaseMetadataTestCase.staticCleanupTestMetadata();
}
Example #26
Source File: BuildCubeWithStreamV2.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public static void afterClass() {
    HBaseMetadataTestCase.staticCleanupTestMetadata();
}
Example #27
Source File: MultipleDictionaryValueEnumeratorTest.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
@BeforeClass
public static void beforeClass() throws Exception {
    ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    System.setProperty(KylinConfig.KYLIN_CONF, HBaseMetadataTestCase.SANDBOX_TEST_DATA);
}
Example #28
Source File: BuildCubeWithEngine.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public static void beforeClass(String confDir) throws Exception {
    logger.info("Adding to classpath: " + new File(confDir).getAbsolutePath());
    ClassUtil.addClasspath(new File(confDir).getAbsolutePath());

    fastBuildMode = isFastBuildMode();
    simpleBuildMode = isSimpleBuildMode();
    if (fastBuildMode) {
        logger.info("Will use fast build mode");
    }
    if (simpleBuildMode) {
        logger.info("Will use simple build mode");
    }

    String specifiedEngineType = System.getProperty("engineType");
    if (StringUtils.isNotEmpty(specifiedEngineType)) {
        engineType = Integer.parseInt(specifiedEngineType);
    }

    System.setProperty(KylinConfig.KYLIN_CONF, confDir);
    System.setProperty("SPARK_HOME", "/usr/local/spark"); // need manually create and put spark to this folder on Jenkins
    System.setProperty("kylin.hadoop.conf.dir", confDir);
    if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
        throw new RuntimeException(
                "No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.4.0.0-169");
    }

    HBaseMetadataTestCase.staticCreateTestMetadata(confDir);

    try {
        //check hdfs permission
        FileSystem fileSystem = HadoopUtil.getWorkingFileSystem();
        String hdfsWorkingDirectory = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory();
        Path coprocessorDir = new Path(hdfsWorkingDirectory);
        boolean success = fileSystem.mkdirs(coprocessorDir);
        if (!success) {
            throw new IOException("mkdir fails");
        }
    } catch (IOException e) {
        throw new RuntimeException(
                "failed to create kylin.env.hdfs-working-dir, Please make sure the user has right to access "
                        + KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory(),
                e);
    }
}
Example #29
Source File: BuildCubeWithEngine.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public static void afterClass() {
    HBaseMetadataTestCase.staticCleanupTestMetadata();
}
Example #30
Source File: BuildCubeWithEngine.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public static void afterClass() {
    HBaseMetadataTestCase.staticCleanupTestMetadata();
}