Java Code Examples for org.apache.kylin.job.JoinedFlatTable#generateDropTableStatement()

The following examples show how to use org.apache.kylin.job.JoinedFlatTable#generateDropTableStatement() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: HiveInputBase.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
protected static AbstractExecutable createFlatHiveTableStep(String hiveInitStatements, String jobWorkingDir,
        String cubeName, IJoinedFlatTableDesc flatDesc) {
    //from hive to hive
    final String dropTableHql = JoinedFlatTable.generateDropTableStatement(flatDesc);
    final String createTableHql = JoinedFlatTable.generateCreateTableStatement(flatDesc, jobWorkingDir);
    String insertDataHqls = JoinedFlatTable.generateInsertDataStatement(flatDesc);

    CreateFlatHiveTableStep step = new CreateFlatHiveTableStep();
    step.setInitStatement(hiveInitStatements);
    step.setCreateTableStatement(dropTableHql + createTableHql + insertDataHqls);
    CubingExecutableUtil.setCubeName(cubeName, step.getParams());
    step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE);
    return step;
}
Example 2
Source File: HiveInputBase.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
protected static AbstractExecutable createFlatHiveTableByLivyStep(String hiveInitStatements, String jobWorkingDir,
        String cubeName, IJoinedFlatTableDesc flatDesc) {
    //from hive to hive
    final String dropTableHql = JoinedFlatTable.generateDropTableStatement(flatDesc);
    final String createTableHql = JoinedFlatTable.generateCreateTableStatement(flatDesc, jobWorkingDir);
    String insertDataHqls = JoinedFlatTable.generateInsertDataStatement(flatDesc);

    CreateFlatHiveTableByLivyStep step = new CreateFlatHiveTableByLivyStep();
    step.setInitStatement(hiveInitStatements);
    step.setCreateTableStatement(dropTableHql + createTableHql + insertDataHqls);
    CubingExecutableUtil.setCubeName(cubeName, step.getParams());
    step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE);
    return step;
}
Example 3
Source File: JdbcHiveInputBase.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private AbstractExecutable createFlatHiveTableFromFiles(String hiveInitStatements, String jobWorkingDir) {
    final String dropTableHql = JoinedFlatTable.generateDropTableStatement(flatDesc);
    String filedDelimiter = getConfig().getJdbcSourceFieldDelimiter();
    // Sqoop does not support exporting SEQUENSEFILE to Hive now SQOOP-869
    final String createTableHql = JoinedFlatTable.generateCreateTableStatement(flatDesc, jobWorkingDir,
            "TEXTFILE", filedDelimiter);

    HiveCmdStep step = new HiveCmdStep();
    step.setCmd(hiveInitStatements + dropTableHql + createTableHql);
    step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE);
    return step;
}
Example 4
Source File: KafkaInputBase.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
protected static AbstractExecutable createFlatTable(final String hiveTableDatabase,
                                                    final String baseLocation, final String cubeName,
                                                    final StreamCubeFactTableDesc streamFactDesc, final List<String> intermediateTables,
                                                    final List<String> intermediatePaths) {
    final IJoinedFlatTableDesc flatDesc = streamFactDesc.getFlatTableDesc();

    final String hiveInitStatements = JoinedFlatTable.generateHiveInitStatements(hiveTableDatabase);

    final String dropFactTableHql = JoinedFlatTable.generateDropTableStatement(streamFactDesc);
    // the table inputformat is sequence file
    final String createFactTableHql = JoinedFlatTable.generateCreateTableStatement(streamFactDesc, baseLocation,
            JoinedFlatTable.SEQUENCEFILE);

    final String dropTableHql = JoinedFlatTable.generateDropTableStatement(flatDesc);
    final String createTableHql = JoinedFlatTable.generateCreateTableStatement(flatDesc, baseLocation);
    String insertDataHqls = JoinedFlatTable.generateInsertDataStatement(flatDesc);
    insertDataHqls = insertDataHqls.replace(
            quoteTableIdentity(flatDesc.getDataModel().getRootFactTable(), null) + " ",
            quoteTableIdentity(hiveTableDatabase, streamFactDesc.getTableName(), null) + " ");

    CreateFlatHiveTableStep step = new CreateFlatHiveTableStep();
    CubingExecutableUtil.setCubeName(cubeName, step.getParams());
    step.setInitStatement(hiveInitStatements);
    step.setCreateTableStatement(
            dropFactTableHql + createFactTableHql + dropTableHql + createTableHql + insertDataHqls);
    step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE);

    intermediateTables.add(flatDesc.getTableName());
    intermediateTables.add(streamFactDesc.getTableName());
    intermediatePaths.add(baseLocation + "/" + flatDesc.getTableName());
    intermediatePaths.add(baseLocation + "/" + streamFactDesc.getTableName());
    return step;
}
Example 5
Source File: HiveInputBase.java    From kylin with Apache License 2.0 5 votes vote down vote up 
protected static AbstractExecutable createFlatHiveTableStep(String hiveInitStatements, String jobWorkingDir,
                                                            String cubeName, IJoinedFlatTableDesc flatDesc) {
    //from hive to hive
    final String dropTableHql = JoinedFlatTable.generateDropTableStatement(flatDesc);
    final String createTableHql = JoinedFlatTable.generateCreateTableStatement(flatDesc, jobWorkingDir);
    String insertDataHqls = JoinedFlatTable.generateInsertDataStatement(flatDesc);

    CreateFlatHiveTableStep step = new CreateFlatHiveTableStep();
    step.setInitStatement(hiveInitStatements);
    step.setCreateTableStatement(dropTableHql + createTableHql + insertDataHqls);
    CubingExecutableUtil.setCubeName(cubeName, step.getParams());
    step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE);
    return step;
}
Example 6
Source File: HiveInputBase.java    From kylin with Apache License 2.0 5 votes vote down vote up 
protected static AbstractExecutable createFlatHiveTableByLivyStep(String hiveInitStatements, String jobWorkingDir,
                                                                  String cubeName, IJoinedFlatTableDesc flatDesc) {
    //from hive to hive
    final String dropTableHql = JoinedFlatTable.generateDropTableStatement(flatDesc);
    final String createTableHql = JoinedFlatTable.generateCreateTableStatement(flatDesc, jobWorkingDir);
    String insertDataHqls = JoinedFlatTable.generateInsertDataStatement(flatDesc);

    CreateFlatHiveTableByLivyStep step = new CreateFlatHiveTableByLivyStep();
    step.setInitStatement(hiveInitStatements);
    step.setCreateTableStatement(dropTableHql + createTableHql + insertDataHqls);
    CubingExecutableUtil.setCubeName(cubeName, step.getParams());
    step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE);
    return step;
}
Example 7
Source File: JdbcHiveInputBase.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private AbstractExecutable createFlatHiveTableFromFiles(String hiveInitStatements, String jobWorkingDir) {
    final String dropTableHql = JoinedFlatTable.generateDropTableStatement(flatDesc);
    String filedDelimiter = getConfig().getJdbcSourceFieldDelimiter();
    // Sqoop does not support exporting SEQUENSEFILE to Hive now SQOOP-869
    final String createTableHql = JoinedFlatTable.generateCreateTableStatement(flatDesc, jobWorkingDir,
            "TEXTFILE", filedDelimiter);

    HiveCmdStep step = new HiveCmdStep();
    step.setCmd(hiveInitStatements + dropTableHql + createTableHql);
    step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE);
    return step;
}
Example 8
Source File: KafkaInputBase.java    From kylin with Apache License 2.0 5 votes vote down vote up 
protected static AbstractExecutable createFlatTable(final String hiveTableDatabase,
                                                    final String baseLocation, final String cubeName,
                                                    final StreamCubeFactTableDesc streamFactDesc, final List<String> intermediateTables,
                                                    final List<String> intermediatePaths) {
    final IJoinedFlatTableDesc flatDesc = streamFactDesc.getFlatTableDesc();

    final String hiveInitStatements = JoinedFlatTable.generateHiveInitStatements(hiveTableDatabase);

    final String dropFactTableHql = JoinedFlatTable.generateDropTableStatement(streamFactDesc);
    // the table inputformat is sequence file
    final String createFactTableHql = JoinedFlatTable.generateCreateTableStatement(streamFactDesc, baseLocation,
            JoinedFlatTable.SEQUENCEFILE);

    final String dropTableHql = JoinedFlatTable.generateDropTableStatement(flatDesc);
    final String createTableHql = JoinedFlatTable.generateCreateTableStatement(flatDesc, baseLocation);
    String insertDataHqls = JoinedFlatTable.generateInsertDataStatement(flatDesc);
    insertDataHqls = insertDataHqls.replace(
            quoteTableIdentity(flatDesc.getDataModel().getRootFactTable(), null) + " ",
            quoteTableIdentity(hiveTableDatabase, streamFactDesc.getTableName(), null) + " ");

    CreateFlatHiveTableStep step = new CreateFlatHiveTableStep();
    CubingExecutableUtil.setCubeName(cubeName, step.getParams());
    step.setInitStatement(hiveInitStatements);
    step.setCreateTableStatement(
            dropFactTableHql + createFactTableHql + dropTableHql + createTableHql + insertDataHqls);
    step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE);

    intermediateTables.add(flatDesc.getTableName());
    intermediateTables.add(streamFactDesc.getTableName());
    intermediatePaths.add(baseLocation + "/" + flatDesc.getTableName());
    intermediatePaths.add(baseLocation + "/" + streamFactDesc.getTableName());
    return step;
}
Example 9
Source File: HiveInputBase.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
protected static AbstractExecutable createFlatHiveTableBySparkSql(String hiveInitStatements,
        String jobWorkingDir, String cubeName, IJoinedFlatTableDesc flatDesc) {
    final String dropTableHql = JoinedFlatTable.generateDropTableStatement(flatDesc);
    final String createTableHql = JoinedFlatTable.generateCreateTableStatement(flatDesc,
            jobWorkingDir);
    String insertDataHqls = JoinedFlatTable.generateInsertDataStatement(flatDesc);

    KylinConfig config = flatDesc.getSegment().getConfig();
    final SparkExecutable sparkExecutable = SparkExecutableFactory.instance(config);
    sparkExecutable.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_TABLE_WITH_SPARK);
    sparkExecutable.setClassName(SparkCreatingFlatTable.class.getName());

    sparkExecutable.setParam(SparkSqlBatch.OPTION_CUBE_NAME.getOpt(), cubeName);
    sparkExecutable.setParam(SparkSqlBatch.OPTION_STEP_NAME.getOpt(),
            base64EncodeStr(ExecutableConstants.STEP_NAME_CREATE_FLAT_TABLE_WITH_SPARK));
    sparkExecutable.setParam(SparkSqlBatch.OPTION_SEGMENT_ID.getOpt(),
            flatDesc.getSegment().getName());
    sparkExecutable.setParam(SparkSqlBatch.OPTION_SQL_COUNT.getOpt(),
            String.valueOf(SparkCreatingFlatTable.SQL_COUNT));

    sparkExecutable.setParam(SparkCreatingFlatTable.getSqlOption(0).getOpt(),
            base64EncodeStr(hiveInitStatements));
    sparkExecutable.setParam(SparkCreatingFlatTable.getSqlOption(1).getOpt(),
            base64EncodeStr(dropTableHql));

    // createTableHql include create table sql and alter table sql
    String[] sqlArr = createTableHql.trim().split(";");
    if (2 != sqlArr.length) {
        throw new RuntimeException("create table hql should combined by a create table sql " +
                "and a alter sql, but got: " + createTableHql);
    }
    sparkExecutable.setParam(SparkCreatingFlatTable.getSqlOption(2).getOpt(),
            base64EncodeStr(sqlArr[0]));
    sparkExecutable.setParam(SparkCreatingFlatTable.getSqlOption(3).getOpt(),
            base64EncodeStr(sqlArr[1]));

    sparkExecutable.setParam(SparkCreatingFlatTable.getSqlOption(4).getOpt(),
            base64EncodeStr(insertDataHqls));

    StringBuilder jars = new StringBuilder();
    StringUtil.appendWithSeparator(jars, config.getSparkAdditionalJars());
    sparkExecutable.setJars(jars.toString());

    return sparkExecutable;
}
Example 10
Source File: HiveInputBase.java    From kylin with Apache License 2.0 4 votes vote down vote up 
protected static AbstractExecutable createFlatHiveTableBySparkSql(String hiveInitStatements,
                                                                  String jobWorkingDir, String cubeName, IJoinedFlatTableDesc flatDesc) {
    final String dropTableHql = JoinedFlatTable.generateDropTableStatement(flatDesc);
    final String createTableHql = JoinedFlatTable.generateCreateTableStatement(flatDesc,
            jobWorkingDir);
    String insertDataHqls = JoinedFlatTable.generateInsertDataStatement(flatDesc);

    KylinConfig config = flatDesc.getSegment().getConfig();
    final SparkExecutable sparkExecutable = SparkExecutableFactory.instance(config);
    sparkExecutable.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_TABLE_WITH_SPARK);
    sparkExecutable.setClassName(SparkCreatingFlatTable.class.getName());

    sparkExecutable.setParam(SparkSqlBatch.OPTION_CUBE_NAME.getOpt(), cubeName);
    sparkExecutable.setParam(SparkSqlBatch.OPTION_STEP_NAME.getOpt(),
            base64EncodeStr(ExecutableConstants.STEP_NAME_CREATE_FLAT_TABLE_WITH_SPARK));
    sparkExecutable.setParam(SparkSqlBatch.OPTION_SEGMENT_ID.getOpt(),
            flatDesc.getSegment().getName());
    sparkExecutable.setParam(SparkSqlBatch.OPTION_SQL_COUNT.getOpt(),
            String.valueOf(SparkCreatingFlatTable.SQL_COUNT));

    sparkExecutable.setParam(SparkCreatingFlatTable.getSqlOption(0).getOpt(),
            base64EncodeStr(hiveInitStatements));
    sparkExecutable.setParam(SparkCreatingFlatTable.getSqlOption(1).getOpt(),
            base64EncodeStr(dropTableHql));

    // createTableHql include create table sql and alter table sql
    String[] sqlArr = createTableHql.trim().split(";");
    if (2 != sqlArr.length) {
        throw new RuntimeException("create table hql should combined by a create table sql " +
                "and a alter sql, but got: " + createTableHql);
    }
    sparkExecutable.setParam(SparkCreatingFlatTable.getSqlOption(2).getOpt(),
            base64EncodeStr(sqlArr[0]));
    sparkExecutable.setParam(SparkCreatingFlatTable.getSqlOption(3).getOpt(),
            base64EncodeStr(sqlArr[1]));

    sparkExecutable.setParam(SparkCreatingFlatTable.getSqlOption(4).getOpt(),
            base64EncodeStr(insertDataHqls));

    StringBuilder jars = new StringBuilder();
    StringUtil.appendWithSeparator(jars, config.getSparkAdditionalJars());
    sparkExecutable.setJars(jars.toString());

    return sparkExecutable;
}
Example 11
Source File: IIFlattenHiveJob.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
@Override
public int run(String[] args) throws Exception {
    Options options = new Options();
    try {
        options.addOption(OPTION_II_NAME);
        parseOptions(options, args);

        String iiname = getOptionValue(OPTION_II_NAME);
        KylinConfig config = KylinConfig.getInstanceFromEnv();

        IIInstance iiInstance = IIManager.getInstance(config).getII(iiname);
        IIDesc iidesc = IIDescManager.getInstance(config).getIIDesc(iiInstance.getDescName());

        String jobUUID = "00bf87b5-c7b5-4420-a12a-07f6b37b3187";
        JobEngineConfig engineConfig = new JobEngineConfig(config);
        IJoinedFlatTableDesc intermediateTableDesc = new IIJoinedFlatTableDesc(iidesc);
        String dropTableHql = JoinedFlatTable.generateDropTableStatement(intermediateTableDesc, jobUUID);
        String createTableHql = JoinedFlatTable.generateCreateTableStatement(intermediateTableDesc, //
                JobInstance.getJobWorkingDir(jobUUID, engineConfig.getHdfsWorkingDirectory()), jobUUID);
        String insertDataHqls = JoinedFlatTable.generateInsertDataStatement(intermediateTableDesc, jobUUID, engineConfig);

        StringBuffer buf = new StringBuffer();
        buf.append("hive -e \"");
        buf.append(dropTableHql + "\n");
        buf.append(createTableHql + "\n");
        buf.append(insertDataHqls + "\n");
        buf.append("\"");
        
        System.out.println(buf.toString());
        System.out.println("========================");

        ShellCmd cmd = new ShellCmd(buf.toString(), null, null, null, false);
        ICommandOutput output = cmd.execute();
        System.out.println(output.getOutput());
        System.out.println(output.getExitCode());
        
        return 0;
    } catch (Exception e) {
        printUsage(options);
        throw e;
    }
}
Example 12
Source File: JoinedFlatTableTest.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
@Test
public void testGenDropTableDDL() {
    String ddl = JoinedFlatTable.generateDropTableStatement(intermediateTableDesc, fakeJobUUID);
    System.out.println(ddl);
    assertEquals(107, ddl.length());
}