Java Code Examples for org.apache.kylin.metadata.model.PartitionDesc#getPartitionDateColumn()
The following examples show how to use
org.apache.kylin.metadata.model.PartitionDesc#getPartitionDateColumn() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: JoinedFormatter.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
|
private void setDateEnv(IJoinedFlatTableDesc flatDesc) {
DataModelDesc model = flatDesc.getDataModel();
PartitionDesc partDesc = model.getPartitionDesc();
SegmentRange segRange = flatDesc.getSegRange();
long startInclusive = (Long) segRange.start.v;
long endExclusive = (Long) segRange.end.v;
//
String startDate = "";
String endDate = "";
String partitionColumnDateFormat = partDesc.getPartitionDateFormat();
if (partDesc.getPartitionTimeColumn() == null && partDesc.getPartitionDateColumn() == null) {
startDate = String.valueOf(startInclusive);
endDate = String.valueOf(endExclusive);
} else {
startDate = DateFormat.formatToDateStr(startInclusive, partitionColumnDateFormat);
endDate = DateFormat.formatToDateStr(endExclusive, partitionColumnDateFormat);
}
setKeyValue(START_DATE, startDate);
setKeyValue(END_DATE, endDate);
}
Example 2
| Source File: JoinedFormatter.java From kylin with Apache License 2.0 | 6 votes |
|
private void setDateEnv(IJoinedFlatTableDesc flatDesc) {
DataModelDesc model = flatDesc.getDataModel();
PartitionDesc partDesc = model.getPartitionDesc();
SegmentRange segRange = flatDesc.getSegRange();
long startInclusive = (Long) segRange.start.v;
long endExclusive = (Long) segRange.end.v;
//
String startDate = "";
String endDate = "";
String partitionColumnDateFormat = partDesc.getPartitionDateFormat();
if (partDesc.getPartitionTimeColumn() == null && partDesc.getPartitionDateColumn() == null) {
startDate = String.valueOf(startInclusive);
endDate = String.valueOf(endExclusive);
} else {
startDate = DateFormat.formatToDateStr(startInclusive, partitionColumnDateFormat);
endDate = DateFormat.formatToDateStr(endExclusive, partitionColumnDateFormat);
}
setKeyValue(START_DATE, startDate);
setKeyValue(END_DATE, endDate);
}
Example 3
| Source File: JdbcHiveInputBase.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
private static void appendWhereStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql, boolean singleLine,
IJdbcMetadata metadata, Map<String, String> metaMap) {
final String sep = singleLine ? " " : "\n";
StringBuilder whereBuilder = new StringBuilder();
whereBuilder.append("WHERE 1=1");
DataModelDesc model = flatDesc.getDataModel();
if (StringUtils.isNotEmpty(model.getFilterCondition())) {
whereBuilder.append(" AND (").append(model.getFilterCondition()).append(") ");
}
if (flatDesc.getSegment() != null) {
PartitionDesc partDesc = model.getPartitionDesc();
if (partDesc != null && partDesc.getPartitionDateColumn() != null) {
SegmentRange segRange = flatDesc.getSegRange();
if (segRange != null && !segRange.isInfinite()) {
whereBuilder.append(" AND (");
whereBuilder.append(partDesc.getPartitionConditionBuilder().buildDateRangeCondition(partDesc,
flatDesc.getSegment(), segRange,
col -> getTableColumnIdentityQuoted(col, metadata, metaMap, true)));
whereBuilder.append(")");
whereBuilder.append(sep);
}
}
}
sql.append(whereBuilder.toString());
}
Example 4
| Source File: JoinedFlatTable.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
private static void appendWhereStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql, boolean singleLine, SqlDialect sqlDialect) {
final String sep = singleLine ? " " : "\n";
StringBuilder whereBuilder = new StringBuilder();
whereBuilder.append("WHERE 1=1");
DataModelDesc model = flatDesc.getDataModel();
if (StringUtils.isNotEmpty(model.getFilterCondition())) {
String filterCondition = model.getFilterCondition();
if (flatDesc.getSegment() != null) {
JoinedFormatter formatter = new JoinedFormatter(flatDesc);
filterCondition = formatter.formatSentence(model.getFilterCondition());
}
String quotedFilterCondition = quoteIdentifierInSqlExpr(flatDesc, filterCondition, null);
whereBuilder.append(" AND (").append(quotedFilterCondition).append(") "); // -> filter condition contains special character may cause bug
}
if (flatDesc.getSegment() != null) {
PartitionDesc partDesc = model.getPartitionDesc();
if (partDesc != null && partDesc.getPartitionDateColumn() != null) {
SegmentRange segRange = flatDesc.getSegRange();
if (segRange != null && !segRange.isInfinite()) {
whereBuilder.append(" AND (");
String quotedPartitionCond = quoteIdentifierInSqlExpr(flatDesc,
partDesc.getPartitionConditionBuilder().buildDateRangeCondition(partDesc, flatDesc.getSegment(), segRange, null), sqlDialect);
whereBuilder.append(quotedPartitionCond);
whereBuilder.append(")" + sep);
}
}
}
sql.append(whereBuilder.toString());
}
Example 5
| Source File: JdbcHiveInputBase.java From kylin with Apache License 2.0 | 5 votes |
|
private static void appendWhereStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql, boolean singleLine,
IJdbcMetadata metadata, Map<String, String> metaMap) {
final String sep = singleLine ? " " : "\n";
StringBuilder whereBuilder = new StringBuilder();
whereBuilder.append("WHERE 1=1");
DataModelDesc model = flatDesc.getDataModel();
if (StringUtils.isNotEmpty(model.getFilterCondition())) {
whereBuilder.append(" AND (").append(model.getFilterCondition()).append(") ");
}
if (flatDesc.getSegment() != null) {
PartitionDesc partDesc = model.getPartitionDesc();
if (partDesc != null && partDesc.getPartitionDateColumn() != null) {
SegmentRange segRange = flatDesc.getSegRange();
if (segRange != null && !segRange.isInfinite()) {
whereBuilder.append(" AND (");
whereBuilder.append(partDesc.getPartitionConditionBuilder().buildDateRangeCondition(partDesc,
flatDesc.getSegment(), segRange,
col -> getTableColumnIdentityQuoted(col, metadata, metaMap, true)));
whereBuilder.append(")");
whereBuilder.append(sep);
}
}
}
sql.append(whereBuilder.toString());
}
Example 6
| Source File: JoinedFlatTable.java From kylin with Apache License 2.0 | 5 votes |
|
private static void appendWhereStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql, boolean singleLine, SqlDialect sqlDialect) {
final String sep = singleLine ? " " : "\n";
StringBuilder whereBuilder = new StringBuilder();
whereBuilder.append("WHERE 1=1");
DataModelDesc model = flatDesc.getDataModel();
if (StringUtils.isNotEmpty(model.getFilterCondition())) {
String filterCondition = model.getFilterCondition();
if (flatDesc.getSegment() != null) {
JoinedFormatter formatter = new JoinedFormatter(flatDesc);
filterCondition = formatter.formatSentence(model.getFilterCondition());
}
String quotedFilterCondition = quoteIdentifierInSqlExpr(flatDesc, filterCondition, null);
whereBuilder.append(" AND (").append(quotedFilterCondition).append(") "); // -> filter condition contains special character may cause bug
}
if (flatDesc.getSegment() != null) {
PartitionDesc partDesc = model.getPartitionDesc();
if (partDesc != null && partDesc.getPartitionDateColumn() != null) {
SegmentRange segRange = flatDesc.getSegRange();
if (segRange != null && !segRange.isInfinite()) {
whereBuilder.append(" AND (");
String quotedPartitionCond = quoteIdentifierInSqlExpr(flatDesc,
partDesc.getPartitionConditionBuilder().buildDateRangeCondition(partDesc, flatDesc.getSegment(), segRange, null), sqlDialect);
whereBuilder.append(quotedPartitionCond);
whereBuilder.append(")" + sep);
}
}
}
sql.append(whereBuilder.toString());
}
Example 7
| Source File: JdbcHiveInputBase.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
|
protected AbstractExecutable createSqoopToFlatHiveStep(String jobWorkingDir, String cubeName) {
KylinConfig config = getConfig();
PartitionDesc partitionDesc = flatDesc.getDataModel().getPartitionDesc();
String partCol = null;
if (partitionDesc.isPartitioned()) {
partCol = partitionDesc.getPartitionDateColumn();//tablename.colname
}
String splitTableAlias;
String splitColumn;
String splitDatabase;
TblColRef splitColRef = determineSplitColumn();
splitTableAlias = splitColRef.getTableAlias();
splitColumn = getColumnIdentityQuoted(splitColRef, jdbcMetadataDialect, metaMap, true);
splitDatabase = splitColRef.getColumnDesc().getTable().getDatabase();
String selectSql = generateSelectDataStatementRDBMS(flatDesc, true, new String[] { partCol },
jdbcMetadataDialect, metaMap);
selectSql = escapeQuotationInSql(selectSql);
String hiveTable = flatDesc.getTableName();
String connectionUrl = config.getJdbcSourceConnectionUrl();
String driverClass = config.getJdbcSourceDriver();
String jdbcUser = config.getJdbcSourceUser();
String jdbcPass = config.getJdbcSourcePass();
String sqoopHome = config.getSqoopHome();
String sqoopNullString = config.getSqoopNullString();
String sqoopNullNonString = config.getSqoopNullNonString();
String filedDelimiter = config.getJdbcSourceFieldDelimiter();
int mapperNum = config.getSqoopMapperNum();
String bquery = String.format(Locale.ROOT, "SELECT min(%s), max(%s) FROM %s.%s ", splitColumn,
splitColumn, getSchemaQuoted(metaMap, splitDatabase, jdbcMetadataDialect, true),
getTableIdentityQuoted(splitColRef.getTableRef(), metaMap, jdbcMetadataDialect, true));
if (partitionDesc.isPartitioned()) {
SegmentRange segRange = flatDesc.getSegRange();
if (segRange != null && !segRange.isInfinite()) {
if (partitionDesc.getPartitionDateColumnRef().getTableAlias().equals(splitTableAlias)
&& (partitionDesc.getPartitionTimeColumnRef() == null || partitionDesc
.getPartitionTimeColumnRef().getTableAlias().equals(splitTableAlias))) {
String quotedPartCond = partitionDesc.getPartitionConditionBuilder().buildDateRangeCondition(
partitionDesc, flatDesc.getSegment(), segRange,
col -> getTableColumnIdentityQuoted(col, jdbcMetadataDialect, metaMap, true));
bquery += " WHERE " + quotedPartCond;
}
}
}
bquery = escapeQuotationInSql(bquery);
// escape ` in cmd
splitColumn = escapeQuotationInSql(splitColumn);
String cmd = String.format(Locale.ROOT, "%s/bin/sqoop import" + generateSqoopConfigArgString()
+ "--connect \"%s\" --driver %s --username %s --password \"%s\" --query \"%s AND \\$CONDITIONS\" "
+ "--target-dir %s/%s --split-by %s --boundary-query \"%s\" --null-string '%s' "
+ "--null-non-string '%s' --fields-terminated-by '%s' --num-mappers %d", sqoopHome, connectionUrl,
driverClass, jdbcUser, jdbcPass, selectSql, jobWorkingDir, hiveTable, splitColumn, bquery,
sqoopNullString, sqoopNullNonString, filedDelimiter, mapperNum);
logger.debug("sqoop cmd : {}", cmd);
CmdStep step = new CmdStep();
step.setCmd(cmd);
step.setName(ExecutableConstants.STEP_NAME_SQOOP_TO_FLAT_HIVE_TABLE);
return step;
}
Example 8
| Source File: JdbcHiveInputBase.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
|
@Override
protected AbstractExecutable createSqoopToFlatHiveStep(String jobWorkingDir, String cubeName) {
KylinConfig config = flatDesc.getDataModel().getConfig();
PartitionDesc partitionDesc = flatDesc.getDataModel().getPartitionDesc();
String partCol = null;
boolean enableQuote = dataSource.getSqlConverter().getConfigurer().enableQuote();
SqlDialect sqlDialect = enableQuote ? dataSource.getSqlConverter().getConfigurer().getSqlDialect() : FlatTableSqlQuoteUtils.NON_QUOTE_DIALECT;
SqlConverter.IConfigurer iconfigurer = dataSource.getSqlConverter().getConfigurer();
if (partitionDesc.isPartitioned()) {
partCol = partitionDesc.getPartitionDateColumn(); //tablename.colname
}
String splitTable;
String splitTableAlias;
String splitColumn;
String splitDatabase;
TblColRef splitColRef = determineSplitColumn();
splitTable = splitColRef.getTableRef().getTableDesc().getName();
splitTableAlias = splitColRef.getTableAlias();
splitDatabase = splitColRef.getColumnDesc().getTable().getDatabase();
if (enableQuote) {
splitColumn = sqlDialect.quoteIdentifier(splitColRef.getTableAlias()) + "."
+ sqlDialect.quoteIdentifier(splitColRef.getName());
splitDatabase = sqlDialect.quoteIdentifier(splitDatabase);
splitTable = sqlDialect.quoteIdentifier(splitTable);
splitTableAlias = sqlDialect.quoteIdentifier(splitTableAlias);
} else {
splitColumn = splitColRef.getTableAlias() + "." + splitColRef.getName();
}
String selectSql = JoinedFlatTable.generateSelectDataStatement(flatDesc, true, new String[]{partCol}, sqlDialect);
selectSql = escapeQuotationInSql(dataSource.convertSql(selectSql));
String hiveTable = flatDesc.getTableName();
String filedDelimiter = config.getJdbcSourceFieldDelimiter();
int mapperNum = config.getSqoopMapperNum();
String bquery;
bquery = String.format(Locale.ROOT, "SELECT min(%s), max(%s) FROM %s.%s as %s",
splitColumn, splitColumn, splitDatabase, splitTable, splitTableAlias);
bquery = dataSource.convertSql(bquery);
if (partitionDesc.isPartitioned()) {
SegmentRange segRange = flatDesc.getSegRange();
if (segRange != null && !segRange.isInfinite()) {
if (partitionDesc.getPartitionDateColumnRef().getTableAlias().equals(splitTableAlias)
&& (partitionDesc.getPartitionTimeColumnRef() == null || partitionDesc
.getPartitionTimeColumnRef().getTableAlias().equals(splitTableAlias))) {
String quotedPartCond = FlatTableSqlQuoteUtils.quoteIdentifierInSqlExpr(flatDesc,
partitionDesc.getPartitionConditionBuilder().buildDateRangeCondition(partitionDesc,
flatDesc.getSegment(), segRange, null), sqlDialect);
bquery += " WHERE " + quotedPartCond;
}
}
}
bquery = escapeQuotationInSql(bquery);
splitColumn = escapeQuotationInSql(splitColumn);
String cmd = StringUtils.format(
"--connect \"%s\" --driver \"%s\" --username \"%s\" --password \"%s\" --query \"%s AND \\$CONDITIONS\" "
+ "--target-dir \"%s/%s\" --split-by \"%s\" --boundary-query \"%s\" --null-string '' "
+ "--fields-terminated-by '%s' --num-mappers %d",
dataSource.getJdbcUrl(), dataSource.getJdbcDriver(), dataSource.getJdbcUser(),
dataSource.getJdbcPassword(), selectSql, jobWorkingDir, hiveTable, splitColumn, bquery,
filedDelimiter, mapperNum);
if (iconfigurer.getTransactionIsolationLevel() != null) {
cmd = cmd + " --relaxed-isolation --metadata-transaction-isolation-level "
+ iconfigurer.getTransactionIsolationLevel();
}
logger.debug("sqoop cmd: {}", cmd);
SqoopCmdStep step = new SqoopCmdStep();
step.setCmd(cmd);
step.setName(ExecutableConstants.STEP_NAME_SQOOP_TO_FLAT_HIVE_TABLE);
return step;
}
Example 9
| Source File: JdbcHiveInputBase.java From kylin with Apache License 2.0 | 4 votes |
|
protected AbstractExecutable createSqoopToFlatHiveStep(String jobWorkingDir, String cubeName) {
KylinConfig config = getConfig();
PartitionDesc partitionDesc = flatDesc.getDataModel().getPartitionDesc();
String partCol = null;
if (partitionDesc.isPartitioned()) {
partCol = partitionDesc.getPartitionDateColumn();//tablename.colname
}
String splitTableAlias;
String splitColumn;
String splitDatabase;
TblColRef splitColRef = determineSplitColumn();
splitTableAlias = splitColRef.getTableAlias();
splitColumn = getColumnIdentityQuoted(splitColRef, jdbcMetadataDialect, metaMap, true);
splitDatabase = splitColRef.getColumnDesc().getTable().getDatabase();
String selectSql = generateSelectDataStatementRDBMS(flatDesc, true, new String[] { partCol },
jdbcMetadataDialect, metaMap);
selectSql = escapeQuotationInSql(selectSql);
String hiveTable = flatDesc.getTableName();
String connectionUrl = config.getJdbcSourceConnectionUrl();
String driverClass = config.getJdbcSourceDriver();
String jdbcUser = config.getJdbcSourceUser();
String jdbcPass = config.getJdbcSourcePass();
String sqoopHome = config.getSqoopHome();
String sqoopNullString = config.getSqoopNullString();
String sqoopNullNonString = config.getSqoopNullNonString();
String filedDelimiter = config.getJdbcSourceFieldDelimiter();
int mapperNum = config.getSqoopMapperNum();
String bquery = String.format(Locale.ROOT, "SELECT min(%s), max(%s) FROM %s.%s ", splitColumn,
splitColumn, getSchemaQuoted(metaMap, splitDatabase, jdbcMetadataDialect, true),
getTableIdentityQuoted(splitColRef.getTableRef(), metaMap, jdbcMetadataDialect, true));
if (partitionDesc.isPartitioned()) {
SegmentRange segRange = flatDesc.getSegRange();
if (segRange != null && !segRange.isInfinite()) {
if (partitionDesc.getPartitionDateColumnRef().getTableAlias().equals(splitTableAlias)
&& (partitionDesc.getPartitionTimeColumnRef() == null || partitionDesc
.getPartitionTimeColumnRef().getTableAlias().equals(splitTableAlias))) {
String quotedPartCond = partitionDesc.getPartitionConditionBuilder().buildDateRangeCondition(
partitionDesc, flatDesc.getSegment(), segRange,
col -> getTableColumnIdentityQuoted(col, jdbcMetadataDialect, metaMap, true));
bquery += " WHERE " + quotedPartCond;
}
}
}
bquery = escapeQuotationInSql(bquery);
// escape ` in cmd
splitColumn = escapeQuotationInSql(splitColumn);
String cmd = String.format(Locale.ROOT, "%s/bin/sqoop import" + generateSqoopConfigArgString()
+ "--connect \"%s\" --driver %s --username %s --password \"%s\" --query \"%s AND \\$CONDITIONS\" "
+ "--target-dir %s/%s --split-by %s --boundary-query \"%s\" --null-string '%s' "
+ "--null-non-string '%s' --fields-terminated-by '%s' --num-mappers %d", sqoopHome, connectionUrl,
driverClass, jdbcUser, jdbcPass, selectSql, jobWorkingDir, hiveTable, splitColumn, bquery,
sqoopNullString, sqoopNullNonString, filedDelimiter, mapperNum);
logger.debug("sqoop cmd : {}", cmd);
CmdStep step = new CmdStep();
step.setCmd(cmd);
step.setName(ExecutableConstants.STEP_NAME_SQOOP_TO_FLAT_HIVE_TABLE);
return step;
}
Example 10
| Source File: JdbcHiveInputBase.java From kylin with Apache License 2.0 | 4 votes |
|
@Override
protected AbstractExecutable createSqoopToFlatHiveStep(String jobWorkingDir, String cubeName) {
KylinConfig config = flatDesc.getDataModel().getConfig();
PartitionDesc partitionDesc = flatDesc.getDataModel().getPartitionDesc();
String partCol = null;
boolean enableQuote = dataSource.getSqlConverter().getConfigurer().enableQuote();
enableQuote = enableQuote && config.enableHiveDdlQuote();
logger.debug("Quote switch is set to {}", enableQuote);
SqlDialect sqlDialect = enableQuote ? dataSource.getSqlConverter().getConfigurer().getSqlDialect() : FlatTableSqlQuoteUtils.NON_QUOTE_DIALECT;
SqlConverter.IConfigurer iconfigurer = dataSource.getSqlConverter().getConfigurer();
if (partitionDesc.isPartitioned()) {
partCol = partitionDesc.getPartitionDateColumn(); //tablename.colname
}
String splitTable;
String splitTableAlias;
String splitColumn;
String splitDatabase;
TblColRef splitColRef = determineSplitColumn();
splitTable = splitColRef.getTableRef().getTableDesc().getName();
splitTableAlias = splitColRef.getTableAlias();
splitDatabase = splitColRef.getColumnDesc().getTable().getDatabase();
if (enableQuote) {
splitColumn = sqlDialect.quoteIdentifier(splitColRef.getTableAlias()) + "."
+ sqlDialect.quoteIdentifier(splitColRef.getName());
splitDatabase = sqlDialect.quoteIdentifier(splitDatabase);
splitTable = sqlDialect.quoteIdentifier(splitTable);
splitTableAlias = sqlDialect.quoteIdentifier(splitTableAlias);
} else {
splitColumn = splitColRef.getTableAlias() + "." + splitColRef.getName();
}
String selectSql = JoinedFlatTable.generateSelectDataStatement(flatDesc, true, new String[]{partCol}, sqlDialect);
selectSql = escapeQuotationInSql(dataSource.convertSql(selectSql));
String hiveTable = flatDesc.getTableName();
String filedDelimiter = config.getJdbcSourceFieldDelimiter();
int mapperNum = config.getSqoopMapperNum();
String bquery;
bquery = String.format(Locale.ROOT, "SELECT min(%s), max(%s) FROM %s.%s as %s",
splitColumn, splitColumn, splitDatabase, splitTable, splitTableAlias);
bquery = dataSource.convertSql(bquery);
if (partitionDesc.isPartitioned()) {
SegmentRange segRange = flatDesc.getSegRange();
if (segRange != null && !segRange.isInfinite()) {
if (partitionDesc.getPartitionDateColumnRef().getTableAlias().equals(splitTableAlias)
&& (partitionDesc.getPartitionTimeColumnRef() == null || partitionDesc
.getPartitionTimeColumnRef().getTableAlias().equals(splitTableAlias))) {
String quotedPartCond = FlatTableSqlQuoteUtils.quoteIdentifierInSqlExpr(flatDesc,
partitionDesc.getPartitionConditionBuilder().buildDateRangeCondition(partitionDesc,
flatDesc.getSegment(), segRange, null), sqlDialect);
bquery += " WHERE " + quotedPartCond;
}
}
}
bquery = escapeQuotationInSql(bquery);
splitColumn = escapeQuotationInSql(splitColumn);
String cmd = StringUtils.format(
"--connect \"%s\" --driver \"%s\" --username \"%s\" --password \"%s\" --query \"%s AND \\$CONDITIONS\" "
+ "--target-dir \"%s/%s\" --split-by \"%s\" --boundary-query \"%s\" --null-string '' "
+ "--fields-terminated-by '%s' --num-mappers %d",
dataSource.getJdbcUrl(), dataSource.getJdbcDriver(), dataSource.getJdbcUser(),
dataSource.getJdbcPassword(), selectSql, jobWorkingDir, hiveTable, splitColumn, bquery,
filedDelimiter, mapperNum);
if (iconfigurer.getTransactionIsolationLevel() != null) {
cmd = cmd + " --relaxed-isolation --metadata-transaction-isolation-level "
+ iconfigurer.getTransactionIsolationLevel();
}
logger.debug("sqoop cmd: {}", cmd);
SqoopCmdStep step = new SqoopCmdStep();
step.setCmd(cmd);
step.setName(ExecutableConstants.STEP_NAME_SQOOP_TO_FLAT_HIVE_TABLE);
return step;
}
