org.apache.kylin.common.util.StringSplitter Java Examples

The following examples show how to use org.apache.kylin.common.util.StringSplitter. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TableDesc.java    From kylin with Apache License 2.0 6 votes vote down vote up 
public void setName(String name) {
    if (name != null) {
        String[] splits = StringSplitter.split(name, ".");
        if (splits.length == 2) {
            this.setDatabase(splits[0]);
            this.name = splits[1];
        } else if (splits.length == 1) {
            this.name = splits[0];
        }
    } else {
        this.name = null;
    }
    if (identity != null) {
        setIdentity();
    }
}
Example #2
Source File: AbstractHadoopJob.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
public void addInputDirs(String input, Job job) throws IOException {
    for (String inp : StringSplitter.split(input, ",")) {
        inp = inp.trim();
        if (inp.endsWith("/*")) {
            inp = inp.substring(0, inp.length() - 2);
            FileSystem fs = FileSystem.get(job.getConfiguration());
            Path path = new Path(inp);
            FileStatus[] fileStatuses = fs.listStatus(path);
            boolean hasDir = false;
            for (FileStatus stat : fileStatuses) {
                if (stat.isDirectory() && !stat.getPath().getName().startsWith("_")) {
                    hasDir = true;
                    addInputDirs(stat.getPath().toString(), job);
                }
            }
            if (fileStatuses.length > 0 && !hasDir) {
                addInputDirs(path.toString(), job);
            }
        } else {
            logger.debug("Add input " + inp);
            FileInputFormat.addInputPath(job, new Path(inp));
        }
    }
}
Example #3
Source File: PartitionDesc.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
public void init(Map<String, TableDesc> tables) {
    if (StringUtils.isNotEmpty(partitionDateColumn)) {
        partitionDateColumn = partitionDateColumn.toUpperCase();

        String[] columns = StringSplitter.split(partitionDateColumn, ".");

        if (null != columns && columns.length == 3) {
            String tableName = columns[0].toUpperCase() + "." + columns[1].toUpperCase();

            TableDesc table = tables.get(tableName);
            ColumnDesc col = table.findColumnByName(columns[2]);
            if (col != null) {
                partitionDateColumnRef = new TblColRef(col);
            } else {
                throw new IllegalStateException("The column '" + partitionDateColumn + "' provided in 'partition_date_column' doesn't exist.");
            }
        } else {
            throw new IllegalStateException("The 'partition_date_column' format is invalid: " + partitionDateColumn + ", it should be {db}.{table}.{column}.");
        }
    }
}
Example #4
Source File: TableDesc.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public void setName(String name) {
    if (name != null) {
        String[] splits = StringSplitter.split(name, ".");
        if (splits.length == 2) {
            this.setDatabase(splits[0]);
            this.name = splits[1];
        } else if (splits.length == 1) {
            this.name = splits[0];
        }
    } else {
        this.name = null;
    }
}
Example #5
Source File: DFSFileTableReader.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private String[] split(String line, String delim) {
    // FIXME CVS line should be parsed considering escapes
    String[] str = StringSplitter.split(line, delim);

    // un-escape CSV
    if (DFSFileTable.DELIM_COMMA.equals(delim)) {
        for (int i = 0; i < str.length; i++) {
            str[i] = unescapeCsv(str[i]);
        }
    }

    return str;
}
Example #6
Source File: DFSFileTableReader.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private String autoDetectDelim(String line) {
    if (expectedColumnNumber > 0) {
        for (String delim : DETECT_DELIMS) {
            if (StringSplitter.split(line, delim).length == expectedColumnNumber) {
                logger.info("Auto detect delim to be '" + delim + "', split line to " + expectedColumnNumber + " columns -- " + line);
                return delim;
            }
        }
    }

    logger.info("Auto detect delim to be null, will take THE-WHOLE-LINE as a single value, for " + filePath);
    return null;
}
Example #7
Source File: DFSFileTableReader.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private String[] split(String line, String delim) {
    // FIXME CVS line should be parsed considering escapes
    String[] str = StringSplitter.split(line, delim);

    // un-escape CSV
    if (DFSFileTable.DELIM_COMMA.equals(delim)) {
        for (int i = 0; i < str.length; i++) {
            str[i] = unescapeCsv(str[i]);
        }
    }

    return str;
}
Example #8
Source File: DFSFileTableReader.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private String autoDetectDelim(String line) {
    if (expectedColumnNumber > 0) {
        for (String delim : DETECT_DELIMS) {
            if (StringSplitter.split(line, delim).length == expectedColumnNumber) {
                logger.info("Auto detect delim to be '" + delim + "', split line to " + expectedColumnNumber + " columns -- " + line);
                return delim;
            }
        }
    }

    logger.info("Auto detect delim to be null, will take THE-WHOLE-LINE as a single value, for " + filePath);
    return null;
}
Example #9
Source File: FileTableReader.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private String[] split(String line, String delim) {
    // FIXME CVS line should be parsed considering escapes
    String str[] = StringSplitter.split(line, delim);

    // un-escape CSV
    if (ReadableTable.DELIM_COMMA.equals(delim)) {
        for (int i = 0; i < str.length; i++) {
            str[i] = unescapeCsv(str[i]);
        }
    }

    return str;
}
Example #10
Source File: FileTableReader.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private String autoDetectDelim(String line) {
    if (expectedColumnNumber > 0) {
        for (String delim : DETECT_DELIMS) {
            if (StringSplitter.split(line, delim).length == expectedColumnNumber) {
                logger.info("Auto detect delim to be '" + delim + "', split line to " + expectedColumnNumber + " columns -- " + line);
                return delim;
            }
        }
    }

    logger.info("Auto detect delim to be null, will take THE-WHOLE-LINE as a single value, for " + filePath);
    return null;
}
Example #11
Source File: TableDesc.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
public void setName(String name) {
    if (name != null) {
        String[] splits = StringSplitter.split(name, ".");
        if (splits.length == 2) {
            this.setDatabase(splits[0]);
            this.name = splits[1];
        } else if (splits.length == 1) {
            this.name = splits[0];
        }
    } else {
        this.name = name;
    }
}
Example #12
Source File: AbstractHadoopJob.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public static int addInputDirs(String input, Job job) throws IOException {
    int folderNum = addInputDirs(StringSplitter.split(input, ","), job);
    logger.info("Number of added folders:" + folderNum);
    return folderNum;
}
Example #13
Source File: AbstractHadoopJob.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public static int addInputDirs(String input, Job job) throws IOException {
    int folderNum = addInputDirs(StringSplitter.split(input, ","), job);
    logger.info("Number of added folders:" + folderNum);
    return folderNum;
}