Java Code Examples for org.apache.kylin.common.KylinConfig#getHdfsWorkingDirectory()

The following examples show how to use org.apache.kylin.common.KylinConfig#getHdfsWorkingDirectory() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: HDFSResourceStore.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
public HDFSResourceStore(KylinConfig kylinConfig, StorageURL metadataUrl) throws Exception {
    super(kylinConfig);
    Preconditions.checkState(HDFS_SCHEME.equals(metadataUrl.getScheme()));

    String path = metadataUrl.getParameter("path");
    if (path == null) {
        // missing path is not expected, but don't fail it
        path = kylinConfig.getHdfsWorkingDirectory(null) + "tmp_metadata";
        logger.warn("Missing path, fall back to {}. ", path);
    }

    fs = HadoopUtil.getFileSystem(path);
    Path metadataPath = new Path(path);
    if (!fs.exists(metadataPath)) {
        logger.warn("Path not exist in HDFS, create it: {}. ", path);
        createMetaFolder(metadataPath);
    }

    hdfsMetaPath = metadataPath;
    logger.info("hdfs meta path : {}", hdfsMetaPath);

}
Example 2
Source File: AppendTrieDictionary.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private String convertToAbsolutePath(String path) {
    KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    Path basicPath = new Path(path);
    if (basicPath.toUri().getScheme() == null)
        return kylinConfig.getHdfsWorkingDirectory() + path;

    String[] paths = path.split("/resources/GlobalDict/");
    if (paths.length == 2)
        return kylinConfig.getHdfsWorkingDirectory() + "/resources/GlobalDict/" + paths[1];

    paths = path.split("/resources/SegmentDict/");
    if (paths.length == 2) {
        return kylinConfig.getHdfsWorkingDirectory() + "/resources/SegmentDict/" + paths[1];
    } else {
        throw new RuntimeException(
                "the basic directory of global dictionary only support the format which contains '/resources/GlobalDict/' or '/resources/SegmentDict/'");
    }
}
Example 3
Source File: NSparkUpdateMetaAndCleanupAfterMergeStep.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    String cubeId = getParam(MetadataConstants.P_CUBE_ID);
    String[] segments = StringUtils.split(getParam(MetadataConstants.P_SEGMENT_NAMES), ",");
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    CubeInstance cube = CubeManager.getInstance(config).getCubeByUuid(cubeId);

    updateMetadataAfterMerge(cubeId);

    for (String segmentName : segments) {
        String path = config.getHdfsWorkingDirectory() + cube.getProject() + "/parquet/" + cube.getName() + "/" + segmentName;
        try {
            HadoopUtil.deletePath(HadoopUtil.getCurrentConfiguration(), new Path(path));
        } catch (IOException e) {
            throw new ExecuteException("Can not delete segment: " + segmentName + ", in cube: " + cube.getName());
        }
    }

    return ExecuteResult.createSucceed();
}
Example 4
Source File: HDFSResourceStore.java    From kylin with Apache License 2.0 6 votes vote down vote up 
public HDFSResourceStore(KylinConfig kylinConfig, StorageURL metadataUrl) throws Exception {
    super(kylinConfig);
    Preconditions.checkState(HDFS_SCHEME.equals(metadataUrl.getScheme()));

    String path = metadataUrl.getParameter("path");
    if (path == null) {
        // missing path is not expected, but don't fail it
        path = kylinConfig.getHdfsWorkingDirectory(null) + "tmp_metadata";
        logger.warn("Missing path, fall back to {}. ", path);
    }

    fs = HadoopUtil.getFileSystem(path);
    Path metadataPath = new Path(path);
    if (!fs.exists(metadataPath)) {
        logger.warn("Path not exist in HDFS, create it: {}. ", path);
        createMetaFolder(metadataPath);
    }

    hdfsMetaPath = metadataPath;
    logger.info("hdfs meta path : {}", hdfsMetaPath);

}
Example 5
Source File: AppendTrieDictionary.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private String convertToAbsolutePath(String path) {
    KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    Path basicPath = new Path(path);
    if (basicPath.toUri().getScheme() == null)
        return kylinConfig.getHdfsWorkingDirectory() + path;

    String[] paths = path.split("/resources/GlobalDict/");
    if (paths.length == 2)
        return kylinConfig.getHdfsWorkingDirectory() + "/resources/GlobalDict/" + paths[1];

    paths = path.split("/resources/SegmentDict/");
    if (paths.length == 2) {
        return kylinConfig.getHdfsWorkingDirectory() + "/resources/SegmentDict/" + paths[1];
    } else {
        throw new RuntimeException(
                "the basic directory of global dictionary only support the format which contains '/resources/GlobalDict/' or '/resources/SegmentDict/'");
    }
}
Example 6
Source File: AppendTrieDictionary.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
/**
 * JIRA: https://issues.apache.org/jira/browse/KYLIN-2945
 * if pass a absolute path, it may produce some problems like cannot find global dict after migration.
 * so convert to relative path can avoid it and be better to maintain flexibility.
 *
 */
private String convertToRelativePath(String path) {
    KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    String hdfsWorkingDir = kylinConfig.getHdfsWorkingDirectory();
    if (!isSaveAbsolutePath && path.startsWith(hdfsWorkingDir)) {
        return path.substring(hdfsWorkingDir.length());
    }
    return path;
}
Example 7
Source File: CubingJob.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private String getCuboidRootPath(CubeSegment seg, KylinConfig kylinConfig) {
    String rootDir = kylinConfig.getHdfsWorkingDirectory();
    if (!rootDir.endsWith("/")) {
        rootDir = rootDir + "/";
    }
    String jobID = this.getId();
    return rootDir + "kylin-" + jobID + "/" + seg.getRealization().getName() + "/cuboid/";
}
Example 8
Source File: DeployCoprocessorCLI.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private static Path getCoprocessorHDFSDir(FileSystem fileSystem, KylinConfig config) throws IOException {
    String hdfsWorkingDirectory = config.getHdfsWorkingDirectory();
    hdfsWorkingDirectory = HBaseConnection.makeQualifiedPathInHBaseCluster(hdfsWorkingDirectory);
    Path coprocessorDir = new Path(hdfsWorkingDirectory, "coprocessor");
    fileSystem.mkdirs(coprocessorDir);
    return coprocessorDir;
}
Example 9
Source File: NGlobalDictionaryV2Test.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private void roundTest(int size) throws IOException {
    System.out.println("GlobalDictionaryV2Test -> roundTest -> " + System.currentTimeMillis());
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    NGlobalDictionaryV2 dict1 = new NGlobalDictionaryV2("t1", "a", "spark", config.getHdfsWorkingDirectory());
    NGlobalDictionaryV2 dict2 = new NGlobalDictionaryV2("t2", "a", "local", config.getHdfsWorkingDirectory());
    List<String> stringList = generateRandomData(size);
    Collections.sort(stringList);
    runWithSparkBuildGlobalDict(dict1, stringList);
    runWithLocalBuildGlobalDict(dict2, stringList);
    compareTwoVersionDict(dict1, dict2);
    compareTwoModeVersionNum(dict1, dict2);
}
Example 10
Source File: AppendTrieDictionary.java    From kylin with Apache License 2.0 5 votes vote down vote up 
/**
 * JIRA: https://issues.apache.org/jira/browse/KYLIN-2945
 * if pass a absolute path, it may produce some problems like cannot find global dict after migration.
 * so convert to relative path can avoid it and be better to maintain flexibility.
 *
 */
private String convertToRelativePath(String path) {
    KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    String hdfsWorkingDir = kylinConfig.getHdfsWorkingDirectory();
    if (!isSaveAbsolutePath && path.startsWith(hdfsWorkingDir)) {
        return path.substring(hdfsWorkingDir.length());
    }
    return path;
}
Example 11
Source File: CubingJob.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private String getCuboidRootPath(CubeSegment seg, KylinConfig kylinConfig) {
    String rootDir = kylinConfig.getHdfsWorkingDirectory();
    if (!rootDir.endsWith("/")) {
        rootDir = rootDir + "/";
    }
    String jobID = this.getId();
    return rootDir + "kylin-" + jobID + "/" + seg.getRealization().getName() + "/cuboid/";
}
Example 12
Source File: HiveInputBase.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public BaseBatchCubingInputSide(IJoinedFlatTableDesc flatDesc) {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    this.flatDesc = flatDesc;
    this.flatTableDatabase = config.getHiveDatabaseForIntermediateTable();
    this.hdfsWorkingDir = config.getHdfsWorkingDirectory();
}
Example 13
Source File: PathManager.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public static String getParquetStoragePath(KylinConfig config, String cubeName, String segName, String identifier, String cuboidId) {
    CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName);
    String hdfsWorkDir = config.getHdfsWorkingDirectory(cube.getProject());
    return hdfsWorkDir + "parquet" + File.separator + cubeName + File.separator + segName + "_" + identifier + File.separator + cuboidId;
}
Example 14
Source File: HiveInputBase.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public BaseBatchCubingInputSide(IJoinedFlatTableDesc flatDesc) {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    this.flatDesc = flatDesc;
    this.flatTableDatabase = config.getHiveDatabaseForIntermediateTable();
    this.hdfsWorkingDir = config.getHdfsWorkingDirectory();
}
Example 15
Source File: DeployCoprocessorCLI.java    From kylin with Apache License 2.0 4 votes vote down vote up 
private static String getHDFSWorkingDirectory(KylinConfig config) {
    String hdfsWorkingDirectory = config.getHdfsWorkingDirectory();
    return HBaseConnection.makeQualifiedPathInHBaseCluster(hdfsWorkingDirectory);
}
Example 16
Source File: DeployCoprocessorCLI.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
private static Path getCoprocessorHDFSDir(FileSystem fileSystem, KylinConfig config) throws IOException {
    String hdfsWorkingDirectory = config.getHdfsWorkingDirectory();
    Path coprocessorDir = new Path(hdfsWorkingDirectory, "coprocessor");
    fileSystem.mkdirs(coprocessorDir);
    return coprocessorDir;
}