Java Code Examples for org.apache.kylin.common.util.HadoopUtil#getFileSystem()

public static Pair<Long, Long> getSizeAndLastModified(String path) throws IOException {
    FileSystem fs = HadoopUtil.getFileSystem(path);

    // get all contained files if path is directory
    ArrayList<FileStatus> allFiles = new ArrayList<>();
    FileStatus status = fs.getFileStatus(new Path(path));
    if (status.isFile()) {
        allFiles.add(status);
    } else {
        FileStatus[] listStatus = fs.listStatus(new Path(path));
        allFiles.addAll(Arrays.asList(listStatus));
    }

    long size = 0;
    long lastModified = 0;
    for (FileStatus file : allFiles) {
        size += file.getLen();
        lastModified = Math.max(lastModified, file.getModificationTime());
    }

    return Pair.newPair(size, lastModified);
}
 

public void flushToHDFS() throws IOException {
    logger.info("start to flush cube:{} segment:{} to hdfs:{}", segment.getCubeName(),
            segment.getSegmentName(), hdfsPath);
    final FileSystem fs = HadoopUtil.getFileSystem(hdfsPath);
    final String localPath = segment.getDataSegmentFolder().getPath();
    final Path remotePath = new Path(hdfsPath);
    if (fs.exists(remotePath)) {
        logger.info("the remote path:{} is already exist, skip copy data to remote", remotePath);
        return;
    }
    final Path remoteTempPath = new Path(hdfsPath + ".tmp");
    if (fs.exists(remoteTempPath)) {
        FileStatus sdst = fs.getFileStatus(remoteTempPath);
        if (sdst.isDirectory()) {
            logger.warn("target temp path: {} is an existed directory, try to delete it.", remoteTempPath);
            fs.delete(remoteTempPath, true);
            logger.warn("target temp path: {} is deleted.", remoteTempPath);
        }
    }
    fs.copyFromLocalFile(new Path(localPath), remoteTempPath);
    logger.info("data copy to remote temp path:{}", remoteTempPath);
    boolean renamed = fs.rename(remoteTempPath, remotePath);
    if (renamed) {
        logger.info("successfully rename the temp path to:{}", remotePath);
    }
}
 

private void convertIndexToOldFormat(String baseDir) throws IOException {
    Path basePath = new Path(baseDir);
    FileSystem fs = HadoopUtil.getFileSystem(basePath);

    GlobalDictHDFSStore store = new GlobalDictHDFSStore(baseDir);
    Long[] versions = store.listAllVersions();
    GlobalDictMetadata metadata = store.getMetadata(versions[versions.length - 1]);

    //convert v2 index to v1 index
    Path versionPath = store.getVersionDir(versions[versions.length - 1]);
    Path v2IndexFile = new Path(versionPath, V2_INDEX_NAME);

    fs.delete(v2IndexFile, true);
    GlobalDictHDFSStore.IndexFormat indexFormatV1 = new GlobalDictHDFSStore.IndexFormatV1(fs,
            HadoopUtil.getCurrentConfiguration());
    indexFormatV1.writeIndexFile(versionPath, metadata);

    //convert v2 fileName format to v1 fileName format
    for (Map.Entry<AppendDictSliceKey, String> entry : metadata.sliceFileMap.entrySet()) {
        fs.rename(new Path(versionPath, entry.getValue()), new Path(versionPath, "cached_" + entry.getKey()));
    }
}
 

public FileTableReader(String filePath, String delim, int expectedColumnNumber) throws IOException {
    this.filePath = filePath;
    this.delim = delim;
    this.expectedColumnNumber = expectedColumnNumber;

    FileSystem fs = HadoopUtil.getFileSystem(filePath);

    try {
        this.reader = new SeqRowReader(HadoopUtil.getCurrentConfiguration(), fs, filePath);

    } catch (IOException e) {
        if (isExceptionSayingNotSeqFile(e) == false)
            throw e;

        this.reader = new CsvRowReader(fs, filePath);
    }
}
 

public HDFSResourceStore(KylinConfig kylinConfig, StorageURL metadataUrl) throws Exception {
    super(kylinConfig);
    Preconditions.checkState(HDFS_SCHEME.equals(metadataUrl.getScheme()));

    String path = metadataUrl.getParameter("path");
    if (path == null) {
        // missing path is not expected, but don't fail it
        path = kylinConfig.getHdfsWorkingDirectory(null) + "tmp_metadata";
        logger.warn("Missing path, fall back to {}. ", path);
    }

    fs = HadoopUtil.getFileSystem(path);
    Path metadataPath = new Path(path);
    if (!fs.exists(metadataPath)) {
        logger.warn("Path not exist in HDFS, create it: {}. ", path);
        createMetaFolder(metadataPath);
    }

    hdfsMetaPath = metadataPath;
    logger.info("hdfs meta path : {}", hdfsMetaPath);

}
 

public void removeCubeHDFSFiles(String cubeName) {
    String segmentHDFSPath = HDFSUtil.getStreamingCubeFilePath(cubeName);
    try {
        FileSystem fs = HadoopUtil.getFileSystem(segmentHDFSPath);
        fs.delete(new Path(segmentHDFSPath), true);
    } catch (Exception e) {
        logger.error("Error when remove hdfs file, hdfs path:{}", segmentHDFSPath);
    }
}
 

public static FileSystem getFileSystemInHBaseCluster(String inPath) {
    Path path = new Path(inPath);
    path = Path.getPathWithoutSchemeAndAuthority(path);

    FileSystem fs = HadoopUtil.getFileSystem(path, getCurrentHBaseConfiguration()); // Must be HBase's FS, not working FS
    return fs;
}
 

public static String makeQualifiedPathInHBaseCluster(String inPath) {
    Path path = new Path(inPath);
    path = Path.getPathWithoutSchemeAndAuthority(path);

    FileSystem fs = HadoopUtil.getFileSystem(path, getCurrentHBaseConfiguration()); // Must be HBase's FS, not working FS
    return fs.makeQualified(path).toString();
}
 

private double getRealSizeByLevel(String rootPath, int level) {
    try {
        String levelPath = JobBuilderSupport.getCuboidOutputPathsByLevel(rootPath, level);
        FileSystem fs = HadoopUtil.getFileSystem(levelPath);
        return fs.getContentSummary(new Path(levelPath)).getLength() / (1024L * 1024L);
    } catch (Exception e) {
        logger.warn("get level real size failed." + e);
        return 0L;
    }
}
 

public static KylinConfig loadKylinConfigFromHdfs(String uri) {
    if (uri == null)
        throw new IllegalArgumentException("meta url should not be null");

    if (!uri.contains("@hdfs"))
        throw new IllegalArgumentException("meta url should like @hdfs schema");

    if (kylinConfigCache.get(uri) != null) {
        logger.info("KylinConfig cached for : {}", uri);
        return kylinConfigCache.get(uri);
    }

    logger.info("Ready to load KylinConfig from uri: {}", uri);
    KylinConfig config;
    FileSystem fs;
    String realHdfsPath = StorageURL.valueOf(uri).getParameter("path") + "/" + KylinConfig.KYLIN_CONF_PROPERTIES_FILE;
    try {
        fs = HadoopUtil.getFileSystem(realHdfsPath);
        InputStream is = fs.open(new Path(realHdfsPath));
        Properties prop = KylinConfig.streamToProps(is);
        config = KylinConfig.createKylinConfig(prop);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }

    kylinConfigCache.put(uri, config);
    return config;
}
 

public static KylinConfig loadKylinConfigFromHdfs(String uri) {
    if (uri == null)
        throw new IllegalArgumentException("meta url should not be null");

    if (!uri.contains("@hdfs"))
        throw new IllegalArgumentException("meta url should like @hdfs schema");

    if (kylinConfigCache.get(uri) != null) {
        logger.info("KylinConfig cached for : {}", uri);
        return kylinConfigCache.get(uri);
    }

    logger.info("Ready to load KylinConfig from uri: {}", uri);
    KylinConfig config;
    FileSystem fs;
    String realHdfsPath = StorageURL.valueOf(uri).getParameter("path") + "/" + KylinConfig.KYLIN_CONF_PROPERTIES_FILE;
    try {
        fs = HadoopUtil.getFileSystem(realHdfsPath);
        InputStream is = fs.open(new Path(realHdfsPath));
        Properties prop = KylinConfig.streamToProps(is);
        config = KylinConfig.createKylinConfig(prop);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }

    kylinConfigCache.put(uri, config);
    return config;
}
 

public void removeCubeHDFSFiles(String cubeName) {
    String segmentHDFSPath = HDFSUtil.getStreamingCubeFilePath(cubeName);
    try {
        FileSystem fs = HadoopUtil.getFileSystem(segmentHDFSPath);
        fs.delete(new Path(segmentHDFSPath), true);
    } catch (Exception e) {
        logger.error("error when remove hdfs file, hdfs path:{}", segmentHDFSPath);
    }
}
 

protected static void checkAndCreateWorkDir(String jobWorkingDir) {
    try {
        Path path = new Path(jobWorkingDir);
        FileSystem fileSystem = HadoopUtil.getFileSystem(path);
        if (!fileSystem.exists(path)) {
            logger.info("Create jobWorkDir : " + jobWorkingDir);
            fileSystem.mkdirs(path);
        }
    } catch (IOException e) {
        logger.error("Could not create lookUp table dir : " + jobWorkingDir);
    }
}
 

@Test
public void testListResourcesImpl() throws Exception {
    String path = "../examples/test_metadata/";
    String cp = new File(path).getCanonicalFile().getPath();
    FileSystem fs = HadoopUtil.getFileSystem(cp);
    HDFSResourceStore store = new HDFSResourceStore(KylinConfig.getInstanceFromEnv(),
            StorageURL.valueOf("hdfs@hdfs"));
    Field field = store.getClass().getDeclaredField("fs");
    field.setAccessible(true);
    field.set(store, fs);

    File f1 = new File(cp + "/resource/resource/e1.json");
    File f2 = new File(cp + "/resource/resource/e2.json");
    if (!f1.getParentFile().exists()) {
        if (!f1.getParentFile().mkdirs()) {
            throw new RuntimeException("Can not create dir.");
        }
    }
    if (!(f1.createNewFile() && f2.createNewFile())) {
        throw new RuntimeException("Can not create file.");
    }

    Path p = new Path(cp);
    TreeSet<String> resources = store.getAllFilePath(new Path(p, "resource"), "/resource/");
    TreeSet<String> expected = new TreeSet<>();
    expected.add("/resource/resource/e1.json");
    expected.add("/resource/resource/e2.json");
    Assert.assertEquals(expected, resources);
}
 

public GlobalDictHDFSStore(String baseDir) throws IOException {
    super(baseDir);
    this.basePath = new Path(baseDir);
    this.conf = HadoopUtil.getCurrentConfiguration();
    this.fileSystem = HadoopUtil.getFileSystem(baseDir);
}
 

@Override
protected FileSystem pushdownFS() {
    return HadoopUtil.getFileSystem(new Path("/"), HBaseConnection.getCurrentHBaseConfiguration());
}
 

@Override
public TableSignature getSignature() throws IOException {
    FileSystem fs = HadoopUtil.getFileSystem(path);
    FileStatus status = fs.getFileStatus(new Path(path));
    return new TableSignature(path, status.getLen(), status.getModificationTime());
}
 

@Override
protected FileSystem pushdownFS() {
    return HadoopUtil.getFileSystem(new Path("/"), HBaseConnection.getCurrentHBaseConfiguration());
}
 

@Override
public int run(String[] args) throws Exception {
    Options options = new Options();

    options.addOption(OPTION_INPUT_PATH);
    options.addOption(OPTION_HTABLE_NAME);
    options.addOption(OPTION_CUBE_NAME);
    parseOptions(options, args);

    String tableName = getOptionValue(OPTION_HTABLE_NAME);
    // e.g
    // /tmp/kylin-3f150b00-3332-41ca-9d3d-652f67f044d7/test_kylin_cube_with_slr_ready_2_segments/hfile/
    // end with "/"
    String input = getOptionValue(OPTION_INPUT_PATH);

    Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
    FsShell shell = new FsShell(conf);

    int exitCode = -1;
    int retryCount = 10;
    while (exitCode != 0 && retryCount >= 1) {
        exitCode = shell.run(new String[] { "-chmod", "-R", "777", input });
        retryCount--;
        Thread.sleep(5000);
    }

    if (exitCode != 0) {
        logger.error("Failed to change the file permissions: " + input);
        throw new IOException("Failed to change the file permissions: " + input);
    }

    String[] newArgs = new String[2];
    newArgs[0] = input;
    newArgs[1] = tableName;

    int count = 0;
    Path inputPath = new Path(input);
    FileSystem fs = HadoopUtil.getFileSystem(inputPath);
    FileStatus[] fileStatuses = fs.listStatus(inputPath);

    for (FileStatus fileStatus : fileStatuses) {
        if (fileStatus.isDirectory()) {
            Path path = fileStatus.getPath();
            if (path.getName().equals(FileOutputCommitter.TEMP_DIR_NAME)) {
                logger.info("Delete temporary path: " + path);
                fs.delete(path, true);
            } else {
                count++;
            }
        }
    }

    int ret = 0;
    if (count > 0) {
        logger.debug("Start to run LoadIncrementalHFiles");
        ret = MRUtil.runMRJob(new LoadIncrementalHFiles(conf), newArgs);
        logger.debug("End to run LoadIncrementalHFiles");
        return ret;
    } else {
        logger.debug("Nothing to load, cube is empty");
        return ret;
    }
}
 

protected FileSystem pushdownFS() {
    return HadoopUtil.getFileSystem(new Path(kylinConfig.getMetastoreBigCellHdfsDirectory()));
}