Java Code Examples for org.apache.kylin.common.persistence.ResourceStore#collectResourceRecursively()

The following examples show how to use org.apache.kylin.common.persistence.ResourceStore#collectResourceRecursively() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MetadataManager.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
private void reloadAllDataModel() throws IOException {
    ResourceStore store = getStore();
    logger.debug("Reloading DataModel from folder " + store.getReadableResourcePath(ResourceStore.DATA_MODEL_DESC_RESOURCE_ROOT));

    dataModelDescMap.clear();

    List<String> paths = store.collectResourceRecursively(ResourceStore.DATA_MODEL_DESC_RESOURCE_ROOT, MetadataConstants.FILE_SURFIX);
    for (String path : paths) {
        try {
            reloadDataModelDescAt(path);
        } catch (IllegalStateException e) {
            logger.error("Error to load DataModel at " + path, e);
            continue;
        }
    }

    logger.debug("Loaded " + dataModelDescMap.size() + " DataModel(s)");
}
Example 2
Source File: StreamingSourceConfigManager.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public List<StreamingSourceConfig> listAllStreaming() throws IOException {
    List<StreamingSourceConfig> results = Lists.newArrayList();
    ResourceStore store = getStore();
    logger.info("Load all streaming metadata from folder "
            + store.getReadableResourcePath(ResourceStore.STREAMING_V2_RESOURCE_ROOT));

    List<String> paths = store.collectResourceRecursively(ResourceStore.STREAMING_V2_RESOURCE_ROOT,
            MetadataConstants.FILE_SURFIX);
    for (String path : paths) {
        StreamingSourceConfig streamingSourceConfig;
        try {
            streamingSourceConfig = loadStreamingConfigAt(path);
        } catch (Exception e) {
            logger.error("Error loading streaming desc " + path, e);
            continue;
        }
        if (path.equals(streamingSourceConfig.getResourcePath()) == false) {
            logger.error("Skip suspicious desc at " + path + ", " + streamingSourceConfig + " should be at "
                    + streamingSourceConfig.getResourcePath());
            continue;
        }
        results.add(streamingSourceConfig);
    }

    logger.debug("Loaded " + results.size() + " StreamingSourceConfig(s)");
    return results;
}
Example 3
Source File: StreamingSourceConfigManager.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public List<StreamingSourceConfig> listAllStreaming() throws IOException {
    List<StreamingSourceConfig> results = Lists.newArrayList();
    ResourceStore store = getStore();
    logger.info("Load all streaming metadata from folder "
            + store.getReadableResourcePath(ResourceStore.STREAMING_V2_RESOURCE_ROOT));

    List<String> paths = store.collectResourceRecursively(ResourceStore.STREAMING_V2_RESOURCE_ROOT,
            MetadataConstants.FILE_SURFIX);
    for (String path : paths) {
        StreamingSourceConfig streamingSourceConfig;
        try {
            streamingSourceConfig = loadStreamingConfigAt(path);
        } catch (Exception e) {
            logger.error("Error loading streaming desc " + path, e);
            continue;
        }
        if (path.equals(streamingSourceConfig.getResourcePath()) == false) {
            logger.error("Skip suspicious desc at " + path + ", " + streamingSourceConfig + " should be at "
                    + streamingSourceConfig.getResourcePath());
            continue;
        }
        results.add(streamingSourceConfig);
    }

    logger.debug("Loaded " + results.size() + " StreamingSourceConfig(s)");
    return results;
}
Example 4
Source File: ProjectManager.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private void reloadAllProjects() throws IOException {
    ResourceStore store = getStore();
    List<String> paths = store.collectResourceRecursively(ResourceStore.PROJECT_RESOURCE_ROOT, ".json");

    logger.debug("Loading Project from folder " + store.getReadableResourcePath(ResourceStore.PROJECT_RESOURCE_ROOT));

    for (String path : paths) {
        reloadProjectAt(path);
    }
    wireProjectAndRealizations(projectMap.values());
    logger.debug("Loaded " + projectMap.size() + " Project(s)");
}
Example 5
Source File: MetadataManager.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private void reloadAllSourceTableExd() throws IOException {
    ResourceStore store = getStore();
    logger.debug("Reloading SourceTable exd info from folder " + store.getReadableResourcePath(ResourceStore.TABLE_EXD_RESOURCE_ROOT));

    srcTableExdMap.clear();

    List<String> paths = store.collectResourceRecursively(ResourceStore.TABLE_EXD_RESOURCE_ROOT, MetadataConstants.FILE_SURFIX);
    for (String path : paths) {
        reloadSourceTableExdAt(path);
    }

    logger.debug("Loaded " + srcTableExdMap.size() + " SourceTable EXD(s)");
}
Example 6
Source File: MetadataManager.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private void reloadAllSourceTable() throws IOException {
    ResourceStore store = getStore();
    logger.debug("Reloading SourceTable from folder " + store.getReadableResourcePath(ResourceStore.TABLE_RESOURCE_ROOT));

    srcTableMap.clear();

    List<String> paths = store.collectResourceRecursively(ResourceStore.TABLE_RESOURCE_ROOT, MetadataConstants.FILE_SURFIX);
    for (String path : paths) {
        reloadSourceTableAt(path);
    }

    logger.debug("Loaded " + srcTableMap.size() + " SourceTable(s)");
}
Example 7
Source File: CubeManager.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private void loadAllCubeInstance() throws IOException {
    ResourceStore store = getStore();
    List<String> paths = store.collectResourceRecursively(ResourceStore.CUBE_RESOURCE_ROOT, ".json");

    logger.debug("Loading Cube from folder " + store.getReadableResourcePath(ResourceStore.CUBE_RESOURCE_ROOT));

    for (String path : paths) {
        loadCubeInstance(path);
    }

    logger.debug("Loaded " + paths.size() + " Cube(s)");
}
Example 8
Source File: CubeDescManager.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private void reloadAllCubeDesc() throws IOException {
    ResourceStore store = getStore();
    logger.info("Reloading Cube Metadata from folder " + store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT));

    cubeDescMap.clear();

    List<String> paths = store.collectResourceRecursively(ResourceStore.CUBE_DESC_RESOURCE_ROOT, MetadataConstants.FILE_SURFIX);
    for (String path : paths) {
        CubeDesc desc;
        try {
            desc = loadCubeDesc(path);
        } catch (Exception e) {
            logger.error("Error loading cube desc " + path, e);
            continue;
        }
        if (path.equals(desc.getResourcePath()) == false) {
            logger.error("Skip suspicious desc at " + path + ", " + desc + " should be at " + desc.getResourcePath());
            continue;
        }
        if (cubeDescMap.containsKey(desc.getName())) {
            logger.error("Dup CubeDesc name '" + desc.getName() + "' on path " + path);
            continue;
        }

        cubeDescMap.putLocal(desc.getName(), desc);
    }

    logger.debug("Loaded " + cubeDescMap.size() + " Cube(s)");
}
Example 9
Source File: IIDescManager.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private void reloadAllIIDesc() throws IOException {
    ResourceStore store = getStore();
    logger.info("Reloading all II desc from folder " + store.getReadableResourcePath(ResourceStore.II_DESC_RESOURCE_ROOT));

    iiDescMap.clear();

    List<String> paths = store.collectResourceRecursively(ResourceStore.II_DESC_RESOURCE_ROOT, MetadataConstants.FILE_SURFIX);
    for (String path : paths) {
        logger.info("loading II Desc from path" + path);
        IIDesc desc;
        try {
            desc = loadIIDesc(path);
        } catch (Exception e) {
            logger.error("Error loading II desc " + path, e);
            continue;
        }
        if (!path.equals(desc.getResourcePath())) {
            logger.error("Skip suspicious desc at " + path + ", " + desc + " should be at " + desc.getResourcePath());
            continue;
        }
        if (iiDescMap.containsKey(desc.getName())) {
            logger.error("Dup IIDesc name '" + desc.getName() + "' on path " + path);
            continue;
        }

        iiDescMap.putLocal(desc.getName(), desc);
    }

    logger.debug("Loaded " + iiDescMap.size() + " II desc(s)");
}
Example 10
Source File: IIManager.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private void loadAllIIInstance() throws IOException {
    ResourceStore store = getStore();
    List<String> paths = store.collectResourceRecursively(ResourceStore.II_RESOURCE_ROOT, ".json");

    logger.debug("Loading II from folder " + store.getReadableResourcePath(ResourceStore.II_RESOURCE_ROOT));

    for (String path : paths) {
        loadIIInstance(path);
    }

    logger.debug("Loaded " + paths.size() + " II(s)");
}