Java Code Examples for org.apache.kylin.common.persistence.ResourceStore#EXECUTE_OUTPUT_RESOURCE_ROOT

The following examples show how to use org.apache.kylin.common.persistence.ResourceStore#EXECUTE_OUTPUT_RESOURCE_ROOT . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ExecutableDao.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
private ExecutableDao(KylinConfig config) throws IOException {
    logger.info("Using metadata url: {}", config);
    this.store = ResourceStore.getStore(config);
    this.executableDigestMap = new CaseInsensitiveStringCache<>(config, "execute");
    this.executableDigestCrud = new CachedCrudAssist<ExecutablePO>(store, ResourceStore.EXECUTE_RESOURCE_ROOT, "",
            ExecutablePO.class, executableDigestMap, false) {
        @Override
        public ExecutablePO reloadAt(String path) {
            try {
                ExecutablePO executablePO = readJobResource(path);
                if (executablePO == null) {
                    logger.warn("No job found at {}, returning null", path);
                    executableDigestMap.removeLocal(resourceName(path));
                    return null;
                }

                // create a digest
                ExecutablePO digestExecutablePO = new ExecutablePO();
                digestExecutablePO.setUuid(executablePO.getUuid());
                digestExecutablePO.setName(executablePO.getName());
                digestExecutablePO.setLastModified(executablePO.getLastModified());
                digestExecutablePO.setType(executablePO.getType());
                digestExecutablePO.setParams(executablePO.getParams());
                executableDigestMap.putLocal(resourceName(path), digestExecutablePO);
                return digestExecutablePO;
            } catch (Exception e) {
                throw new IllegalStateException("Error loading execute at " + path, e);
            }
        }

        @Override
        protected ExecutablePO initEntityAfterReload(ExecutablePO entity, String resourceName) {
            return entity;
        }
    };
    this.executableDigestCrud.setCheckCopyOnWrite(true);
    this.executableDigestCrud.reloadAll();

    this.executableOutputDigestMap = new CaseInsensitiveStringCache<>(config, "execute_output");
    this.executableOutputDigestCrud = new CachedCrudAssist<ExecutableOutputPO>(store,
            ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT, "", ExecutableOutputPO.class, executableOutputDigestMap,
            false) {
        @Override
        public void reloadAll() throws IOException {
            logger.debug("Reloading execute_output from " + ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT);
            executableOutputDigestMap.clear();

            NavigableSet<String> paths = store.listResources(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT);

            if (paths != null) {
                for (String path : paths) {
                    if (!isTaskExecutableOutput(resourceName(path)))
                        reloadAt(path);
                }

                logger.debug("Loaded {} execute_output digest(s) out of {} resource",
                        executableOutputDigestMap.size(), paths.size());
            }
        }

        @Override
        public ExecutableOutputPO reloadAt(String path) {
            try {
                ExecutableOutputPO executableOutputPO = readJobOutputResource(path);
                if (executableOutputPO == null) {
                    logger.warn("No job output found at {}, returning null", path);
                    executableOutputDigestMap.removeLocal(resourceName(path));
                    return null;
                }

                // create a digest
                ExecutableOutputPO digestExecutableOutputPO = new ExecutableOutputPO();
                digestExecutableOutputPO.setUuid(executableOutputPO.getUuid());
                digestExecutableOutputPO.setLastModified(executableOutputPO.getLastModified());
                digestExecutableOutputPO.setStatus(executableOutputPO.getStatus());
                executableOutputDigestMap.putLocal(resourceName(path), digestExecutableOutputPO);
                return digestExecutableOutputPO;
            } catch (Exception e) {
                throw new IllegalStateException("Error loading execute at " + path, e);
            }
        }

        @Override
        protected ExecutableOutputPO initEntityAfterReload(ExecutableOutputPO entity, String resourceName) {
            return entity;
        }
    };
    this.executableOutputDigestCrud.setCheckCopyOnWrite(true);
    this.executableOutputDigestCrud.reloadAll();
    Broadcaster.getInstance(config).registerListener(new JobSyncListener(), "execute");
    Broadcaster.getInstance(config).registerListener(new JobOutputSyncListener(), "execute_output");
}
Example 2
Source File: ExecutableDao.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public static String pathOfJobOutput(String uuid) {
    return ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + uuid;
}
Example 3
Source File: ExecutableDao.java    From kylin with Apache License 2.0 4 votes vote down vote up 
private ExecutableDao(KylinConfig config) throws IOException {
    logger.info("Using metadata url: {}", config);
    this.store = ResourceStore.getStore(config);
    this.executableDigestMap = new CaseInsensitiveStringCache<>(config, "execute");
    this.executableDigestCrud = new CachedCrudAssist<ExecutablePO>(store, ResourceStore.EXECUTE_RESOURCE_ROOT, "",
            ExecutablePO.class, executableDigestMap, false) {
        @Override
        public ExecutablePO reloadAt(String path) {
            try {
                ExecutablePO executablePO = readJobResource(path);
                if (executablePO == null) {
                    logger.warn("No job found at {}, returning null", path);
                    executableDigestMap.removeLocal(resourceName(path));
                    return null;
                }

                // create a digest
                ExecutablePO digestExecutablePO = new ExecutablePO();
                digestExecutablePO.setUuid(executablePO.getUuid());
                digestExecutablePO.setName(executablePO.getName());
                digestExecutablePO.setLastModified(executablePO.getLastModified());
                digestExecutablePO.setType(executablePO.getType());
                digestExecutablePO.setParams(executablePO.getParams());
                executableDigestMap.putLocal(resourceName(path), digestExecutablePO);
                return digestExecutablePO;
            } catch (Exception e) {
                throw new IllegalStateException("Error loading execute at " + path, e);
            }
        }

        @Override
        protected ExecutablePO initEntityAfterReload(ExecutablePO entity, String resourceName) {
            return entity;
        }
    };
    this.executableDigestCrud.setCheckCopyOnWrite(true);
    this.executableDigestCrud.reloadAll();

    this.executableOutputDigestMap = new CaseInsensitiveStringCache<>(config, "execute_output");
    this.executableOutputDigestCrud = new CachedCrudAssist<ExecutableOutputPO>(store,
            ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT, "", ExecutableOutputPO.class, executableOutputDigestMap,
            false) {
        @Override
        public void reloadAll() throws IOException {
            logger.debug("Reloading execute_output from " + ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT);
            executableOutputDigestMap.clear();

            NavigableSet<String> paths = store.listResources(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT);

            if (paths != null) {
                for (String path : paths) {
                    if (!isTaskExecutableOutput(resourceName(path)))
                        reloadAt(path);
                }

                logger.debug("Loaded {} execute_output digest(s) out of {} resource",
                        executableOutputDigestMap.size(), paths.size());
            }
        }

        @Override
        public ExecutableOutputPO reloadAt(String path) {
            try {
                ExecutableOutputPO executableOutputPO = readJobOutputResource(path);
                if (executableOutputPO == null) {
                    logger.warn("No job output found at {}, returning null", path);
                    executableOutputDigestMap.removeLocal(resourceName(path));
                    return null;
                }

                // create a digest
                ExecutableOutputPO digestExecutableOutputPO = new ExecutableOutputPO();
                digestExecutableOutputPO.setUuid(executableOutputPO.getUuid());
                digestExecutableOutputPO.setLastModified(executableOutputPO.getLastModified());
                digestExecutableOutputPO.setStatus(executableOutputPO.getStatus());
                executableOutputDigestMap.putLocal(resourceName(path), digestExecutableOutputPO);
                return digestExecutableOutputPO;
            } catch (Exception e) {
                throw new IllegalStateException("Error loading execute at " + path, e);
            }
        }

        @Override
        protected ExecutableOutputPO initEntityAfterReload(ExecutableOutputPO entity, String resourceName) {
            return entity;
        }
    };
    this.executableOutputDigestCrud.setCheckCopyOnWrite(true);
    this.executableOutputDigestCrud.reloadAll();
    Broadcaster.getInstance(config).registerListener(new JobSyncListener(), "execute");
    Broadcaster.getInstance(config).registerListener(new JobOutputSyncListener(), "execute_output");
}
Example 4
Source File: ExecutableDao.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public static String pathOfJobOutput(String uuid) {
    return ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + uuid;
}