Java Code Examples for org.apache.kylin.job.engine.JobEngineConfig#getConfig()

The following examples show how to use org.apache.kylin.job.engine.JobEngineConfig#getConfig() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CubingJob.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private static CubingJob initCubingJob(CubeSegment seg, String jobType, String submitter, JobEngineConfig config) {
    KylinConfig kylinConfig = config.getConfig();
    CubeInstance cube = seg.getCubeInstance();
    List<ProjectInstance> projList = ProjectManager.getInstance(kylinConfig).findProjects(cube.getType(),
            cube.getName());
    if (projList == null || projList.size() == 0) {
        throw new RuntimeException("Cannot find the project containing the cube " + cube.getName() + "!!!");
    } else if (projList.size() >= 2) {
        String msg = "Find more than one project containing the cube " + cube.getName()
                + ". It does't meet the uniqueness requirement!!! ";
        if (!config.getConfig().allowCubeAppearInMultipleProjects()) {
            throw new RuntimeException(msg);
        } else {
            logger.warn(msg);
        }
    }

    CubingJob result = new CubingJob();
    SimpleDateFormat format = new SimpleDateFormat("z yyyy-MM-dd HH:mm:ss", Locale.ROOT);
    format.setTimeZone(TimeZone.getTimeZone(config.getTimeZone()));
    result.setDeployEnvName(kylinConfig.getDeployEnv());
    result.setProjectName(projList.get(0).getName());
    result.setJobType(jobType);
    CubingExecutableUtil.setCubeName(seg.getCubeInstance().getName(), result.getParams());
    CubingExecutableUtil.setSegmentId(seg.getUuid(), result.getParams());
    CubingExecutableUtil.setSegmentName(seg.getName(), result.getParams());
    result.setName(jobType + " CUBE - " + seg.getCubeInstance().getDisplayName() + " - " + seg.getName() + " - "
            + format.format(new Date(System.currentTimeMillis())));
    result.setSubmitter(submitter);
    result.setNotifyList(seg.getCubeInstance().getDescriptor().getNotifyList());
    return result;
}
Example 2
Source File: CubingJob.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private static CubingJob initCubingJob(CubeSegment seg, String jobType, String submitter, JobEngineConfig config) {
    KylinConfig kylinConfig = config.getConfig();
    CubeInstance cube = seg.getCubeInstance();
    List<ProjectInstance> projList = ProjectManager.getInstance(kylinConfig).findProjects(cube.getType(),
            cube.getName());
    if (projList == null || projList.size() == 0) {
        throw new RuntimeException("Cannot find the project containing the cube " + cube.getName() + "!!!");
    } else if (projList.size() >= 2) {
        String msg = "Find more than one project containing the cube " + cube.getName()
                + ". It does't meet the uniqueness requirement!!! ";
        if (!config.getConfig().allowCubeAppearInMultipleProjects()) {
            throw new RuntimeException(msg);
        } else {
            logger.warn(msg);
        }
    }

    CubingJob result = new CubingJob();
    SimpleDateFormat format = new SimpleDateFormat("z yyyy-MM-dd HH:mm:ss", Locale.ROOT);
    format.setTimeZone(TimeZone.getTimeZone(config.getTimeZone()));
    result.setDeployEnvName(kylinConfig.getDeployEnv());
    result.setProjectName(projList.get(0).getName());
    result.setJobType(jobType);
    CubingExecutableUtil.setCubeName(seg.getCubeInstance().getName(), result.getParams());
    CubingExecutableUtil.setSegmentId(seg.getUuid(), result.getParams());
    CubingExecutableUtil.setSegmentName(seg.getName(), result.getParams());
    result.setName(jobType + " CUBE - " + seg.getCubeInstance().getDisplayName() + " - " + seg.getName() + " - "
            + format.format(new Date(System.currentTimeMillis())));
    result.setSubmitter(submitter);
    result.setNotifyList(seg.getCubeInstance().getDescriptor().getNotifyList());
    return result;
}
Example 3
Source File: CuratorScheduler.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public void init(JobEngineConfig jobEngineConfig, JobLock jobLock) throws SchedulerException {
    kylinConfig = jobEngineConfig.getConfig();

    synchronized (this) {
        if (started) {
            logger.info("CuratorScheduler already started, skipped.");
            return;
        }

        // curatorClient can be assigned before only for test cases
        // due to creating independent curator client rather than share a cached one to avoid influences
        if (curatorClient == null) {
            curatorClient = ZKUtil.getZookeeperClient(kylinConfig);
        }

        String restAddress = kylinConfig.getServerRestAddress();

        String jobEnginePath = JOB_ENGINE_LEADER_PATH;

        if (ServerMode.isJob(jobEngineConfig.getConfig())) {
            jobClient = new CuratorLeaderSelector(curatorClient, jobEnginePath, restAddress, jobEngineConfig);
            try {
                logger.info("start Job Engine, lock path is: " + jobEnginePath);
                jobClient.start();
                monitorJobEngine();
            } catch (IOException e) {
                throw new SchedulerException(e);
            }
        } else {
            logger.info("server mode: " + jobEngineConfig.getConfig().getServerMode()
                    + ", no need to run job scheduler");
        }
        started = true;
    }
}
Example 4
Source File: DistributedScheduler.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
@Override
public synchronized void init(JobEngineConfig jobEngineConfig, JobLock jobLock) throws SchedulerException {
    String serverMode = jobEngineConfig.getConfig().getServerMode();
    if (!("job".equals(serverMode.toLowerCase(Locale.ROOT)) || "all".equals(serverMode.toLowerCase(Locale.ROOT)))) {
        logger.info("server mode: " + serverMode + ", no need to run job scheduler");
        return;
    }
    logger.info("Initializing Job Engine ....");

    if (!initialized) {
        initialized = true;
    } else {
        return;
    }

    this.jobEngineConfig = jobEngineConfig;
    this.jobLock = (DistributedLock) jobLock;
    this.serverName = this.jobLock.getClient(); // the lock's client string contains node name of this server

    executableManager = ExecutableManager.getInstance(jobEngineConfig.getConfig());
    //load all executable, set them to a consistent status
    fetcherPool = Executors.newScheduledThreadPool(1);

    //watch the zookeeper node change, so that when one job server is down, other job servers can take over.
    watchPool = Executors.newFixedThreadPool(1);
    WatcherProcessImpl watcherProcess = new WatcherProcessImpl(this.serverName);
    lockWatch = this.jobLock.watchLocks(getWatchPath(), watchPool, watcherProcess);

    int corePoolSize = jobEngineConfig.getMaxConcurrentJobLimit();
    jobPool = new ThreadPoolExecutor(corePoolSize, corePoolSize, Long.MAX_VALUE, TimeUnit.DAYS,
            new SynchronousQueue<Runnable>());
    context = new DefaultContext(Maps.<String, Executable> newConcurrentMap(), jobEngineConfig.getConfig());

    int pollSecond = jobEngineConfig.getPollIntervalSecond();
    logger.info("Fetching jobs every {} seconds", pollSecond);
    JobExecutor jobExecutor = new JobExecutor() {
        @Override
        public void execute(AbstractExecutable executable) {
            jobPool.execute(new JobRunner(executable));
        }
    };
    fetcher = jobEngineConfig.getJobPriorityConsidered()
            ? new PriorityFetcherRunner(jobEngineConfig, context, jobExecutor)
            : new DefaultFetcherRunner(jobEngineConfig, context, jobExecutor);
    fetcherPool.scheduleAtFixedRate(fetcher, pollSecond / 10, pollSecond, TimeUnit.SECONDS);
    hasStarted = true;

    resumeAllRunningJobs();
}
Example 5
Source File: DefaultScheduler.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
@Override
public synchronized void init(JobEngineConfig jobEngineConfig, JobLock lock) throws SchedulerException {
    jobLock = lock;

    String serverMode = jobEngineConfig.getConfig().getServerMode();
    if (!("job".equals(serverMode.toLowerCase(Locale.ROOT)) || "all".equals(serverMode.toLowerCase(Locale.ROOT)))) {
        logger.info("server mode: " + serverMode + ", no need to run job scheduler");
        return;
    }
    logger.info("Initializing Job Engine ....");

    if (!initialized) {
        initialized = true;
    } else {
        return;
    }

    this.jobEngineConfig = jobEngineConfig;

    if (jobLock.lockJobEngine() == false) {
        throw new IllegalStateException("Cannot start job scheduler due to lack of job lock");
    }

    //load all executable, set them to a consistent status
    fetcherPool = Executors.newScheduledThreadPool(1);
    int corePoolSize = jobEngineConfig.getMaxConcurrentJobLimit();
    jobPool = new ThreadPoolExecutor(corePoolSize, corePoolSize, Long.MAX_VALUE, TimeUnit.DAYS,
            new SynchronousQueue<Runnable>());
    context = new DefaultContext(Maps.<String, Executable> newConcurrentMap(), jobEngineConfig.getConfig());

    logger.info("Starting resume all running jobs.");
    ExecutableManager executableManager = getExecutableManager();
    executableManager.resumeAllRunningJobs();
    logger.info("Finishing resume all running jobs.");

    int pollSecond = jobEngineConfig.getPollIntervalSecond();

    logger.info("Fetching jobs every {} seconds", pollSecond);
    JobExecutor jobExecutor = new JobExecutor() {
        @Override
        public void execute(AbstractExecutable executable) {
            jobPool.execute(new JobRunner(executable));
        }
    };
    fetcher = jobEngineConfig.getJobPriorityConsidered()
            ? new PriorityFetcherRunner(jobEngineConfig, context, jobExecutor)
            : new DefaultFetcherRunner(jobEngineConfig, context, jobExecutor);
    logger.info("Creating fetcher pool instance:" + System.identityHashCode(fetcher));
    fetcherPool.scheduleAtFixedRate(fetcher, pollSecond / 10, pollSecond, TimeUnit.SECONDS);
    hasStarted = true;
}
Example 6
Source File: DistributedScheduler.java    From kylin with Apache License 2.0 4 votes vote down vote up 
@Override
public synchronized void init(JobEngineConfig jobEngineConfig, JobLock jobLock) throws SchedulerException {
    String serverMode = jobEngineConfig.getConfig().getServerMode();
    if (!("job".equals(serverMode.toLowerCase(Locale.ROOT)) || "all".equals(serverMode.toLowerCase(Locale.ROOT)))) {
        logger.info("server mode: " + serverMode + ", no need to run job scheduler");
        return;
    }
    logger.info("Initializing Job Engine ....");

    if (!initialized) {
        initialized = true;
    } else {
        return;
    }

    this.jobEngineConfig = jobEngineConfig;
    this.jobLock = (DistributedLock) jobLock;
    this.serverName = this.jobLock.getClient(); // the lock's client string contains node name of this server

    executableManager = ExecutableManager.getInstance(jobEngineConfig.getConfig());
    //load all executable, set them to a consistent status
    fetcherPool = Executors.newScheduledThreadPool(1);

    //watch the zookeeper node change, so that when one job server is down, other job servers can take over.
    watchPool = Executors.newFixedThreadPool(1);
    WatcherProcessImpl watcherProcess = new WatcherProcessImpl(this.serverName);
    lockWatch = this.jobLock.watchLocks(getWatchPath(), watchPool, watcherProcess);

    int corePoolSize = jobEngineConfig.getMaxConcurrentJobLimit();
    jobPool = new ThreadPoolExecutor(corePoolSize, corePoolSize, Long.MAX_VALUE, TimeUnit.DAYS,
            new SynchronousQueue<Runnable>());
    context = new DefaultContext(Maps.<String, Executable> newConcurrentMap(), jobEngineConfig.getConfig());

    int pollSecond = jobEngineConfig.getPollIntervalSecond();
    logger.info("Fetching jobs every {} seconds", pollSecond);
    JobExecutor jobExecutor = new JobExecutor() {
        @Override
        public void execute(AbstractExecutable executable) {
            jobPool.execute(new JobRunner(executable));
        }
    };
    fetcher = jobEngineConfig.getJobPriorityConsidered()
            ? new PriorityFetcherRunner(jobEngineConfig, context, jobExecutor)
            : new DefaultFetcherRunner(jobEngineConfig, context, jobExecutor);
    fetcherPool.scheduleAtFixedRate(fetcher, pollSecond / 10, pollSecond, TimeUnit.SECONDS);
    hasStarted = true;

    resumeAllRunningJobs();
}
Example 7
Source File: DefaultScheduler.java    From kylin with Apache License 2.0 4 votes vote down vote up 
@Override
public synchronized void init(JobEngineConfig jobEngineConfig, JobLock lock) throws SchedulerException {
    jobLock = lock;

    String serverMode = jobEngineConfig.getConfig().getServerMode();
    if (!("job".equals(serverMode.toLowerCase(Locale.ROOT)) || "all".equals(serverMode.toLowerCase(Locale.ROOT)))) {
        logger.info("server mode: " + serverMode + ", no need to run job scheduler");
        return;
    }
    logger.info("Initializing Job Engine ....");

    if (!initialized) {
        initialized = true;
    } else {
        return;
    }

    this.jobEngineConfig = jobEngineConfig;

    if (jobLock.lockJobEngine() == false) {
        throw new IllegalStateException("Cannot start job scheduler due to lack of job lock");
    }

    //load all executable, set them to a consistent status
    fetcherPool = Executors.newScheduledThreadPool(1);
    int corePoolSize = jobEngineConfig.getMaxConcurrentJobLimit();
    jobPool = new ThreadPoolExecutor(corePoolSize, corePoolSize, Long.MAX_VALUE, TimeUnit.DAYS,
            new SynchronousQueue<Runnable>());
    context = new DefaultContext(Maps.<String, Executable> newConcurrentMap(), jobEngineConfig.getConfig());

    logger.info("Starting resume all running jobs.");
    ExecutableManager executableManager = getExecutableManager();
    executableManager.resumeAllRunningJobs();
    logger.info("Finishing resume all running jobs.");

    int pollSecond = jobEngineConfig.getPollIntervalSecond();

    logger.info("Fetching jobs every {} seconds", pollSecond);
    JobExecutor jobExecutor = new JobExecutor() {
        @Override
        public void execute(AbstractExecutable executable) {
            jobPool.execute(new JobRunner(executable));
        }
    };
    fetcher = jobEngineConfig.getJobPriorityConsidered()
            ? new PriorityFetcherRunner(jobEngineConfig, context, jobExecutor)
            : new DefaultFetcherRunner(jobEngineConfig, context, jobExecutor);
    logger.info("Creating fetcher pool instance:" + System.identityHashCode(fetcher));
    fetcherPool.scheduleAtFixedRate(fetcher, pollSecond / 10, pollSecond, TimeUnit.SECONDS);
    hasStarted = true;
}