org.apache.kylin.job.execution.Executable Java Examples

The following examples show how to use org.apache.kylin.job.execution.Executable. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ExecutableManagerTest.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
private static void assertJobEqual(Executable one, Executable another) {
    assertEquals(one.getClass(), another.getClass());
    assertEquals(one.getId(), another.getId());
    assertEquals(one.getStatus(), another.getStatus());
    assertEquals(one.isRunnable(), another.isRunnable());
    assertEquals(one.getOutput(), another.getOutput());
    assertTrue((one.getParams() == null && another.getParams() == null) || (one.getParams() != null && another.getParams() != null));
    if (one.getParams() != null) {
        assertEquals(one.getParams().size(), another.getParams().size());
        for (String key : one.getParams().keySet()) {
            assertEquals(one.getParams().get(key), another.getParams().get(key));
        }
    }
    if (one instanceof ChainedExecutable) {
        assertTrue(another instanceof ChainedExecutable);
        List<? extends Executable> onesSubs = ((ChainedExecutable) one).getTasks();
        List<? extends Executable> anotherSubs = ((ChainedExecutable) another).getTasks();
        assertTrue((onesSubs == null && anotherSubs == null) || (onesSubs != null && anotherSubs != null));
        if (onesSubs != null) {
            assertEquals(onesSubs.size(), anotherSubs.size());
            for (int i = 0; i < onesSubs.size(); ++i) {
                assertJobEqual(onesSubs.get(i), anotherSubs.get(i));
            }
        }
    }
}
Example #2
Source File: ExecutableManagerTest.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private static void assertJobEqual(Executable one, Executable another) {
    assertEquals(one.getClass(), another.getClass());
    assertEquals(one.getId(), another.getId());
    assertEquals(one.getStatus(), another.getStatus());
    assertEquals(one.isRunnable(), another.isRunnable());
    assertEquals(one.getOutput(), another.getOutput());
    assertTrue((one.getParams() == null && another.getParams() == null) || (one.getParams() != null && another.getParams() != null));
    if (one.getParams() != null) {
        assertEquals(one.getParams().size(), another.getParams().size());
        for (String key : one.getParams().keySet()) {
            assertEquals(one.getParams().get(key), another.getParams().get(key));
        }
    }
    if (one instanceof ChainedExecutable) {
        assertTrue(another instanceof ChainedExecutable);
        List<? extends Executable> onesSubs = ((ChainedExecutable) one).getTasks();
        List<? extends Executable> anotherSubs = ((ChainedExecutable) another).getTasks();
        assertTrue((onesSubs == null && anotherSubs == null) || (onesSubs != null && anotherSubs != null));
        if (onesSubs != null) {
            assertEquals(onesSubs.size(), anotherSubs.size());
            for (int i = 0; i < onesSubs.size(); ++i) {
                assertJobEqual(onesSubs.get(i), anotherSubs.get(i));
            }
        }
    }
}
Example #3
Source File: ExecutableManagerTest.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private static void assertJobEqual(Executable one, Executable another) {
    assertEquals(one.getClass(), another.getClass());
    assertEquals(one.getId(), another.getId());
    assertEquals(one.getStatus(), another.getStatus());
    assertEquals(one.isRunnable(), another.isRunnable());
    assertEquals(one.getOutput(), another.getOutput());
    assertTrue((one.getParams() == null && another.getParams() == null) || (one.getParams() != null && another.getParams() != null));
    if (one.getParams() != null) {
        assertEquals(one.getParams().size(), another.getParams().size());
        for (String key : one.getParams().keySet()) {
            assertEquals(one.getParams().get(key), another.getParams().get(key));
        }
    }
    if (one instanceof ChainedExecutable) {
        assertTrue(another instanceof ChainedExecutable);
        List<? extends Executable> onesSubs = ((ChainedExecutable) one).getTasks();
        List<? extends Executable> anotherSubs = ((ChainedExecutable) another).getTasks();
        assertTrue((onesSubs == null && anotherSubs == null) || (onesSubs != null && anotherSubs != null));
        if (onesSubs != null) {
            assertEquals(onesSubs.size(), anotherSubs.size());
            for (int i = 0; i < onesSubs.size(); ++i) {
                assertJobEqual(onesSubs.get(i), anotherSubs.get(i));
            }
        }
    }
}
Example #4
Source File: DefaultScheduler.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public void run() {
    // logger.debug("Job Fetcher is running...");
    Map<String, Executable> runningJobs = context.getRunningJobs();
    if (runningJobs.size() >= jobEngineConfig.getMaxConcurrentJobLimit()) {
        logger.warn("There are too many jobs running, Job Fetch will wait until next schedule time");
        return;
    }

    int nRunning = 0, nReady = 0, nOthers = 0;
    for (final String id : executableManager.getAllJobIds()) {
        if (runningJobs.containsKey(id)) {
            // logger.debug("Job id:" + id + " is already running");
            nRunning++;
            continue;
        }
        final Output output = executableManager.getOutput(id);
        if ((output.getState() != ExecutableState.READY)) {
            // logger.debug("Job id:" + id + " not runnable");
            nOthers++;
            continue;
        }
        nReady++;
        AbstractExecutable executable = executableManager.getJob(id);
        String jobDesc = executable.toString();
        logger.info(jobDesc + " prepare to schedule");
        try {
            context.addRunningJob(executable);
            jobPool.execute(new JobRunner(executable));
            logger.info(jobDesc + " scheduled");
        } catch (Exception ex) {
            context.removeRunningJob(executable);
            logger.warn(jobDesc + " fail to schedule", ex);
        }
    }
    logger.info("Job Fetcher: " + nRunning + " running, " + runningJobs.size() + " actual running, " + nReady + " ready, " + nOthers + " others");
}
Example #5
Source File: FetcherRunner.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
protected boolean isJobPoolFull() {
    Map<String, Executable> runningJobs = context.getRunningJobs();
    if (runningJobs.size() >= jobEngineConfig.getMaxConcurrentJobLimit()) {
        logger.warn("There are too many jobs running, Job Fetch will wait until next schedule time");
        return true;
    }

    return false;
}
Example #6
Source File: UpdateSnapshotCacheForQueryServersStepTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Test
public void testExecute() throws ExecuteException {
    UpdateSnapshotCacheForQueryServersStep step = new UpdateSnapshotCacheForQueryServersStep();
    ExecuteResult result = step.doWork(new DefaultContext(Maps.<String, Executable>newConcurrentMap(), kylinConfig));
    System.out.println(result.output());
    assertTrue(result.succeed());
}
Example #7
Source File: UpdateSnapshotCacheForQueryServersStepTest.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Test
public void testExecute() throws ExecuteException {
    UpdateSnapshotCacheForQueryServersStep step = new UpdateSnapshotCacheForQueryServersStep();
    ExecuteResult result = step.doWork(new DefaultContext(Maps.<String, Executable>newConcurrentMap(), kylinConfig));
    System.out.println(result.output());
    assertTrue(result.succeed());
}
Example #8
Source File: FetcherRunner.java    From kylin with Apache License 2.0 5 votes vote down vote up 
protected boolean isJobPoolFull() {
    Map<String, Executable> runningJobs = context.getRunningJobs();
    if (runningJobs.size() >= jobEngineConfig.getMaxConcurrentJobLimit()) {
        logger.warn("There are too many jobs running, Job Fetch will wait until next schedule time");
        return true;
    }

    return false;
}
Example #9
Source File: DefaultContext.java    From kylin with Apache License 2.0 4 votes vote down vote up 
void addRunningJob(Executable executable) {
    runningJobs.put(executable.getId(), executable);
}
Example #10
Source File: DefaultContext.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
public Map<String, Executable> getRunningJobs() {
    return Collections.unmodifiableMap(runningJobs);
}
Example #11
Source File: DefaultContext.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
void removeRunningJob(Executable executable) {
    runningJobs.remove(executable.getId());
}
Example #12
Source File: DefaultContext.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
void addRunningJob(Executable executable) {
    runningJobs.put(executable.getId(), executable);
}
Example #13
Source File: DefaultContext.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
public DefaultContext(ConcurrentMap<String, Executable> runningJobs, KylinConfig kylinConfig) {
    this.runningJobs = runningJobs;
    this.kylinConfig = kylinConfig;
}
Example #14
Source File: DefaultScheduler.java    From kylin with Apache License 2.0 4 votes vote down vote up 
@Override
public synchronized void init(JobEngineConfig jobEngineConfig, JobLock lock) throws SchedulerException {
    jobLock = lock;

    String serverMode = jobEngineConfig.getConfig().getServerMode();
    if (!("job".equals(serverMode.toLowerCase(Locale.ROOT)) || "all".equals(serverMode.toLowerCase(Locale.ROOT)))) {
        logger.info("server mode: " + serverMode + ", no need to run job scheduler");
        return;
    }
    logger.info("Initializing Job Engine ....");

    if (!initialized) {
        initialized = true;
    } else {
        return;
    }

    this.jobEngineConfig = jobEngineConfig;

    if (jobLock.lockJobEngine() == false) {
        throw new IllegalStateException("Cannot start job scheduler due to lack of job lock");
    }

    //load all executable, set them to a consistent status
    fetcherPool = Executors.newScheduledThreadPool(1);
    int corePoolSize = jobEngineConfig.getMaxConcurrentJobLimit();
    jobPool = new ThreadPoolExecutor(corePoolSize, corePoolSize, Long.MAX_VALUE, TimeUnit.DAYS,
            new SynchronousQueue<Runnable>());
    context = new DefaultContext(Maps.<String, Executable> newConcurrentMap(), jobEngineConfig.getConfig());

    logger.info("Starting resume all running jobs.");
    ExecutableManager executableManager = getExecutableManager();
    executableManager.resumeAllRunningJobs();
    logger.info("Finishing resume all running jobs.");

    int pollSecond = jobEngineConfig.getPollIntervalSecond();

    logger.info("Fetching jobs every {} seconds", pollSecond);
    JobExecutor jobExecutor = new JobExecutor() {
        @Override
        public void execute(AbstractExecutable executable) {
            jobPool.execute(new JobRunner(executable));
        }
    };
    fetcher = jobEngineConfig.getJobPriorityConsidered()
            ? new PriorityFetcherRunner(jobEngineConfig, context, jobExecutor)
            : new DefaultFetcherRunner(jobEngineConfig, context, jobExecutor);
    logger.info("Creating fetcher pool instance:" + System.identityHashCode(fetcher));
    fetcherPool.scheduleAtFixedRate(fetcher, pollSecond / 10, pollSecond, TimeUnit.SECONDS);
    hasStarted = true;
}
Example #15
Source File: DefaultContext.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public Map<String, Executable> getRunningJobs() {
    return Collections.unmodifiableMap(runningJobs);
}
Example #16
Source File: DefaultContext.java    From kylin with Apache License 2.0 4 votes vote down vote up 
void removeRunningJob(Executable executable) {
    runningJobs.remove(executable.getId());
}
Example #17
Source File: DefaultFetcherRunner.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
@Override
synchronized public void run() {
    try (SetThreadName ignored = new SetThreadName(//
            "FetcherRunner %s", System.identityHashCode(this))) {//
        // logger.debug("Job Fetcher is running...");
        Map<String, Executable> runningJobs = context.getRunningJobs();
        if (isJobPoolFull()) {
            return;
        }

        nRunning = 0;
        nReady = 0;
        nStopped = 0;
        nOthers = 0;
        nError = 0;
        nDiscarded = 0;
        nSUCCEED = 0;
        for (final String id : getExecutableManager().getAllJobIdsInCache()) {
            if (isJobPoolFull()) {
                return;
            }
            if (runningJobs.containsKey(id)) {
                // logger.debug("Job id:" + id + " is already running");
                nRunning++;
                continue;
            }
            if (succeedJobs.contains(id)) {
                nSUCCEED++;
                continue;
            }

            final Output outputDigest;
            try {
                outputDigest = getExecutableManager().getOutputDigest(id);
            } catch (IllegalArgumentException e) {
                logger.warn("job " + id + " output digest is null, skip.", e);
                nOthers++;
                continue;
            }
            if ((outputDigest.getState() != ExecutableState.READY)) {
                // logger.debug("Job id:" + id + " not runnable");
                jobStateCount(id);
                continue;
            }

            final AbstractExecutable executable = getExecutableManager().getJob(id);
            if (executable == null) {
                logger.info("job " + id + " get job is null, skip.");
                nOthers++;
                continue;
            }
            if (!executable.isReady()) {
                nOthers++;
                continue;
            }

            KylinConfig config = jobEngineConfig.getConfig();
            if (config.isSchedulerSafeMode()) {
                String cubeName = executable.getCubeName();
                String projectName = CubeManager.getInstance(config).getCube(cubeName).getProject();
                if (!config.getSafeModeRunnableProjects().contains(projectName) &&
                        executable.getStartTime() == 0) {
                    logger.info("New job is pending for scheduler in safe mode. Project: {}, job: {}",
                            projectName, executable.getName());
                    continue;
                }
            }

            nReady++;
            addToJobPool(executable, executable.getDefaultPriority());
        }

        fetchFailed = false;
        logger.info("Job Fetcher: " + nRunning + " should running, " + runningJobs.size() + " actual running, "
                + nStopped + " stopped, " + nReady + " ready, " + nSUCCEED + " already succeed, " + nError
                + " error, " + nDiscarded + " discarded, " + nOthers + " others");
    } catch (Throwable th) {
        fetchFailed = true; // this could happen when resource store is unavailable
        logger.warn("Job Fetcher caught a exception ", th);
    }
}
Example #18
Source File: DefaultContext.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public DefaultContext(ConcurrentMap<String, Executable> runningJobs, KylinConfig kylinConfig) {
    this.runningJobs = runningJobs;
    this.kylinConfig = kylinConfig;
}
Example #19
Source File: DistributedScheduler.java    From kylin with Apache License 2.0 4 votes vote down vote up 
@Override
public synchronized void init(JobEngineConfig jobEngineConfig, JobLock jobLock) throws SchedulerException {
    String serverMode = jobEngineConfig.getConfig().getServerMode();
    if (!("job".equals(serverMode.toLowerCase(Locale.ROOT)) || "all".equals(serverMode.toLowerCase(Locale.ROOT)))) {
        logger.info("server mode: " + serverMode + ", no need to run job scheduler");
        return;
    }
    logger.info("Initializing Job Engine ....");

    if (!initialized) {
        initialized = true;
    } else {
        return;
    }

    this.jobEngineConfig = jobEngineConfig;
    this.jobLock = (DistributedLock) jobLock;
    this.serverName = this.jobLock.getClient(); // the lock's client string contains node name of this server

    executableManager = ExecutableManager.getInstance(jobEngineConfig.getConfig());
    //load all executable, set them to a consistent status
    fetcherPool = Executors.newScheduledThreadPool(1);

    //watch the zookeeper node change, so that when one job server is down, other job servers can take over.
    watchPool = Executors.newFixedThreadPool(1);
    WatcherProcessImpl watcherProcess = new WatcherProcessImpl(this.serverName);
    lockWatch = this.jobLock.watchLocks(getWatchPath(), watchPool, watcherProcess);

    int corePoolSize = jobEngineConfig.getMaxConcurrentJobLimit();
    jobPool = new ThreadPoolExecutor(corePoolSize, corePoolSize, Long.MAX_VALUE, TimeUnit.DAYS,
            new SynchronousQueue<Runnable>());
    context = new DefaultContext(Maps.<String, Executable> newConcurrentMap(), jobEngineConfig.getConfig());

    int pollSecond = jobEngineConfig.getPollIntervalSecond();
    logger.info("Fetching jobs every {} seconds", pollSecond);
    JobExecutor jobExecutor = new JobExecutor() {
        @Override
        public void execute(AbstractExecutable executable) {
            jobPool.execute(new JobRunner(executable));
        }
    };
    fetcher = jobEngineConfig.getJobPriorityConsidered()
            ? new PriorityFetcherRunner(jobEngineConfig, context, jobExecutor)
            : new DefaultFetcherRunner(jobEngineConfig, context, jobExecutor);
    fetcherPool.scheduleAtFixedRate(fetcher, pollSecond / 10, pollSecond, TimeUnit.SECONDS);
    hasStarted = true;

    resumeAllRunningJobs();
}
Example #20
Source File: DefaultFetcherRunner.java    From kylin with Apache License 2.0 4 votes vote down vote up 
@Override
synchronized public void run() {
    try (SetThreadName ignored = new SetThreadName(//
            "FetcherRunner %s", System.identityHashCode(this))) {//
        // logger.debug("Job Fetcher is running...");
        Map<String, Executable> runningJobs = context.getRunningJobs();
        if (isJobPoolFull()) {
            return;
        }

        nRunning = 0;
        nReady = 0;
        nStopped = 0;
        nOthers = 0;
        nError = 0;
        nDiscarded = 0;
        nSUCCEED = 0;
        for (final String id : getExecutableManager().getAllJobIdsInCache()) {
            if (isJobPoolFull()) {
                return;
            }
            if (runningJobs.containsKey(id)) {
                // logger.debug("Job id:" + id + " is already running");
                nRunning++;
                continue;
            }
            if (succeedJobs.contains(id)) {
                nSUCCEED++;
                continue;
            }

            final Output outputDigest;
            try {
                outputDigest = getExecutableManager().getOutputDigest(id);
            } catch (IllegalArgumentException e) {
                logger.warn("job " + id + " output digest is null, skip.", e);
                nOthers++;
                continue;
            }
            if ((outputDigest.getState() != ExecutableState.READY)) {
                // logger.debug("Job id:" + id + " not runnable");
                jobStateCount(id);
                continue;
            }

            final AbstractExecutable executable = getExecutableManager().getJob(id);
            if (executable == null) {
                logger.info("job " + id + " get job is null, skip.");
                nOthers++;
                continue;
            }
            if (!executable.isReady()) {
                nOthers++;
                continue;
            }

            KylinConfig config = jobEngineConfig.getConfig();
            if (config.isSchedulerSafeMode()) {
                String cubeName = executable.getCubeName();
                String projectName = CubeManager.getInstance(config).getCube(cubeName).getProject();
                if (!config.getSafeModeRunnableProjects().contains(projectName) &&
                        executable.getStartTime() == 0) {
                    logger.info("New job is pending for scheduler in safe mode. Project: {}, job: {}",
                            projectName, executable.getName());
                    continue;
                }
            }

            nReady++;
            addToJobPool(executable, executable.getDefaultPriority());
        }

        fetchFailed = false;
        logger.info("Job Fetcher: " + nRunning + " should running, " + runningJobs.size() + " actual running, "
                + nStopped + " stopped, " + nReady + " ready, " + nSUCCEED + " already succeed, " + nError
                + " error, " + nDiscarded + " discarded, " + nOthers + " others");
    } catch (Throwable th) {
        fetchFailed = true; // this could happen when resource store is unavailable
        logger.warn("Job Fetcher caught a exception ", th);
    }
}
Example #21
Source File: DefaultScheduler.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
@Override
public synchronized void init(JobEngineConfig jobEngineConfig, JobLock lock) throws SchedulerException {
    jobLock = lock;

    String serverMode = jobEngineConfig.getConfig().getServerMode();
    if (!("job".equals(serverMode.toLowerCase(Locale.ROOT)) || "all".equals(serverMode.toLowerCase(Locale.ROOT)))) {
        logger.info("server mode: " + serverMode + ", no need to run job scheduler");
        return;
    }
    logger.info("Initializing Job Engine ....");

    if (!initialized) {
        initialized = true;
    } else {
        return;
    }

    this.jobEngineConfig = jobEngineConfig;

    if (jobLock.lockJobEngine() == false) {
        throw new IllegalStateException("Cannot start job scheduler due to lack of job lock");
    }

    //load all executable, set them to a consistent status
    fetcherPool = Executors.newScheduledThreadPool(1);
    int corePoolSize = jobEngineConfig.getMaxConcurrentJobLimit();
    jobPool = new ThreadPoolExecutor(corePoolSize, corePoolSize, Long.MAX_VALUE, TimeUnit.DAYS,
            new SynchronousQueue<Runnable>());
    context = new DefaultContext(Maps.<String, Executable> newConcurrentMap(), jobEngineConfig.getConfig());

    logger.info("Starting resume all running jobs.");
    ExecutableManager executableManager = getExecutableManager();
    executableManager.resumeAllRunningJobs();
    logger.info("Finishing resume all running jobs.");

    int pollSecond = jobEngineConfig.getPollIntervalSecond();

    logger.info("Fetching jobs every {} seconds", pollSecond);
    JobExecutor jobExecutor = new JobExecutor() {
        @Override
        public void execute(AbstractExecutable executable) {
            jobPool.execute(new JobRunner(executable));
        }
    };
    fetcher = jobEngineConfig.getJobPriorityConsidered()
            ? new PriorityFetcherRunner(jobEngineConfig, context, jobExecutor)
            : new DefaultFetcherRunner(jobEngineConfig, context, jobExecutor);
    logger.info("Creating fetcher pool instance:" + System.identityHashCode(fetcher));
    fetcherPool.scheduleAtFixedRate(fetcher, pollSecond / 10, pollSecond, TimeUnit.SECONDS);
    hasStarted = true;
}
Example #22
Source File: DefaultContext.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public Map<String, Executable> getRunningJobs() {
    return Collections.unmodifiableMap(runningJobs);
}
Example #23
Source File: DefaultContext.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
void removeRunningJob(Executable executable) {
    runningJobs.remove(executable.getId());
}
Example #24
Source File: DefaultContext.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
void addRunningJob(Executable executable) {
    runningJobs.put(executable.getId(), executable);
}
Example #25
Source File: DefaultContext.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public DefaultContext(ConcurrentMap<String, Executable> runningJobs, KylinConfig kylinConfig) {
    this.runningJobs = runningJobs;
    this.kylinConfig = kylinConfig;
}
Example #26
Source File: DistributedScheduler.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
@Override
public synchronized void init(JobEngineConfig jobEngineConfig, JobLock jobLock) throws SchedulerException {
    String serverMode = jobEngineConfig.getConfig().getServerMode();
    if (!("job".equals(serverMode.toLowerCase(Locale.ROOT)) || "all".equals(serverMode.toLowerCase(Locale.ROOT)))) {
        logger.info("server mode: " + serverMode + ", no need to run job scheduler");
        return;
    }
    logger.info("Initializing Job Engine ....");

    if (!initialized) {
        initialized = true;
    } else {
        return;
    }

    this.jobEngineConfig = jobEngineConfig;
    this.jobLock = (DistributedLock) jobLock;
    this.serverName = this.jobLock.getClient(); // the lock's client string contains node name of this server

    executableManager = ExecutableManager.getInstance(jobEngineConfig.getConfig());
    //load all executable, set them to a consistent status
    fetcherPool = Executors.newScheduledThreadPool(1);

    //watch the zookeeper node change, so that when one job server is down, other job servers can take over.
    watchPool = Executors.newFixedThreadPool(1);
    WatcherProcessImpl watcherProcess = new WatcherProcessImpl(this.serverName);
    lockWatch = this.jobLock.watchLocks(getWatchPath(), watchPool, watcherProcess);

    int corePoolSize = jobEngineConfig.getMaxConcurrentJobLimit();
    jobPool = new ThreadPoolExecutor(corePoolSize, corePoolSize, Long.MAX_VALUE, TimeUnit.DAYS,
            new SynchronousQueue<Runnable>());
    context = new DefaultContext(Maps.<String, Executable> newConcurrentMap(), jobEngineConfig.getConfig());

    int pollSecond = jobEngineConfig.getPollIntervalSecond();
    logger.info("Fetching jobs every {} seconds", pollSecond);
    JobExecutor jobExecutor = new JobExecutor() {
        @Override
        public void execute(AbstractExecutable executable) {
            jobPool.execute(new JobRunner(executable));
        }
    };
    fetcher = jobEngineConfig.getJobPriorityConsidered()
            ? new PriorityFetcherRunner(jobEngineConfig, context, jobExecutor)
            : new DefaultFetcherRunner(jobEngineConfig, context, jobExecutor);
    fetcherPool.scheduleAtFixedRate(fetcher, pollSecond / 10, pollSecond, TimeUnit.SECONDS);
    hasStarted = true;

    resumeAllRunningJobs();
}