org.apache.kylin.job.exception.SchedulerException Java Examples

@Override
public void shutdown() throws SchedulerException {
    logger.info("Will shut down Job Engine ....");

    try {
        lockWatch.close();
    } catch (IOException e) {
        throw new SchedulerException(e);
    }

    releaseAllLocks();
    logger.info("The all locks has released");

    fetcherPool.shutdown();
    logger.info("The fetcherPool has down");

    jobPool.shutdown();
    logger.info("The jobPoll has down");
}
 

@Override
public void shutdown() throws SchedulerException {
    logger.info("Will shut down Job Engine ....");

    try {
        lockWatch.close();
    } catch (IOException e) {
        throw new SchedulerException(e);
    }

    releaseAllLocks();
    logger.info("The all locks has released");

    fetcherPool.shutdown();
    logger.info("The fetcherPool has down");

    jobPool.shutdown();
    logger.info("The jobPoll has down");
}
 

@Before
public void setup() throws SchedulerException {
    this.createTestMetadata("../../examples/test_case_data/file_prunning");
    System.setProperty("kylin.env", "UT");
    System.setProperty("kylin.query.enable-dynamic-column", "false");
    Map<RealizationType, Integer> priorities = Maps.newHashMap();
    priorities.put(RealizationType.HYBRID, 0);
    priorities.put(RealizationType.CUBE, 0);
    Candidate.setPriorities(priorities);
    overwriteSystemProp("kylin.job.scheduler.poll-interval-second", "1");
    overwriteSystemProp("calcite.keep-in-clause", "true");
    overwriteSystemProp("kylin.metadata.distributed-lock-impl", "org.apache.kylin.engine.spark.utils.MockedDistributedLock$MockedFactory");
    DefaultScheduler scheduler = DefaultScheduler.getInstance();
    scheduler.init(new JobEngineConfig(KylinConfig.getInstanceFromEnv()), new MockJobLock());
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
    config = KylinConfig.getInstanceFromEnv();
    cubeMgr = CubeManager.getInstance(config);
    execMgr = ExecutableManager.getInstance(config);
}
 

@Override
public void shutdown() throws SchedulerException {
    IOUtils.closeQuietly(serviceCache);
    IOUtils.closeQuietly(serviceDiscovery);
    IOUtils.closeQuietly(curatorClient);
    IOUtils.closeQuietly(jobClient);
    started = false;
}
 

@Override
public void init(JobEngineConfig jobEngineConfig, JobLock jobLock) throws SchedulerException {
    kylinConfig = jobEngineConfig.getConfig();

    synchronized (this) {
        if (started) {
            logger.info("CuratorScheduler already started, skipped.");
            return;
        }

        // curatorClient can be assigned before only for test cases
        // due to creating independent curator client rather than share a cached one to avoid influences
        if (curatorClient == null) {
            curatorClient = ZKUtil.getZookeeperClient(kylinConfig);
        }

        String restAddress = kylinConfig.getServerRestAddress();

        String jobEnginePath = JOB_ENGINE_LEADER_PATH;

        if (ServerMode.isJob(jobEngineConfig.getConfig())) {
            jobClient = new CuratorLeaderSelector(curatorClient, jobEnginePath, restAddress, jobEngineConfig);
            try {
                logger.info("start Job Engine, lock path is: " + jobEnginePath);
                jobClient.start();
                monitorJobEngine();
            } catch (IOException e) {
                throw new SchedulerException(e);
            }
        } else {
            logger.info("server mode: " + jobEngineConfig.getConfig().getServerMode()
                    + ", no need to run job scheduler");
        }
        started = true;
    }
}
 

@Override
public void setup() throws SchedulerException {
    super.setup();
    overwriteSystemProp("kylin.env", "UT");
    overwriteSystemProp("isDeveloperMode", "true");
    overwriteSystemProp("kylin.query.enable-dynamic-column", "false");
    Map<RealizationType, Integer> priorities = Maps.newHashMap();
    priorities.put(RealizationType.HYBRID, 0);
    priorities.put(RealizationType.CUBE, 0);
    Candidate.setPriorities(priorities);
    config = KylinConfig.getInstanceFromEnv();
    cubeMgr = CubeManager.getInstance(config);
    execMgr = ExecutableManager.getInstance(config);
}
 

@Override
public void setup() throws SchedulerException {
    super.setup();
    config = KylinConfig.getInstanceFromEnv();
    cubeMgr = CubeManager.getInstance(config);
    execMgr = ExecutableManager.getInstance(config);
}
 

@Override
public void setup() throws SchedulerException {
    super.setup();
    System.setProperty("spark.local", "true");
    System.setProperty("noBuild", "false");
    System.setProperty("isDeveloperMode", "false");
}
 

@Before
public void setup() throws SchedulerException {
    overwriteSystemProp("kylin.job.scheduler.poll-interval-second", "1");
    overwriteSystemProp("calcite.keep-in-clause", "true");
    overwriteSystemProp("kylin.metadata.distributed-lock-impl", "org.apache.kylin.engine.spark.utils.MockedDistributedLock$MockedFactory");
    this.createTestMetadata();
    DefaultScheduler scheduler = DefaultScheduler.getInstance();
    scheduler.init(new JobEngineConfig(KylinConfig.getInstanceFromEnv()), new MockJobLock());
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
}
 

@Override
public void setup() throws SchedulerException {
    super.setup();
    kylinConfig = KylinConfig.getInstanceFromEnv();
    kylinConfig.setProperty("kylin.source.provider.0", "org.apache.kylin.engine.spark.source.HiveSource");
    cubeMgr = CubeManager.getInstance(kylinConfig);
    execMgr = ExecutableManager.getInstance(kylinConfig);
}
 

public static synchronized void destroyInstance() {
    DefaultScheduler tmp = INSTANCE;
    INSTANCE = null;
    if (tmp != null) {
        try {
            tmp.shutdown();
        } catch (SchedulerException e) {
            logger.error("error stop DefaultScheduler", e);
            throw new RuntimeException(e);
        }
    }
}
 

@SuppressWarnings("unchecked")
@Override
public void afterPropertiesSet() throws Exception {

    String timeZone = getConfig().getTimeZone();
    TimeZone tzone = TimeZone.getTimeZone(timeZone);
    TimeZone.setDefault(tzone);

    final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();

    // In case of that kylin.server.cluster-name is not set,
    // this method have to be called first to avoid the influence of the change of kylin.metadata.url
    String clusterName = kylinConfig.getClusterName();
    logger.info("starting to initialize an instance in cluster {}", clusterName);

    final Scheduler<AbstractExecutable> scheduler = (Scheduler<AbstractExecutable>) SchedulerFactory
            .scheduler(kylinConfig.getSchedulerType());

    scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock());

    Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
        @Override
        public void run() {
            try {
                scheduler.shutdown();
            } catch (SchedulerException e) {
                logger.error("error occurred to shutdown scheduler", e);
            }
        }
    }));
}
 

protected void startScheduler() throws SchedulerException {
    scheduler = DefaultScheduler.createInstance();
    scheduler.init(new JobEngineConfig(KylinConfig.getInstanceFromEnv()), new MockJobLock());
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
}
 

@Override
public void close() throws IOException {

    if (scheduler!= null)
        try {
            scheduler.shutdown();
        } catch (SchedulerException e) {
           //
        }
}
 

protected void startScheduler() throws SchedulerException {
    scheduler = DefaultScheduler.createInstance();
    scheduler.init(new JobEngineConfig(KylinConfig.getInstanceFromEnv()), new MockJobLock());
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
}
 

@SuppressWarnings("unchecked")
@Override
public void afterPropertiesSet() throws Exception {

    String timeZone = getConfig().getTimeZone();
    TimeZone tzone = TimeZone.getTimeZone(timeZone);
    TimeZone.setDefault(tzone);

    final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();

    // In case of that kylin.server.cluster-name is not set,
    // this method have to be called first to avoid the influence of the change of kylin.metadata.url
    String clusterName = kylinConfig.getClusterName();
    logger.info("starting to initialize an instance in cluster {}", clusterName);

    final Scheduler<AbstractExecutable> scheduler = (Scheduler<AbstractExecutable>) SchedulerFactory
            .scheduler(kylinConfig.getSchedulerType());

    if (kylinConfig.getServerSelfDiscoveryEnabled()) {
        KylinServerDiscovery.getInstance();
    }
    logger.info("Cluster servers: {}", Lists.newArrayList(kylinConfig.getRestServers()));
    
    scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock());

    Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
        @Override
        public void run() {
            try {
                scheduler.shutdown();
            } catch (SchedulerException e) {
                logger.error("error occurred to shutdown scheduler", e);
            }
        }
    }));
}
 

@Override
public void stateChanged(CuratorFramework client, ConnectionState newState) {
    if ((newState == ConnectionState.SUSPENDED) || (newState == ConnectionState.LOST)) {
        try {
            shutdown();
        } catch (SchedulerException e) {
            throw new RuntimeException("failed to shutdown scheduler", e);
        }
    }
}
 

public static synchronized void destroyInstance() {
    DefaultScheduler tmp = INSTANCE;
    INSTANCE = null;
    if (tmp != null) {
        try {
            tmp.shutdown();
        } catch (SchedulerException e) {
            logger.error("error stop DefaultScheduler", e);
            throw new RuntimeException(e);
        }
    }
}
 

@Override
public boolean stop(AbstractExecutable executable) throws SchedulerException {
    if (hasStarted) {
        return true;
    } else {
        //TODO should try to stop this executable
        return true;
    }
}
 

@Override
public synchronized void init(JobEngineConfig jobEngineConfig, JobLock lock) throws SchedulerException {
    jobLock = lock;

    String serverMode = jobEngineConfig.getConfig().getServerMode();
    if (!("job".equals(serverMode.toLowerCase(Locale.ROOT)) || "all".equals(serverMode.toLowerCase(Locale.ROOT)))) {
        logger.info("server mode: " + serverMode + ", no need to run job scheduler");
        return;
    }
    logger.info("Initializing Job Engine ....");

    if (!initialized) {
        initialized = true;
    } else {
        return;
    }

    this.jobEngineConfig = jobEngineConfig;

    if (jobLock.lockJobEngine() == false) {
        throw new IllegalStateException("Cannot start job scheduler due to lack of job lock");
    }

    //load all executable, set them to a consistent status
    fetcherPool = Executors.newScheduledThreadPool(1);
    int corePoolSize = jobEngineConfig.getMaxConcurrentJobLimit();
    jobPool = new ThreadPoolExecutor(corePoolSize, corePoolSize, Long.MAX_VALUE, TimeUnit.DAYS,
            new SynchronousQueue<Runnable>());
    context = new DefaultContext(Maps.<String, Executable> newConcurrentMap(), jobEngineConfig.getConfig());

    logger.info("Starting resume all running jobs.");
    ExecutableManager executableManager = getExecutableManager();
    executableManager.resumeAllRunningJobs();
    logger.info("Finishing resume all running jobs.");

    int pollSecond = jobEngineConfig.getPollIntervalSecond();

    logger.info("Fetching jobs every {} seconds", pollSecond);
    JobExecutor jobExecutor = new JobExecutor() {
        @Override
        public void execute(AbstractExecutable executable) {
            jobPool.execute(new JobRunner(executable));
        }
    };
    fetcher = jobEngineConfig.getJobPriorityConsidered()
            ? new PriorityFetcherRunner(jobEngineConfig, context, jobExecutor)
            : new DefaultFetcherRunner(jobEngineConfig, context, jobExecutor);
    logger.info("Creating fetcher pool instance:" + System.identityHashCode(fetcher));
    fetcherPool.scheduleAtFixedRate(fetcher, pollSecond / 10, pollSecond, TimeUnit.SECONDS);
    hasStarted = true;
}
 

@Test
public void testGetBasic() throws SchedulerException, IOException, InterruptedException {
    KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    final String zkString = zkTestServer.getConnectString();
    final String server1 = "server1:1111";
    final String server2 = "server2:2222";
    String jobEnginePath = CuratorScheduler.JOB_ENGINE_LEADER_PATH;
    CuratorFramework client = CuratorFrameworkFactory.newClient(zkString, new ExponentialBackoffRetry(3000, 3));
    client.start();
    CuratorLeaderSelector s1 = new CuratorLeaderSelector(client //
            , jobEnginePath //
            , server1 //
            , new JobEngineConfig(kylinConfig)); //
    Assert.assertFalse(s1.hasDefaultSchedulerStarted());
    CuratorLeaderSelector s2 = new CuratorLeaderSelector(client //
            , jobEnginePath //
            , server2 //
            , new JobEngineConfig(kylinConfig)); //
    s1.start();
    //wait for Selector starting
    Thread.sleep(1000);
    Assert.assertEquals(1, s1.getParticipants().size());
    Assert.assertEquals(s1.getParticipants(), s2.getParticipants());
    s2.start();
    Thread.sleep(1000);
    Assert.assertEquals(2, s1.getParticipants().size());
    Assert.assertEquals(s1.getParticipants(), s2.getParticipants());

    Assert.assertEquals(new Participant(server1, true), s1.getLeader());
    Assert.assertEquals(s1.getLeader(), s2.getLeader());
    assertSchedulerStart(s1);
    s1.close();
    Thread.sleep(1000);
    Assert.assertEquals(1, s1.getParticipants().size());
    Assert.assertEquals(s1.getParticipants(), s2.getParticipants());
    Assert.assertEquals(new Participant(server2, true), s1.getLeader());
    assertSchedulerStart(s2);
    s2.close();
    Thread.sleep(1000);
    Assert.assertEquals(0, s1.getParticipants().size());
    Assert.assertEquals(s1.getParticipants(), s2.getParticipants());
}
 

@Override
public void shutdown() throws SchedulerException {
}
 

@Override
public void init(JobEngineConfig jobEngineConfig, JobLock jobLock) throws SchedulerException {
}
 

@Override
public void shutdown() throws SchedulerException {
    fetcherPool.shutdown();
    jobPool.shutdown();
    releaseLock();
}
 

@Override
public synchronized void init(JobEngineConfig jobEngineConfig, JobLock jobLock) throws SchedulerException {
    String serverMode = jobEngineConfig.getConfig().getServerMode();
    if (!("job".equals(serverMode.toLowerCase(Locale.ROOT)) || "all".equals(serverMode.toLowerCase(Locale.ROOT)))) {
        logger.info("server mode: " + serverMode + ", no need to run job scheduler");
        return;
    }
    logger.info("Initializing Job Engine ....");

    if (!initialized) {
        initialized = true;
    } else {
        return;
    }

    this.jobEngineConfig = jobEngineConfig;
    this.jobLock = (DistributedLock) jobLock;
    this.serverName = this.jobLock.getClient(); // the lock's client string contains node name of this server

    executableManager = ExecutableManager.getInstance(jobEngineConfig.getConfig());
    //load all executable, set them to a consistent status
    fetcherPool = Executors.newScheduledThreadPool(1);

    //watch the zookeeper node change, so that when one job server is down, other job servers can take over.
    watchPool = Executors.newFixedThreadPool(1);
    WatcherProcessImpl watcherProcess = new WatcherProcessImpl(this.serverName);
    lockWatch = this.jobLock.watchLocks(getWatchPath(), watchPool, watcherProcess);

    int corePoolSize = jobEngineConfig.getMaxConcurrentJobLimit();
    jobPool = new ThreadPoolExecutor(corePoolSize, corePoolSize, Long.MAX_VALUE, TimeUnit.DAYS,
            new SynchronousQueue<Runnable>());
    context = new DefaultContext(Maps.<String, Executable> newConcurrentMap(), jobEngineConfig.getConfig());

    int pollSecond = jobEngineConfig.getPollIntervalSecond();
    logger.info("Fetching jobs every {} seconds", pollSecond);
    JobExecutor jobExecutor = new JobExecutor() {
        @Override
        public void execute(AbstractExecutable executable) {
            jobPool.execute(new JobRunner(executable));
        }
    };
    fetcher = jobEngineConfig.getJobPriorityConsidered()
            ? new PriorityFetcherRunner(jobEngineConfig, context, jobExecutor)
            : new DefaultFetcherRunner(jobEngineConfig, context, jobExecutor);
    fetcherPool.scheduleAtFixedRate(fetcher, pollSecond / 10, pollSecond, TimeUnit.SECONDS);
    hasStarted = true;

    resumeAllRunningJobs();
}
 

@Override
public synchronized void init(JobEngineConfig jobEngineConfig, JobLock jobLock) throws SchedulerException {
    String serverMode = jobEngineConfig.getConfig().getServerMode();
    if (!("job".equals(serverMode.toLowerCase(Locale.ROOT)) || "all".equals(serverMode.toLowerCase(Locale.ROOT)))) {
        logger.info("server mode: " + serverMode + ", no need to run job scheduler");
        return;
    }
    logger.info("Initializing Job Engine ....");

    if (!initialized) {
        initialized = true;
    } else {
        return;
    }

    this.jobEngineConfig = jobEngineConfig;
    this.jobLock = (DistributedLock) jobLock;
    this.serverName = this.jobLock.getClient(); // the lock's client string contains node name of this server

    executableManager = ExecutableManager.getInstance(jobEngineConfig.getConfig());
    //load all executable, set them to a consistent status
    fetcherPool = Executors.newScheduledThreadPool(1);

    //watch the zookeeper node change, so that when one job server is down, other job servers can take over.
    watchPool = Executors.newFixedThreadPool(1);
    WatcherProcessImpl watcherProcess = new WatcherProcessImpl(this.serverName);
    lockWatch = this.jobLock.watchLocks(getWatchPath(), watchPool, watcherProcess);

    int corePoolSize = jobEngineConfig.getMaxConcurrentJobLimit();
    jobPool = new ThreadPoolExecutor(corePoolSize, corePoolSize, Long.MAX_VALUE, TimeUnit.DAYS,
            new SynchronousQueue<Runnable>());
    context = new DefaultContext(Maps.<String, Executable> newConcurrentMap(), jobEngineConfig.getConfig());

    int pollSecond = jobEngineConfig.getPollIntervalSecond();
    logger.info("Fetching jobs every {} seconds", pollSecond);
    JobExecutor jobExecutor = new JobExecutor() {
        @Override
        public void execute(AbstractExecutable executable) {
            jobPool.execute(new JobRunner(executable));
        }
    };
    fetcher = jobEngineConfig.getJobPriorityConsidered()
            ? new PriorityFetcherRunner(jobEngineConfig, context, jobExecutor)
            : new DefaultFetcherRunner(jobEngineConfig, context, jobExecutor);
    fetcherPool.scheduleAtFixedRate(fetcher, pollSecond / 10, pollSecond, TimeUnit.SECONDS);
    hasStarted = true;

    resumeAllRunningJobs();
}
 

@Override
public void setup() throws SchedulerException {
    super.setup();
    SparderContext.setSparkSession(KylinSparkEnv.getSparkSession());
}
 

@Override
public void init(JobEngineConfig jobEngineConfig, JobLock jobLock) throws SchedulerException {
}
 

@Override
public void shutdown() throws SchedulerException {
}
 

@Override
public synchronized void init(JobEngineConfig jobEngineConfig, JobLock lock) throws SchedulerException {
    jobLock = lock;

    String serverMode = jobEngineConfig.getConfig().getServerMode();
    if (!("job".equals(serverMode.toLowerCase(Locale.ROOT)) || "all".equals(serverMode.toLowerCase(Locale.ROOT)))) {
        logger.info("server mode: " + serverMode + ", no need to run job scheduler");
        return;
    }
    logger.info("Initializing Job Engine ....");

    if (!initialized) {
        initialized = true;
    } else {
        return;
    }

    this.jobEngineConfig = jobEngineConfig;

    if (jobLock.lockJobEngine() == false) {
        throw new IllegalStateException("Cannot start job scheduler due to lack of job lock");
    }

    //load all executable, set them to a consistent status
    fetcherPool = Executors.newScheduledThreadPool(1);
    int corePoolSize = jobEngineConfig.getMaxConcurrentJobLimit();
    jobPool = new ThreadPoolExecutor(corePoolSize, corePoolSize, Long.MAX_VALUE, TimeUnit.DAYS,
            new SynchronousQueue<Runnable>());
    context = new DefaultContext(Maps.<String, Executable> newConcurrentMap(), jobEngineConfig.getConfig());

    logger.info("Starting resume all running jobs.");
    ExecutableManager executableManager = getExecutableManager();
    executableManager.resumeAllRunningJobs();
    logger.info("Finishing resume all running jobs.");

    int pollSecond = jobEngineConfig.getPollIntervalSecond();

    logger.info("Fetching jobs every {} seconds", pollSecond);
    JobExecutor jobExecutor = new JobExecutor() {
        @Override
        public void execute(AbstractExecutable executable) {
            jobPool.execute(new JobRunner(executable));
        }
    };
    fetcher = jobEngineConfig.getJobPriorityConsidered()
            ? new PriorityFetcherRunner(jobEngineConfig, context, jobExecutor)
            : new DefaultFetcherRunner(jobEngineConfig, context, jobExecutor);
    logger.info("Creating fetcher pool instance:" + System.identityHashCode(fetcher));
    fetcherPool.scheduleAtFixedRate(fetcher, pollSecond / 10, pollSecond, TimeUnit.SECONDS);
    hasStarted = true;
}