org.apache.kylin.job.impl.threadpool.DefaultScheduler Java Examples

The following examples show how to use org.apache.kylin.job.impl.threadpool.DefaultScheduler. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BuildCubeWithEngine.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
public void before() throws Exception {
    deployEnv();

    final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    jobService = ExecutableManager.getInstance(kylinConfig);
    scheduler = DefaultScheduler.createInstance();
    scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock());
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
    cubeManager = CubeManager.getInstance(kylinConfig);
    for (String jobId : jobService.getAllJobIds()) {
        AbstractExecutable executable = jobService.getJob(jobId);
        if (executable instanceof CubingJob || executable instanceof CheckpointExecutable) {
            jobService.deleteJob(jobId);
        }
    }

    cubeDescManager = CubeDescManager.getInstance(kylinConfig);

    // update enginType
    updateCubeEngineType(Lists.newArrayList("ci_inner_join_cube", "ci_left_join_cube"));
}
Example #2
Source File: BuildCubeWithEngineTest.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
@Before
public void before() throws Exception {
    HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);

    DeployUtil.initCliWorkDir();
    DeployUtil.deployMetadata();
    DeployUtil.overrideJobJarLocations();


    final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    jobService = ExecutableManager.getInstance(kylinConfig);
    scheduler = DefaultScheduler.getInstance();
    scheduler.init(new JobEngineConfig(kylinConfig));
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
    cubeManager = CubeManager.getInstance(kylinConfig);
    jobEngineConfig = new JobEngineConfig(kylinConfig);
    for (String jobId : jobService.getAllJobIds()) {
        if(jobService.getJob(jobId) instanceof CubingJob){
            jobService.deleteJob(jobId);
        }
    }

}
Example #3
Source File: BuildCubeWithEngine.java    From kylin with Apache License 2.0 6 votes vote down vote up 
public void before() throws Exception {
    deployEnv();

    final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    jobService = ExecutableManager.getInstance(kylinConfig);
    scheduler = DefaultScheduler.createInstance();
    scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock());
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
    cubeManager = CubeManager.getInstance(kylinConfig);
    for (String jobId : jobService.getAllJobIds()) {
        AbstractExecutable executable = jobService.getJob(jobId);
        if (executable instanceof CubingJob || executable instanceof CheckpointExecutable) {
            jobService.deleteJob(jobId);
        }
    }

    cubeDescManager = CubeDescManager.getInstance(kylinConfig);

    // update enginType
    updateCubeEngineType(Lists.newArrayList("ci_inner_join_cube", "ci_left_join_cube"));
}
Example #4
Source File: NFilePruningTest.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
@Before
public void setup() throws SchedulerException {
    this.createTestMetadata("../../examples/test_case_data/file_prunning");
    System.setProperty("kylin.env", "UT");
    System.setProperty("kylin.query.enable-dynamic-column", "false");
    Map<RealizationType, Integer> priorities = Maps.newHashMap();
    priorities.put(RealizationType.HYBRID, 0);
    priorities.put(RealizationType.CUBE, 0);
    Candidate.setPriorities(priorities);
    overwriteSystemProp("kylin.job.scheduler.poll-interval-second", "1");
    overwriteSystemProp("calcite.keep-in-clause", "true");
    overwriteSystemProp("kylin.metadata.distributed-lock-impl", "org.apache.kylin.engine.spark.utils.MockedDistributedLock$MockedFactory");
    DefaultScheduler scheduler = DefaultScheduler.getInstance();
    scheduler.init(new JobEngineConfig(KylinConfig.getInstanceFromEnv()), new MockJobLock());
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
    config = KylinConfig.getInstanceFromEnv();
    cubeMgr = CubeManager.getInstance(config);
    execMgr = ExecutableManager.getInstance(config);
}
Example #5
Source File: BuildCubeWithStream.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public void before() throws Exception {
    deployEnv();
    simpleBuildMode = isSimpleBuildMode();
    if (simpleBuildMode) {
        logger.info("Will use simple build mode");
    } else {
        logger.info("Will not use simple build mode");
    }

    final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    jobService = ExecutableManager.getInstance(kylinConfig);
    scheduler = DefaultScheduler.createInstance();
    scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock());
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
    cubeManager = CubeManager.getInstance(kylinConfig);

    final CubeInstance cubeInstance = CubeManager.getInstance(kylinConfig).getCube(cubeName);
    final String factTable = cubeInstance.getRootFactTable();

    final StreamingManager streamingManager = StreamingManager.getInstance(kylinConfig);
    final StreamingConfig streamingConfig = streamingManager.getStreamingConfig(factTable);
    kafkaConfig = KafkaConfigManager.getInstance(kylinConfig).getKafkaConfig(streamingConfig.getName());

    String topicName = RandomUtil.randomUUID().toString();
    BrokerConfig brokerConfig = kafkaConfig.getKafkaClusterConfigs().get(0).getBrokerConfigs().get(0);
    kafkaConfig.setTopic(topicName);
    KafkaConfigManager.getInstance(kylinConfig).updateKafkaConfig(kafkaConfig);

    startEmbeddedKafka(topicName, brokerConfig);
}
Example #6
Source File: JobController.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public void afterPropertiesSet() throws Exception {
    String timeZone = jobService.getKylinConfig().getTimeZone();
    TimeZone tzone = TimeZone.getTimeZone(timeZone);
    TimeZone.setDefault(tzone);

    final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    String serverMode = kylinConfig.getServerMode();

    if (Constant.SERVER_MODE_JOB.equals(serverMode.toLowerCase()) || Constant.SERVER_MODE_ALL.equals(serverMode.toLowerCase())) {
        logger.info("Initializing Job Engine ....");

        new Thread(new Runnable() {
            @Override
            public void run() {
                try {
                    DefaultScheduler scheduler = DefaultScheduler.getInstance();
                    scheduler.init(new JobEngineConfig(kylinConfig));
                    if (!scheduler.hasStarted()) {
                        logger.error("scheduler has not been started");
                        System.exit(1);
                    }
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            }
        }).start();
    }
}
Example #7
Source File: BuildIIWithEngineTest.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
@Before
public void before() throws Exception {
    HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);

    DeployUtil.initCliWorkDir();
    //        DeployUtil.deployMetadata();
    DeployUtil.overrideJobJarLocations();

    final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    jobService = ExecutableManager.getInstance(kylinConfig);
    scheduler = DefaultScheduler.getInstance();
    scheduler.init(new JobEngineConfig(kylinConfig));
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
    iiManager = IIManager.getInstance(kylinConfig);
    jobEngineConfig = new JobEngineConfig(kylinConfig);
    for (String jobId : jobService.getAllJobIds()) {
        if(jobService.getJob(jobId) instanceof IIJob){
            jobService.deleteJob(jobId);
        }
    }

    IIInstance ii = iiManager.getII(TEST_II_NAME);
    if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
        ii.setStatus(RealizationStatusEnum.DISABLED);
        iiManager.updateII(ii);
    }
}
Example #8
Source File: CuratorLeaderSelector.java    From kylin with Apache License 2.0 5 votes vote down vote up 
CuratorLeaderSelector(CuratorFramework client, String path, String name, JobEngineConfig jobEngineConfig) {
    this.name = name;
    this.leaderSelector = new LeaderSelector(client, path, this);
    this.leaderSelector.setId(name);
    this.leaderSelector.autoRequeue();
    this.jobEngineConfig = jobEngineConfig;
    this.defaultScheduler = DefaultScheduler.getInstance();
}
Example #9
Source File: BuildCubeWithStreamV2.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public void after() throws Exception {
    coordinator.unAssignCube(CUBE_NAME);
    streamingServer.removeFromReplicaSet();
    if (kafkaServer != null) {
        kafkaServer.stop();

    }
    ZKUtil.cleanZkPath(kafkaZkPath);
    DefaultScheduler.destroyInstance();
}
Example #10
Source File: BuildCubeWithStream.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public void after() {
    if (kafkaServer != null) {
        kafkaServer.stop();
    }
    ZKUtil.cleanZkPath(kafkaZkPath);
    DefaultScheduler.destroyInstance();
}
Example #11
Source File: BuildCubeWithStream.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public void before() throws Exception {
    deployEnv();
    simpleBuildMode = isSimpleBuildMode();
    if (simpleBuildMode) {
        logger.info("Will use simple build mode");
    } else {
        logger.info("Will not use simple build mode");
    }

    final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    jobService = ExecutableManager.getInstance(kylinConfig);
    scheduler = DefaultScheduler.createInstance();
    scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock());
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
    cubeManager = CubeManager.getInstance(kylinConfig);

    final CubeInstance cubeInstance = CubeManager.getInstance(kylinConfig).getCube(cubeName);
    final String factTable = cubeInstance.getRootFactTable();

    final StreamingManager streamingManager = StreamingManager.getInstance(kylinConfig);
    final StreamingConfig streamingConfig = streamingManager.getStreamingConfig(factTable);
    kafkaConfig = KafkaConfigManager.getInstance(kylinConfig).getKafkaConfig(streamingConfig.getName());

    String topicName = RandomUtil.randomUUID().toString();
    BrokerConfig brokerConfig = kafkaConfig.getKafkaClusterConfigs().get(0).getBrokerConfigs().get(0);
    kafkaConfig.setTopic(topicName);
    KafkaConfigManager.getInstance(kylinConfig).updateKafkaConfig(kafkaConfig);

    startEmbeddedKafka(topicName, brokerConfig);
}
Example #12
Source File: LocalWithSparkSessionTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Before
public void setup() throws SchedulerException {
    overwriteSystemProp("kylin.job.scheduler.poll-interval-second", "1");
    overwriteSystemProp("calcite.keep-in-clause", "true");
    overwriteSystemProp("kylin.metadata.distributed-lock-impl", "org.apache.kylin.engine.spark.utils.MockedDistributedLock$MockedFactory");
    this.createTestMetadata();
    DefaultScheduler scheduler = DefaultScheduler.getInstance();
    scheduler.init(new JobEngineConfig(KylinConfig.getInstanceFromEnv()), new MockJobLock());
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
}
Example #13
Source File: CuratorLeaderSelector.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
CuratorLeaderSelector(CuratorFramework client, String path, String name, JobEngineConfig jobEngineConfig) {
    this.name = name;
    this.leaderSelector = new LeaderSelector(client, path, this);
    this.leaderSelector.setId(name);
    this.leaderSelector.autoRequeue();
    this.jobEngineConfig = jobEngineConfig;
    this.defaultScheduler = DefaultScheduler.getInstance();
}
Example #14
Source File: BuildCubeWithStreamV2.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public void after() throws Exception {
    coordinator.unAssignCube(CUBE_NAME);
    streamingServer.removeFromReplicaSet();
    if (kafkaServer != null) {
        kafkaServer.stop();

    }
    ZKUtil.cleanZkPath(kafkaZkPath);
    DefaultScheduler.destroyInstance();
}
Example #15
Source File: BuildCubeWithStream.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public void after() {
    if (kafkaServer != null) {
        kafkaServer.stop();
    }
    ZKUtil.cleanZkPath(kafkaZkPath);
    DefaultScheduler.destroyInstance();
}
Example #16
Source File: LocalWithSparkSessionTest.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
@After
public void after() {
    DefaultScheduler.destroyInstance();
    this.cleanupTestMetadata();
    restoreAllSystemProp();
}
Example #17
Source File: BuildCubeWithEngine.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public void after() {
    DefaultScheduler.destroyInstance();
}
Example #18
Source File: BuildCubeWithStreamV2.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public void before() throws Exception {
    deployEnv();

    final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    scheduler = DefaultScheduler.createInstance();
    scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock());
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }

    final CubeInstance cubeInstance = CubeManager.getInstance(kylinConfig).getCube(CUBE_NAME);
    final String streamingTableName = cubeInstance.getRootFactTable();
    final StreamingSourceConfig sourceConfig = StreamingSourceConfigManager.getInstance(kylinConfig).getConfig(streamingTableName);

    topicName = KafkaSource.getTopicName(sourceConfig.getProperties());
    String bootstrapServers = KafkaSource.getBootstrapServers(sourceConfig.getProperties());

    String server = bootstrapServers.split(",")[0];
    int brokerId = 0;

    // start kafka broker and create topic
    startEmbeddedKafka(topicName, server, brokerId);

    // start streamingServer
    streamingServer.start();

    // add receiver to replicaSet
    ReplicaSet replicaSet = new ReplicaSet();
    replicaSet.addNode(new Node());
    coordinator.createReplicaSet(replicaSet);
    replicaSetID = replicaSet.getReplicaSetID();

    // becomeLeader
    streamingServer.becomeLeader();

    // enabse streaming cube
    CubeManager.getInstance(kylinConfig).updateCubeStatus(cubeInstance, RealizationStatusEnum.READY);

    // assign cube/start consumer from LATEST
    coordinator.assignCube(CUBE_NAME);

    // init producer
    Properties properties = new Properties();
    properties.put("key.serializer", ByteArraySerializer.class.getName());
    properties.put("value.serializer", ByteArraySerializer.class.getName());
    producer = new KafkaProducer<byte[], byte[]>(Kafka10DataLoader.constructDefaultKafkaProducerProperties(bootstrapServers, properties));
}
Example #19
Source File: BuildCubeWithStreamV2.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public void before() throws Exception {
    deployEnv();

    final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    scheduler = DefaultScheduler.createInstance();
    scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock());
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }

    final CubeInstance cubeInstance = CubeManager.getInstance(kylinConfig).getCube(CUBE_NAME);
    final String streamingTableName = cubeInstance.getRootFactTable();
    final StreamingSourceConfig sourceConfig = StreamingSourceConfigManager.getInstance(kylinConfig).getConfig(streamingTableName);

    topicName = KafkaSource.getTopicName(sourceConfig.getProperties());
    String bootstrapServers = KafkaSource.getBootstrapServers(sourceConfig.getProperties());

    String server = bootstrapServers.split(",")[0];
    int brokerId = 0;

    // start kafka broker and create topic
    startEmbeddedKafka(topicName, server, brokerId);

    // start streamingServer
    streamingServer.start();

    // add receiver to replicaSet
    ReplicaSet replicaSet = new ReplicaSet();
    replicaSet.addNode(new Node());
    coordinator.createReplicaSet(replicaSet);
    replicaSetID = replicaSet.getReplicaSetID();

    // becomeLeader
    streamingServer.becomeLeader();

    // enabse streaming cube
    CubeManager.getInstance(kylinConfig).updateCubeStatus(cubeInstance, RealizationStatusEnum.READY);

    // assign cube/start consumer from LATEST
    coordinator.assignCube(CUBE_NAME);

    // init producer
    Properties properties = new Properties();
    properties.put("key.serializer", ByteArraySerializer.class.getName());
    properties.put("value.serializer", ByteArraySerializer.class.getName());
    producer = new KafkaProducer<byte[], byte[]>(Kafka10DataLoader.constructDefaultKafkaProducerProperties(bootstrapServers, properties));
}
Example #20
Source File: BuildCubeWithEngine.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public void after() {
    DefaultScheduler.destroyInstance();
}