Java Code Examples for org.apache.kylin.job.impl.threadpool.DefaultScheduler#createInstance()

The following examples show how to use org.apache.kylin.job.impl.threadpool.DefaultScheduler#createInstance() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: BuildCubeWithEngine.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
public void before() throws Exception {
    deployEnv();

    final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    jobService = ExecutableManager.getInstance(kylinConfig);
    scheduler = DefaultScheduler.createInstance();
    scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock());
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
    cubeManager = CubeManager.getInstance(kylinConfig);
    for (String jobId : jobService.getAllJobIds()) {
        AbstractExecutable executable = jobService.getJob(jobId);
        if (executable instanceof CubingJob || executable instanceof CheckpointExecutable) {
            jobService.deleteJob(jobId);
        }
    }

    cubeDescManager = CubeDescManager.getInstance(kylinConfig);

    // update enginType
    updateCubeEngineType(Lists.newArrayList("ci_inner_join_cube", "ci_left_join_cube"));
}
Example 2
Source File: BuildCubeWithEngine.java    From kylin with Apache License 2.0 6 votes vote down vote up 
public void before() throws Exception {
    deployEnv();

    final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    jobService = ExecutableManager.getInstance(kylinConfig);
    scheduler = DefaultScheduler.createInstance();
    scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock());
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
    cubeManager = CubeManager.getInstance(kylinConfig);
    for (String jobId : jobService.getAllJobIds()) {
        AbstractExecutable executable = jobService.getJob(jobId);
        if (executable instanceof CubingJob || executable instanceof CheckpointExecutable) {
            jobService.deleteJob(jobId);
        }
    }

    cubeDescManager = CubeDescManager.getInstance(kylinConfig);

    // update enginType
    updateCubeEngineType(Lists.newArrayList("ci_inner_join_cube", "ci_left_join_cube"));
}
Example 3
Source File: BuildCubeWithStream.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public void before() throws Exception {
    deployEnv();
    simpleBuildMode = isSimpleBuildMode();
    if (simpleBuildMode) {
        logger.info("Will use simple build mode");
    } else {
        logger.info("Will not use simple build mode");
    }

    final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    jobService = ExecutableManager.getInstance(kylinConfig);
    scheduler = DefaultScheduler.createInstance();
    scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock());
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
    cubeManager = CubeManager.getInstance(kylinConfig);

    final CubeInstance cubeInstance = CubeManager.getInstance(kylinConfig).getCube(cubeName);
    final String factTable = cubeInstance.getRootFactTable();

    final StreamingManager streamingManager = StreamingManager.getInstance(kylinConfig);
    final StreamingConfig streamingConfig = streamingManager.getStreamingConfig(factTable);
    kafkaConfig = KafkaConfigManager.getInstance(kylinConfig).getKafkaConfig(streamingConfig.getName());

    String topicName = RandomUtil.randomUUID().toString();
    BrokerConfig brokerConfig = kafkaConfig.getKafkaClusterConfigs().get(0).getBrokerConfigs().get(0);
    kafkaConfig.setTopic(topicName);
    KafkaConfigManager.getInstance(kylinConfig).updateKafkaConfig(kafkaConfig);

    startEmbeddedKafka(topicName, brokerConfig);
}
Example 4
Source File: BuildCubeWithStream.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public void before() throws Exception {
    deployEnv();
    simpleBuildMode = isSimpleBuildMode();
    if (simpleBuildMode) {
        logger.info("Will use simple build mode");
    } else {
        logger.info("Will not use simple build mode");
    }

    final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    jobService = ExecutableManager.getInstance(kylinConfig);
    scheduler = DefaultScheduler.createInstance();
    scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock());
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
    cubeManager = CubeManager.getInstance(kylinConfig);

    final CubeInstance cubeInstance = CubeManager.getInstance(kylinConfig).getCube(cubeName);
    final String factTable = cubeInstance.getRootFactTable();

    final StreamingManager streamingManager = StreamingManager.getInstance(kylinConfig);
    final StreamingConfig streamingConfig = streamingManager.getStreamingConfig(factTable);
    kafkaConfig = KafkaConfigManager.getInstance(kylinConfig).getKafkaConfig(streamingConfig.getName());

    String topicName = RandomUtil.randomUUID().toString();
    BrokerConfig brokerConfig = kafkaConfig.getKafkaClusterConfigs().get(0).getBrokerConfigs().get(0);
    kafkaConfig.setTopic(topicName);
    KafkaConfigManager.getInstance(kylinConfig).updateKafkaConfig(kafkaConfig);

    startEmbeddedKafka(topicName, brokerConfig);
}
Example 5
Source File: BuildCubeWithStreamV2.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public void before() throws Exception {
    deployEnv();

    final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    scheduler = DefaultScheduler.createInstance();
    scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock());
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }

    final CubeInstance cubeInstance = CubeManager.getInstance(kylinConfig).getCube(CUBE_NAME);
    final String streamingTableName = cubeInstance.getRootFactTable();
    final StreamingSourceConfig sourceConfig = StreamingSourceConfigManager.getInstance(kylinConfig).getConfig(streamingTableName);

    topicName = KafkaSource.getTopicName(sourceConfig.getProperties());
    String bootstrapServers = KafkaSource.getBootstrapServers(sourceConfig.getProperties());

    String server = bootstrapServers.split(",")[0];
    int brokerId = 0;

    // start kafka broker and create topic
    startEmbeddedKafka(topicName, server, brokerId);

    // start streamingServer
    streamingServer.start();

    // add receiver to replicaSet
    ReplicaSet replicaSet = new ReplicaSet();
    replicaSet.addNode(new Node());
    coordinator.createReplicaSet(replicaSet);
    replicaSetID = replicaSet.getReplicaSetID();

    // becomeLeader
    streamingServer.becomeLeader();

    // enabse streaming cube
    CubeManager.getInstance(kylinConfig).updateCubeStatus(cubeInstance, RealizationStatusEnum.READY);

    // assign cube/start consumer from LATEST
    coordinator.assignCube(CUBE_NAME);

    // init producer
    Properties properties = new Properties();
    properties.put("key.serializer", ByteArraySerializer.class.getName());
    properties.put("value.serializer", ByteArraySerializer.class.getName());
    producer = new KafkaProducer<byte[], byte[]>(Kafka10DataLoader.constructDefaultKafkaProducerProperties(bootstrapServers, properties));
}
Example 6
Source File: BuildCubeWithStreamV2.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public void before() throws Exception {
    deployEnv();

    final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    scheduler = DefaultScheduler.createInstance();
    scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock());
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }

    final CubeInstance cubeInstance = CubeManager.getInstance(kylinConfig).getCube(CUBE_NAME);
    final String streamingTableName = cubeInstance.getRootFactTable();
    final StreamingSourceConfig sourceConfig = StreamingSourceConfigManager.getInstance(kylinConfig).getConfig(streamingTableName);

    topicName = KafkaSource.getTopicName(sourceConfig.getProperties());
    String bootstrapServers = KafkaSource.getBootstrapServers(sourceConfig.getProperties());

    String server = bootstrapServers.split(",")[0];
    int brokerId = 0;

    // start kafka broker and create topic
    startEmbeddedKafka(topicName, server, brokerId);

    // start streamingServer
    streamingServer.start();

    // add receiver to replicaSet
    ReplicaSet replicaSet = new ReplicaSet();
    replicaSet.addNode(new Node());
    coordinator.createReplicaSet(replicaSet);
    replicaSetID = replicaSet.getReplicaSetID();

    // becomeLeader
    streamingServer.becomeLeader();

    // enabse streaming cube
    CubeManager.getInstance(kylinConfig).updateCubeStatus(cubeInstance, RealizationStatusEnum.READY);

    // assign cube/start consumer from LATEST
    coordinator.assignCube(CUBE_NAME);

    // init producer
    Properties properties = new Properties();
    properties.put("key.serializer", ByteArraySerializer.class.getName());
    properties.put("value.serializer", ByteArraySerializer.class.getName());
    producer = new KafkaProducer<byte[], byte[]>(Kafka10DataLoader.constructDefaultKafkaProducerProperties(bootstrapServers, properties));
}