org.apache.kylin.job.exception.JobException Java Examples

@Test
public void testBasics() throws JobException, IOException, SQLException {
    Assert.assertNotNull(queryService.getConfig());
    Assert.assertNotNull(queryService.getKylinConfig());
    Assert.assertNotNull(queryService.getMetadataManager());
    Assert.assertNotNull(queryService.getOLAPDataSource(ProjectInstance.DEFAULT_PROJECT_NAME));

    //        Assert.assertTrue(queryService.getQueries("ADMIN").size() == 0);
    //
    //        queryService.saveQuery("test", "test", "select * from test_table", "test");
    //        Assert.assertTrue(queryService.getQueries("ADMIN").size() == 1);
    //
    //        queryService.removeQuery(queryService.getQueries("ADMIN").get(0).getProperty("id"));
    //        Assert.assertTrue(queryService.getQueries("ADMIN").size() == 0);

    SQLRequest request = new SQLRequest();
    request.setSql("select * from test_table");
    request.setAcceptPartial(true);
    SQLResponse response = new SQLResponse();
    response.setHitCache(true);
    queryService.logQuery(request, response, new Date(), new Date());
}
 

public static double getTotalMapInputMB(Job job)
        throws ClassNotFoundException, IOException, InterruptedException, JobException {
    if (job == null) {
        throw new JobException("Job is null");
    }

    long mapInputBytes = 0;
    InputFormat<?, ?> input = ReflectionUtils.newInstance(job.getInputFormatClass(), job.getConfiguration());
    for (InputSplit split : input.getSplits(job)) {
        mapInputBytes += split.getLength();
    }
    
    // 0 input bytes is possible when the segment range hits no partition on a partitioned hive table (KYLIN-2470) 
    if (mapInputBytes == 0) {
        logger.warn("Map input splits are 0 bytes, something is wrong?");
    }
    
    double totalMapInputMB = (double) mapInputBytes / 1024 / 1024;
    return totalMapInputMB;
}
 

@Test
public void testBasics() throws JsonProcessingException, JobException, UnknownHostException {
    Assert.assertNotNull(cubeService.getConfig());
    Assert.assertNotNull(cubeService.getKylinConfig());
    Assert.assertNotNull(cubeService.getMetadataManager());
    Assert.assertNotNull(cubeService.getOLAPDataSource(ProjectInstance.DEFAULT_PROJECT_NAME));

    Assert.assertTrue(CubeService.getCubeDescNameFromCube("testCube").equals("testCube_desc"));
    Assert.assertTrue(CubeService.getCubeNameFromDesc("testCube_desc").equals("testCube"));

    List<CubeInstance> cubes = cubeService.getCubes(null, null, null, null);
    Assert.assertNotNull(cubes);
    CubeInstance cube = cubes.get(0);
    cubeService.isCubeDescEditable(cube.getDescriptor());
    cubeService.isCubeEditable(cube);

    cubes = cubeService.getCubes(null, null, 1, 0);
    Assert.assertTrue(cubes.size() == 1);
}
 

@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#job, 'ADMINISTRATION') or hasPermission(#job, 'OPERATION') or hasPermission(#job, 'MANAGEMENT')")
public JobInstance cancelJob(String jobId) throws IOException, JobException {
    //        CubeInstance cube = this.getCubeManager().getCube(job.getRelatedCube());
    //        for (BuildCubeJob cubeJob: listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING))) {
    //            getExecutableManager().stopJob(cubeJob.getId());
    //        }
    final JobInstance jobInstance = getJobInstance(jobId);
    final String segmentId = jobInstance.getRelatedSegment();
    CubeInstance cubeInstance = getCubeManager().getCube(jobInstance.getRelatedCube());
    final CubeSegment segment = cubeInstance.getSegmentById(segmentId);
    if (segment.getStatus() == SegmentStatusEnum.NEW) {
        cubeInstance.getSegments().remove(segment);
        getCubeManager().updateCube(cubeInstance);
    }
    getExecutableManager().discardJob(jobId);
    return jobInstance;
}
 

public List<JobInstance> listAllJobs(final String cubeName, final String projectName, final List<JobStatusEnum> statusList, final Integer limitValue, final Integer offsetValue) throws IOException, JobException {
    Integer limit = (null == limitValue) ? 30 : limitValue;
    Integer offset = (null == offsetValue) ? 0 : offsetValue;
    List<JobInstance> jobs = listAllJobs(cubeName, projectName, statusList);
    Collections.sort(jobs);

    if (jobs.size() <= offset) {
        return Collections.emptyList();
    }

    if ((jobs.size() - offset) < limit) {
        return jobs.subList(offset, jobs.size());
    }

    return jobs.subList(offset, offset + limit);
}
 

@Test
public void testBasics() throws JobException, IOException, SQLException {
    Assert.assertNotNull(queryService.getConfig());
    Assert.assertNotNull(queryService.getConfig());
    Assert.assertNotNull(queryService.getDataModelManager());
    Assert.assertNotNull(QueryConnection.getConnection(ProjectInstance.DEFAULT_PROJECT_NAME));

    //        Assert.assertTrue(queryService.getQueries("ADMIN").size() == 0);
    //
    //        queryService.saveQuery("test", "test", "select * from test_table", "test");
    //        Assert.assertTrue(queryService.getQueries("ADMIN").size() == 1);
    //
    //        queryService.removeQuery(queryService.getQueries("ADMIN").get(0).getProperty("id"));
    //        Assert.assertTrue(queryService.getQueries("ADMIN").size() == 0);

    SQLRequest request = new SQLRequest();
    request.setSql("select * from test_table");
    request.setAcceptPartial(true);
    QueryContext queryContext = QueryContextFacade.current();
    SQLResponse response = new SQLResponse();
    response.setHitExceptionCache(true);
    queryService.logQuery(queryContext.getQueryId(), request, response);
}
 

public static double getTotalMapInputMB(Job job)
        throws ClassNotFoundException, IOException, InterruptedException, JobException {
    if (job == null) {
        throw new JobException("Job is null");
    }

    long mapInputBytes = 0;
    InputFormat<?, ?> input = ReflectionUtils.newInstance(job.getInputFormatClass(), job.getConfiguration());
    for (InputSplit split : input.getSplits(job)) {
        mapInputBytes += split.getLength();
    }
    
    // 0 input bytes is possible when the segment range hits no partition on a partitioned hive table (KYLIN-2470) 
    if (mapInputBytes == 0) {
        logger.warn("Map input splits are 0 bytes, something is wrong?");
    }
    
    double totalMapInputMB = (double) mapInputBytes / 1024 / 1024;
    return totalMapInputMB;
}
 

/**
 * Update a cube status from ready to disabled.
 *
 * @return
 * @throws CubeIntegrityException
 * @throws IOException
 * @throws JobException
 */
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')")
@Caching(evict = {@CacheEvict(value = QueryController.SUCCESS_QUERY_CACHE, allEntries = true), @CacheEvict(value = QueryController.EXCEPTION_QUERY_CACHE, allEntries = true)})
public CubeInstance disableCube(CubeInstance cube) throws IOException, JobException {
    String cubeName = cube.getName();

    RealizationStatusEnum ostatus = cube.getStatus();
    if (null != ostatus && !RealizationStatusEnum.READY.equals(ostatus)) {
        throw new InternalErrorException("Only ready cube can be disabled, status of " + cubeName + " is " + ostatus);
    }

    cube.setStatus(RealizationStatusEnum.DISABLED);

    try {
        return getCubeManager().updateCube(cube);
    } catch (IOException e) {
        cube.setStatus(ostatus);
        throw e;
    }
}
 

/**
 * Stop all jobs belonging to this cube and clean out all segments
 *
 * @param cube
 * @return
 * @throws IOException
 * @throws CubeIntegrityException
 * @throws JobException
 */
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')")
@Caching(evict = {@CacheEvict(value = QueryController.SUCCESS_QUERY_CACHE, allEntries = true), @CacheEvict(value = QueryController.EXCEPTION_QUERY_CACHE, allEntries = true)})
public CubeInstance purgeCube(CubeInstance cube) throws IOException, JobException {
    String cubeName = cube.getName();

    RealizationStatusEnum ostatus = cube.getStatus();
    if (null != ostatus && !RealizationStatusEnum.DISABLED.equals(ostatus)) {
        throw new InternalErrorException("Only disabled cube can be purged, status of " + cubeName + " is " + ostatus);
    }

    try {
        this.releaseAllSegments(cube);
        return cube;
    } catch (IOException e) {
        throw e;
    }

}
 

@Test
public void testBasics() throws JobException, IOException, SQLException {
    Assert.assertNotNull(queryService.getConfig());
    Assert.assertNotNull(queryService.getConfig());
    Assert.assertNotNull(queryService.getDataModelManager());
    Assert.assertNotNull(QueryConnection.getConnection(ProjectInstance.DEFAULT_PROJECT_NAME));

    //        Assert.assertTrue(queryService.getQueries("ADMIN").size() == 0);
    //
    //        queryService.saveQuery("test", "test", "select * from test_table", "test");
    //        Assert.assertTrue(queryService.getQueries("ADMIN").size() == 1);
    //
    //        queryService.removeQuery(queryService.getQueries("ADMIN").get(0).getProperty("id"));
    //        Assert.assertTrue(queryService.getQueries("ADMIN").size() == 0);

    SQLRequest request = new SQLRequest();
    request.setSql("select * from test_table");
    request.setAcceptPartial(true);
    QueryContext queryContext = QueryContextFacade.current();
    SQLResponse response = new SQLResponse();
    response.setHitExceptionCache(true);
    queryService.logQuery(queryContext.getQueryId(), request, response);
}
 

@Test
public void testSuccessModelUpdate() throws IOException, JobException {
    Serializer<DataModelDesc> serializer = modelService.getDataModelManager().getDataModelSerializer();
    
    List<DataModelDesc> dataModelDescs = modelService.listAllModels("ci_inner_join_model", "default", true);
    Assert.assertTrue(dataModelDescs.size() == 1);

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    serializer.serialize(dataModelDescs.get(0), new DataOutputStream(baos));
    ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
    DataModelDesc deserialize = serializer.deserialize(new DataInputStream(bais));

    deserialize.setOwner("somebody");
    DataModelDesc dataModelDesc = modelService.updateModelAndDesc("default", deserialize);
    Assert.assertTrue(dataModelDesc.getOwner().equals("somebody"));
}
 

@Test
public void testSuccessModelUpdate() throws IOException, JobException {
    Serializer<DataModelDesc> serializer = modelService.getDataModelManager().getDataModelSerializer();
    
    List<DataModelDesc> dataModelDescs = modelService.listAllModels("ci_inner_join_model", "default", true);
    Assert.assertTrue(dataModelDescs.size() == 1);

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    serializer.serialize(dataModelDescs.get(0), new DataOutputStream(baos));
    ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
    DataModelDesc deserialize = serializer.deserialize(new DataInputStream(bais));

    deserialize.setOwner("somebody");
    DataModelDesc dataModelDesc = modelService.updateModelAndDesc("default", deserialize);
    Assert.assertTrue(dataModelDesc.getOwner().equals("somebody"));
}
 

protected double getTotalMapInputMB() throws ClassNotFoundException, IOException, InterruptedException, JobException {
    if (job == null) {
        throw new JobException("Job is null");
    }

    long mapInputBytes = 0;
    InputFormat<?, ?> input = ReflectionUtils.newInstance(job.getInputFormatClass(), job.getConfiguration());
    for (InputSplit split : input.getSplits(job)) {
        mapInputBytes += split.getLength();
    }
    if (mapInputBytes == 0) {
        throw new IllegalArgumentException("Map input splits are 0 bytes, something is wrong!");
    }
    double totalMapInputMB = (double) mapInputBytes / 1024 / 1024;
    return totalMapInputMB;
}
 

public void kill() throws JobException {
    if (job != null) {
        try {
            job.killJob();
        } catch (IOException e) {
            throw new JobException(e);
        }
    }
}
 

@Test
public void testBasics() throws JsonProcessingException, JobException, UnknownHostException, SQLException {
    Assert.assertNotNull(cubeService.getConfig());
    Assert.assertNotNull(cubeService.getConfig());
    Assert.assertNotNull(cubeService.getDataModelManager());
    Assert.assertNotNull(QueryConnection.getConnection(ProjectInstance.DEFAULT_PROJECT_NAME));

    List<CubeInstance> cubes = cubeService.listAllCubes(null, null, null, true);
    Assert.assertNotNull(cubes);
    CubeInstance cube = cubes.get(0);
}
 

protected int getMapInputSplitCount() throws ClassNotFoundException, JobException, IOException, InterruptedException {
    if (job == null) {
        throw new JobException("Job is null");
    }
    InputFormat<?, ?> input = ReflectionUtils.newInstance(job.getInputFormatClass(), job.getConfiguration());
    return input.getSplits(job).size();
}
 

private void run(String cubeName, long startDate, long endDate, String buildType) throws IOException, JobException {
    CubeInstance cube = cubeManager.getCube(cubeName);
    Preconditions.checkArgument(cube != null, "Cube named " + cubeName + " does not exist!!!");
    CubeBuildTypeEnum buildTypeEnum = CubeBuildTypeEnum.valueOf(buildType);
    Preconditions.checkArgument(buildTypeEnum != null, "Build type named " + buildType + " does not exist!!!");
    submitJob(cube, new TSRange(startDate, endDate), buildTypeEnum, false, "SYSTEM");
}
 

@Test
public void testBasics() throws JobException, IOException, SQLException {
    Assert.assertNotNull(jobService.getConfig());
    Assert.assertNotNull(jobService.getConfig());
    Assert.assertNotNull(jobService.getDataModelManager());
    Assert.assertNotNull(QueryConnection.getConnection(ProjectInstance.DEFAULT_PROJECT_NAME));
    Assert.assertNull(jobService.getJobInstance("job_not_exist"));
    Assert.assertNotNull(jobService.searchJobs(null, null, null, 0, 0, JobTimeFilterEnum.ALL, JobService.JobSearchMode.ALL));
}
 

public Map<String, String> getInfo() throws JobException {
    if (job != null) {
        Map<String, String> status = new HashMap<String, String>();
        if (null != job.getJobID()) {
            status.put(JobInstance.MR_JOB_ID, job.getJobID().toString());
        }
        if (null != job.getTrackingURL()) {
            status.put(JobInstance.YARN_APP_URL, job.getTrackingURL().toString());
        }

        return status;
    } else {
        throw new JobException("Job is null");
    }
}
 

public Counters getCounters() throws JobException {
    if (job != null) {
        try {
            return job.getCounters();
        } catch (IOException e) {
            throw new JobException(e);
        }
    } else {
        throw new JobException("Job is null");
    }
}
 

public static void main(String[] args) throws JobException {
    ShellCmdOutput output = new ShellCmdOutput();
    ShellCmd shellCmd = new ShellCmd(args[0], output, args[1], args[2], args[3], false);
    shellCmd.execute();

    System.out.println("============================================================================");
    System.out.println(output.getExitCode());
    System.out.println(output.getOutput());
}
 

@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
public CubeDesc updateCubeAndDesc(CubeInstance cube, CubeDesc desc, String newProjectName) throws UnknownHostException, IOException, JobException {
    final List<CubingJob> cubingJobs = listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
    if (!cubingJobs.isEmpty()) {
        throw new JobException("Cube schema shouldn't be changed with running job.");
    }

    try {
        if (!cube.getDescriptor().calculateSignature().equals(cube.getDescriptor().getSignature())) {
            this.releaseAllSegments(cube);
        }

        CubeDesc updatedCubeDesc = getCubeDescManager().updateCubeDesc(desc);

        int cuboidCount = CuboidCLI.simulateCuboidGeneration(updatedCubeDesc);
        logger.info("Updated cube " + cube.getName() + " has " + cuboidCount + " cuboids");

        ProjectManager projectManager = getProjectManager();
        if (!isCubeInProject(newProjectName, cube)) {
            String owner = SecurityContextHolder.getContext().getAuthentication().getName();
            ProjectInstance newProject = projectManager.moveRealizationToProject(RealizationType.CUBE, cube.getName(), newProjectName, owner);
            accessService.inherit(cube, newProject);
        }

        return updatedCubeDesc;
    } catch (IOException e) {
        throw new InternalErrorException("Failed to deal with the request.", e);
    }
}
 

@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
public void deleteCube(CubeInstance cube) throws IOException, JobException {
    final List<CubingJob> cubingJobs = listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
    if (!cubingJobs.isEmpty()) {
        throw new JobException("The cube " + cube.getName() + " has running job, please discard it and try again.");
    }

    this.releaseAllSegments(cube);
    getCubeManager().dropCube(cube.getName(), true);
    accessService.clean(cube, true);
}
 

/**
 * Update a cube status from disable to ready.
 *
 * @return
 * @throws CubeIntegrityException
 * @throws IOException
 * @throws JobException
 */
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION')  or hasPermission(#cube, 'MANAGEMENT')")
public CubeInstance enableCube(CubeInstance cube) throws IOException, JobException {
    String cubeName = cube.getName();

    RealizationStatusEnum ostatus = cube.getStatus();
    if (!cube.getStatus().equals(RealizationStatusEnum.DISABLED)) {
        throw new InternalErrorException("Only disabled cube can be enabled, status of " + cubeName + " is " + ostatus);
    }

    if (cube.getSegments(SegmentStatusEnum.READY).size() == 0) {
        throw new InternalErrorException("Cube " + cubeName + " dosen't contain any READY segment");
    }

    final List<CubingJob> cubingJobs = listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
    if (!cubingJobs.isEmpty()) {
        throw new JobException("Enable is not allowed with a running job.");
    }
    if (!cube.getDescriptor().calculateSignature().equals(cube.getDescriptor().getSignature())) {
        this.releaseAllSegments(cube);
    }

    cube.setStatus(RealizationStatusEnum.READY);
    try {
        return getCubeManager().updateCube(cube);
    } catch (IOException e) {
        cube.setStatus(ostatus);
        throw e;
    }
}
 

/**
 * purge the cube
 *
 * @throws IOException
 * @throws JobException
 * @throws CubeIntegrityException
 */
private void releaseAllSegments(CubeInstance cube) throws IOException, JobException {
    final List<CubingJob> cubingJobs = listAllCubingJobs(cube.getName(), null);
    for (CubingJob cubingJob : cubingJobs) {
        final ExecutableState status = cubingJob.getStatus();
        if (status != ExecutableState.SUCCEED && status != ExecutableState.STOPPED && status != ExecutableState.DISCARDED) {
            getExecutableManager().discardJob(cubingJob.getId());
        }
    }
    cube.getSegments().clear();
    CubeManager.getInstance(getConfig()).updateCube(cube);
}
 

@Test
public void testBasics() throws JobException, IOException {
    Assert.assertNotNull(jobService.getConfig());
    Assert.assertNotNull(jobService.getKylinConfig());
    Assert.assertNotNull(jobService.getMetadataManager());
    Assert.assertNotNull(jobService.getOLAPDataSource(ProjectInstance.DEFAULT_PROJECT_NAME));
    Assert.assertNull(jobService.getJobInstance("job_not_exist"));
    Assert.assertNotNull(jobService.listAllJobs(null, null, null));
}
 

public Counters getCounters() throws JobException {
    if (job != null) {
        try {
            return job.getCounters();
        } catch (IOException e) {
            throw new JobException(e);
        }
    } else {
        throw new JobException("Job is null");
    }
}
 

@Test
public void testSchedulerLock() throws Exception {
    if (!lock(jobLock1, jobId1)) {
        throw new JobException("fail to get the lock");
    }
    DefaultChainedExecutable job = new DefaultChainedExecutable();
    job.setId(jobId1);
    AbstractExecutable task1 = new SucceedTestExecutable();
    AbstractExecutable task2 = new SucceedTestExecutable();
    AbstractExecutable task3 = new SucceedTestExecutable();
    job.addTask(task1);
    job.addTask(task2);
    job.addTask(task3);
    execMgr.addJob(job);

    Assert.assertEquals(serverName1, getServerName(jobId1));

    waitForJobFinish(job.getId());

    Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task1.getId()).getState());
    Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task2.getId()).getState());
    Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task3.getId()).getState());
    Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(job.getId()).getState());
    
    Thread.sleep(5000);

    Assert.assertEquals(null, getServerName(jobId1));
}
 

@Test
public void testSchedulerTakeOver() throws Exception {
    if (!lock(jobLock1, jobId2)) {
        throw new JobException("fail to get the lock");
    }

    DefaultChainedExecutable job = new DefaultChainedExecutable();
    job.setId(jobId2);
    AbstractExecutable task1 = new SucceedTestExecutable();
    AbstractExecutable task2 = new SucceedTestExecutable();
    AbstractExecutable task3 = new SucceedTestExecutable();
    job.addTask(task1);
    job.addTask(task2);
    job.addTask(task3);
    execMgr.addJob(job);

    waitForJobStatus(job.getId(), ExecutableState.RUNNING, 500);

    scheduler1.shutdown();
    scheduler1 = null;

    waitForJobFinish(job.getId());

    Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task1.getId()).getState());
    Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task2.getId()).getState());
    Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task3.getId()).getState());
    Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(job.getId()).getState());
}
 

protected int getMapInputSplitCount()
        throws ClassNotFoundException, JobException, IOException, InterruptedException {
    if (job == null) {
        throw new JobException("Job is null");
    }
    InputFormat<?, ?> input = ReflectionUtils.newInstance(job.getInputFormatClass(), job.getConfiguration());
    return input.getSplits(job).size();
}