Java Code Examples for org.apache.kylin.job.dao.ExecutablePO#getTasks()
The following examples show how to use
org.apache.kylin.job.dao.ExecutablePO#getTasks() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: YarnLogExtractor.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
|
public void extractYarnLogAndMRJob(String jobId, File yarnLogDir) throws Exception {
logger.info("Collecting Yarn logs and MR counters for the Job {}", jobId);
kylinConfig = KylinConfig.getInstanceFromEnv();
executableDao = ExecutableDao.getInstance(kylinConfig);
ExecutablePO executablePO = null;
executablePO = executableDao.getJob(jobId);
if (executablePO == null) {
logger.error("Can not find executablePO.");
return;
}
for (ExecutablePO task : executablePO.getTasks()) {
yarnLogsResources.add(task.getUuid());
}
for (String stepId : yarnLogsResources) {
logger.info("Checking step {}", stepId);
extractYarnLog(stepId, new File(yarnLogDir, stepId));
extractMRJob(stepId, new File(yarnLogDir, stepId));
}
}
Example 2
| Source File: YarnLogExtractor.java From kylin with Apache License 2.0 | 6 votes |
|
public void extractYarnLogAndMRJob(String jobId, File yarnLogDir) throws Exception {
logger.info("Collecting Yarn logs and MR counters for the Job {}", jobId);
kylinConfig = KylinConfig.getInstanceFromEnv();
executableDao = ExecutableDao.getInstance(kylinConfig);
ExecutablePO executablePO = null;
executablePO = executableDao.getJob(jobId);
if (executablePO == null) {
logger.error("Can not find executablePO.");
return;
}
for (ExecutablePO task : executablePO.getTasks()) {
yarnLogsResources.add(task.getUuid());
}
for (String stepId : yarnLogsResources) {
logger.info("Checking step {}", stepId);
extractYarnLog(stepId, new File(yarnLogDir, stepId));
extractMRJob(stepId, new File(yarnLogDir, stepId));
}
}
Example 3
| Source File: ExecutableManager.java From Kylin with Apache License 2.0 | 6 votes |
|
private static AbstractExecutable parseTo(ExecutablePO executablePO) {
if (executablePO == null) {
return null;
}
String type = executablePO.getType();
try {
Class<? extends AbstractExecutable> clazz = ClassUtil.forName(type, AbstractExecutable.class);
Constructor<? extends AbstractExecutable> constructor = clazz.getConstructor();
AbstractExecutable result = constructor.newInstance();
result.setId(executablePO.getUuid());
result.setName(executablePO.getName());
result.setParams(executablePO.getParams());
List<ExecutablePO> tasks = executablePO.getTasks();
if (tasks != null && !tasks.isEmpty()) {
Preconditions.checkArgument(result instanceof DefaultChainedExecutable);
for (ExecutablePO subTask: tasks) {
((DefaultChainedExecutable) result).addTask(parseTo(subTask));
}
}
return result;
} catch (ReflectiveOperationException e) {
throw new IllegalArgumentException("cannot parse this job:" + executablePO.getId(), e);
}
}
Example 4
| Source File: ExecutableManager.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
private AbstractExecutable parseTo(ExecutablePO executablePO) {
if (executablePO == null) {
logger.warn("executablePO is null");
return null;
}
String type = executablePO.getType();
AbstractExecutable result = newExecutable(type);
result.initConfig(config);
result.setId(executablePO.getUuid());
result.setName(executablePO.getName());
result.setParams(executablePO.getParams());
result.setPriority(executablePO.getPriority());
if (!(result instanceof BrokenExecutable)) {
List<ExecutablePO> tasks = executablePO.getTasks();
if (tasks != null && !tasks.isEmpty()) {
Preconditions.checkArgument(result instanceof ChainedExecutable);
for (ExecutablePO subTask : tasks) {
AbstractExecutable subTaskExecutable = parseTo(subTask);
if (subTaskExecutable != null) {
subTaskExecutable.setParentExecutable(result);
}
((ChainedExecutable) result).addTask(parseTo(subTask));
}
}
List<ExecutablePO> tasksForCheck = executablePO.getTasksForCheck();
if (tasksForCheck != null && !tasksForCheck.isEmpty()) {
Preconditions.checkArgument(result instanceof CheckpointExecutable);
for (ExecutablePO subTaskForCheck : tasksForCheck) {
((CheckpointExecutable) result).addTaskForCheck(parseTo(subTaskForCheck));
}
}
}
return result;
}
Example 5
| Source File: ExecutableManager.java From kylin with Apache License 2.0 | 5 votes |
|
private AbstractExecutable parseTo(ExecutablePO executablePO) {
if (executablePO == null) {
logger.warn("executablePO is null");
return null;
}
String type = executablePO.getType();
AbstractExecutable result = newExecutable(type);
result.initConfig(config);
result.setId(executablePO.getUuid());
result.setName(executablePO.getName());
result.setParams(executablePO.getParams());
result.setPriority(executablePO.getPriority());
if (!(result instanceof BrokenExecutable)) {
List<ExecutablePO> tasks = executablePO.getTasks();
if (tasks != null && !tasks.isEmpty()) {
Preconditions.checkArgument(result instanceof ChainedExecutable);
for (ExecutablePO subTask : tasks) {
AbstractExecutable subTaskExecutable = parseTo(subTask);
if (subTaskExecutable != null) {
subTaskExecutable.setParentExecutable(result);
}
((ChainedExecutable) result).addTask(parseTo(subTask));
}
}
List<ExecutablePO> tasksForCheck = executablePO.getTasksForCheck();
if (tasksForCheck != null && !tasksForCheck.isEmpty()) {
Preconditions.checkArgument(result instanceof CheckpointExecutable);
for (ExecutablePO subTaskForCheck : tasksForCheck) {
((CheckpointExecutable) result).addTaskForCheck(parseTo(subTaskForCheck));
}
}
}
return result;
}
Example 6
| Source File: CubeMetaExtractor.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
|
private void addSegAndJob(CubeInstance cube) {
if (includeSegments) {
addRequired(CubeInstance.concatResourcePath(cube.getName()));
for (CubeSegment segment : cube.getSegments(SegmentStatusEnum.READY)) {
addRequired(CubeSegment.getStatisticsResourcePath(cube.getName(), segment.getUuid()));
if (includeSegmentDetails) {
for (String dictPat : segment.getDictionaryPaths()) {
addRequired(dictPat);
}
for (String snapshotPath : segment.getSnapshotPaths()) {
addRequired(snapshotPath);
}
}
if (includeJobs) {
String lastJobId = segment.getLastBuildJobID();
if (StringUtils.isEmpty(lastJobId)) {
throw new RuntimeException("No job exist for segment :" + segment);
} else {
try {
if (onlyJobOutput) {
addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + lastJobId);
} else {
ExecutablePO executablePO = executableDao.getJob(lastJobId);
addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + lastJobId);
addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + lastJobId);
for (ExecutablePO task : executablePO.getTasks()) {
addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + task.getUuid());
addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + task.getUuid());
}
}
} catch (PersistentException e) {
throw new RuntimeException("PersistentException", e);
}
}
}
}
} else {
if (includeJobs) {
logger.warn("It's useless to set includeJobs to true when includeSegments is set to false");
}
cube.setStatus(RealizationStatusEnum.DISABLED);
cubesToTrimAndSave.add(cube);
}
}
Example 7
| Source File: MetadataCleanupJob.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
|
public void cleanup() throws Exception {
CubeManager cubeManager = CubeManager.getInstance(config);
List<String> toDeleteResource = Lists.newArrayList();
// two level resources, snapshot tables and cube statistics
for (String resourceRoot : new String[] { ResourceStore.SNAPSHOT_RESOURCE_ROOT, ResourceStore.CUBE_STATISTICS_ROOT }) {
NavigableSet<String> snapshotTables = getStore().listResources(resourceRoot);
if (snapshotTables != null) {
for (String snapshotTable : snapshotTables) {
NavigableSet<String> snapshotNames = getStore().listResources(snapshotTable);
if (snapshotNames != null)
for (String snapshot : snapshotNames) {
if (isOlderThanThreshold(getStore().getResourceTimestamp(snapshot)))
toDeleteResource.add(snapshot);
}
}
}
}
// three level resources, only dictionaries
NavigableSet<String> dictTables = getStore().listResources(ResourceStore.DICT_RESOURCE_ROOT);
if (dictTables != null) {
for (String table : dictTables) {
NavigableSet<String> tableColNames = getStore().listResources(table);
if (tableColNames != null)
for (String tableCol : tableColNames) {
NavigableSet<String> dictionaries = getStore().listResources(tableCol);
if (dictionaries != null)
for (String dict : dictionaries)
if (isOlderThanThreshold(getStore().getResourceTimestamp(dict)))
toDeleteResource.add(dict);
}
}
}
Set<String> activeResourceList = Sets.newHashSet();
for (org.apache.kylin.cube.CubeInstance cube : cubeManager.listAllCubes()) {
for (org.apache.kylin.cube.CubeSegment segment : cube.getSegments()) {
activeResourceList.addAll(segment.getSnapshotPaths());
activeResourceList.addAll(segment.getDictionaryPaths());
activeResourceList.add(segment.getStatisticsResourcePath());
}
}
toDeleteResource.removeAll(activeResourceList);
// delete old and completed jobs
ExecutableDao executableDao = ExecutableDao.getInstance(KylinConfig.getInstanceFromEnv());
List<ExecutablePO> allExecutable = executableDao.getJobs();
for (ExecutablePO executable : allExecutable) {
long lastModified = executable.getLastModified();
ExecutableOutputPO output = executableDao.getJobOutput(executable.getUuid());
if (System.currentTimeMillis() - lastModified > TIME_THREADSHOLD_FOR_JOB && (ExecutableState.SUCCEED.toString().equals(output.getStatus()) || ExecutableState.DISCARDED.toString().equals(output.getStatus()))) {
toDeleteResource.add(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + executable.getUuid());
toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + executable.getUuid());
for (ExecutablePO task : executable.getTasks()) {
toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + task.getUuid());
}
}
}
if (toDeleteResource.size() > 0) {
logger.info("The following resources have no reference or is too old, will be cleaned from metadata store: \n");
for (String s : toDeleteResource) {
logger.info(s);
if (delete == true) {
getStore().deleteResource(s);
}
}
} else {
logger.info("No resource to be cleaned up from metadata store;");
}
}
Example 8
| Source File: CubeMetaExtractor.java From kylin with Apache License 2.0 | 4 votes |
|
private void addSegAndJob(CubeInstance cube) {
if (includeSegments) {
addRequired(CubeInstance.concatResourcePath(cube.getName()));
for (CubeSegment segment : cube.getSegments(SegmentStatusEnum.READY)) {
addRequired(CubeSegment.getStatisticsResourcePath(cube.getName(), segment.getUuid()));
if (includeSegmentDetails) {
for (String dictPat : segment.getDictionaryPaths()) {
addRequired(dictPat);
}
for (String snapshotPath : segment.getSnapshotPaths()) {
addRequired(snapshotPath);
}
}
if (includeJobs) {
String lastJobId = segment.getLastBuildJobID();
if (StringUtils.isEmpty(lastJobId)) {
throw new RuntimeException("No job exist for segment :" + segment);
} else {
try {
if (onlyJobOutput) {
addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + lastJobId);
} else {
ExecutablePO executablePO = executableDao.getJob(lastJobId);
addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + lastJobId);
addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + lastJobId);
for (ExecutablePO task : executablePO.getTasks()) {
addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + task.getUuid());
addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + task.getUuid());
}
}
} catch (PersistentException e) {
throw new RuntimeException("PersistentException", e);
}
}
}
}
} else {
if (includeJobs) {
logger.warn("It's useless to set includeJobs to true when includeSegments is set to false");
}
cube.setStatus(RealizationStatusEnum.DISABLED);
cubesToTrimAndSave.add(cube);
}
}
Example 9
| Source File: MetadataCleanupJob.java From kylin with Apache License 2.0 | 4 votes |
|
public void cleanup() throws Exception {
CubeManager cubeManager = CubeManager.getInstance(config);
List<String> toDeleteResource = Lists.newArrayList();
// two level resources, snapshot tables and cube statistics
for (String resourceRoot : new String[] { ResourceStore.SNAPSHOT_RESOURCE_ROOT, ResourceStore.CUBE_STATISTICS_ROOT }) {
NavigableSet<String> snapshotTables = getStore().listResources(resourceRoot);
if (snapshotTables != null) {
for (String snapshotTable : snapshotTables) {
NavigableSet<String> snapshotNames = getStore().listResources(snapshotTable);
if (snapshotNames != null)
for (String snapshot : snapshotNames) {
if (isOlderThanThreshold(getStore().getResourceTimestamp(snapshot)))
toDeleteResource.add(snapshot);
}
}
}
}
// three level resources, only dictionaries
NavigableSet<String> dictTables = getStore().listResources(ResourceStore.DICT_RESOURCE_ROOT);
if (dictTables != null) {
for (String table : dictTables) {
NavigableSet<String> tableColNames = getStore().listResources(table);
if (tableColNames != null)
for (String tableCol : tableColNames) {
NavigableSet<String> dictionaries = getStore().listResources(tableCol);
if (dictionaries != null)
for (String dict : dictionaries)
if (isOlderThanThreshold(getStore().getResourceTimestamp(dict)))
toDeleteResource.add(dict);
}
}
}
Set<String> activeResourceList = Sets.newHashSet();
for (org.apache.kylin.cube.CubeInstance cube : cubeManager.listAllCubes()) {
for (org.apache.kylin.cube.CubeSegment segment : cube.getSegments()) {
activeResourceList.addAll(segment.getSnapshotPaths());
activeResourceList.addAll(segment.getDictionaryPaths());
activeResourceList.add(segment.getStatisticsResourcePath());
}
}
toDeleteResource.removeAll(activeResourceList);
// delete old and completed jobs
ExecutableDao executableDao = ExecutableDao.getInstance(KylinConfig.getInstanceFromEnv());
List<ExecutablePO> allExecutable = executableDao.getJobs();
for (ExecutablePO executable : allExecutable) {
long lastModified = executable.getLastModified();
ExecutableOutputPO output = executableDao.getJobOutput(executable.getUuid());
if (System.currentTimeMillis() - lastModified > TIME_THREADSHOLD_FOR_JOB && (ExecutableState.SUCCEED.toString().equals(output.getStatus()) || ExecutableState.DISCARDED.toString().equals(output.getStatus()))) {
toDeleteResource.add(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + executable.getUuid());
toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + executable.getUuid());
for (ExecutablePO task : executable.getTasks()) {
toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + task.getUuid());
}
}
}
if (toDeleteResource.size() > 0) {
logger.info("The following resources have no reference or is too old, will be cleaned from metadata store: \n");
for (String s : toDeleteResource) {
logger.info(s);
if (delete == true) {
getStore().deleteResource(s);
}
}
} else {
logger.info("No resource to be cleaned up from metadata store;");
}
}
