org.apache.kylin.common.persistence.ResourceStore Java Examples

private SnapshotTable updateDictLastModifiedTime(String snapshotPath) throws IOException {
    ResourceStore store = getStore();

    int retry = 7;
    while (retry-- > 0) {
        try {
            long now = System.currentTimeMillis();
            store.updateTimestamp(snapshotPath, now);
            logger.info("Update snapshotTable {} lastModifiedTime to {}", snapshotPath, now);
            return loadAndUpdateLocalCache(snapshotPath);
        } catch (WriteConflictException e) {
            if (retry <= 0) {
                logger.error("Retry is out, till got error, abandoning...", e);
                throw e;
            }
            logger.warn("Write conflict to update snapshotTable " +  snapshotPath + " retry remaining " + retry
                    + ", will retry...");
        }
    }
    // update cache
    return loadAndUpdateLocalCache(snapshotPath);
}
 

private HybridManager(KylinConfig cfg) throws IOException {
    logger.info("Initializing HybridManager with config " + cfg);
    this.config = cfg;
    this.hybridMap = new CaseInsensitiveStringCache<HybridInstance>(config, "hybrid");
    this.crud = new CachedCrudAssist<HybridInstance>(getStore(), ResourceStore.HYBRID_RESOURCE_ROOT,
            HybridInstance.class, hybridMap) {
        @Override
        protected HybridInstance initEntityAfterReload(HybridInstance hybridInstance, String resourceName) {
            hybridInstance.setConfig(config);
            return hybridInstance; // noop
        }
    };

    // touch lower level metadata before registering my listener
    crud.reloadAll();
    Broadcaster.getInstance(config).registerListener(new HybridSyncListener(), "hybrid", "cube");
}
 

public static void dumpResources(KylinConfig kylinConfig, File metaDir, Set<String> dumpList) throws IOException {
    long startTime = System.currentTimeMillis();

    ResourceStore from = ResourceStore.getStore(kylinConfig);
    KylinConfig localConfig = KylinConfig.createInstanceFromUri(metaDir.getAbsolutePath());
    ResourceStore to = ResourceStore.getStore(localConfig);
    for (String path : dumpList) {
        RawResource res = from.getResource(path);
        if (res == null)
            throw new IllegalStateException("No resource found at -- " + path);
        to.putResource(path, res.content(), res.lastModified());
        res.content().close();
    }

    logger.debug("Dump resources to {} took {} ms", metaDir, System.currentTimeMillis() - startTime);
}
 

private String checkDupByInfo(SnapshotTable snapshot) throws IOException {
    ResourceStore store = getStore();
    String resourceDir = snapshot.getResourceDir();
    NavigableSet<String> existings = store.listResources(resourceDir);
    if (existings == null)
        return null;

    TableSignature sig = snapshot.getSignature();
    for (String existing : existings) {
        SnapshotTable existingTable = load(existing, false); // skip cache,
        // direct load from store
        if (existingTable != null && sig.equals(existingTable.getSignature()))
            return existing;
    }

    return null;
}
 

private DictionaryInfo findLargestDictInfo(DictionaryInfo dictInfo) throws IOException {
    final ResourceStore store = getStore();
    final List<DictionaryInfo> allResources = store.getAllResources(dictInfo.getResourceDir(),
            DictionaryInfoSerializer.INFO_SERIALIZER);

    DictionaryInfo largestDict = null;
    for (DictionaryInfo dictionaryInfo : allResources) {
        if (largestDict == null) {
            largestDict = dictionaryInfo;
            continue;
        }

        if (largestDict.getCardinality() < dictionaryInfo.getCardinality()) {
            largestDict = dictionaryInfo;
        }
    }
    return largestDict;
}
 

@Test
public void testCreateAndDrop() throws Exception {

    KylinConfig config = getTestConfig();
    CubeManager cubeMgr = CubeManager.getInstance(config);
    ProjectManager prjMgr = ProjectManager.getInstance(config);
    ResourceStore store = getStore();

    // clean legacy in case last run failed
    store.deleteResource("/cube/a_whole_new_cube.json");

    CubeDescManager cubeDescMgr = getCubeDescManager();
    CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc");
    CubeInstance createdCube = cubeMgr.createCube("a_whole_new_cube", ProjectInstance.DEFAULT_PROJECT_NAME, desc,
            null);
    assertTrue(createdCube.equals(cubeMgr.getCube("a_whole_new_cube")));

    assertTrue(prjMgr.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME).contains(createdCube));

    CubeInstance droppedCube = CubeManager.getInstance(getTestConfig()).dropCube("a_whole_new_cube", false);
    assertTrue(createdCube.equals(droppedCube));

    assertTrue(!prjMgr.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME).contains(droppedCube));

    assertNull(CubeManager.getInstance(getTestConfig()).getCube("a_whole_new_cube"));
}
 

private TempStatementManager(KylinConfig cfg) throws IOException {
    this.config = cfg;
    this.tmpStatMap = new CaseInsensitiveStringCache<>(config, "temp_statement");
    this.crud = new CachedCrudAssist<TempStatementEntity>(getStore(), ResourceStore.TEMP_STATMENT_RESOURCE_ROOT,
            TempStatementEntity.class, tmpStatMap) {
        @Override
        protected TempStatementEntity initEntityAfterReload(TempStatementEntity t, String resourceName) {
            return t; // noop
        }
    };

    crud.reloadAll();

    // touch lower level metadata before registering my listener
    Broadcaster.getInstance(config).registerListener(new TempStatementSyncListener(), "temp_statement");
}
 

@Test
public void testReloadCache() throws Exception {
    ResourceStore store = getStore();

    // clean legacy in case last run failed
    store.deleteResource("/cube/a_whole_new_cube.json");
    CubeDescManager cubeDescMgr = getCubeDescManager();
    CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc");
    cubeManager.createCube("a_whole_new_cube", "default", desc, null);

    CubeInstance createdCube = cubeManager.getCube("a_whole_new_cube");
    assertEquals(0, createdCube.getSegments().size());
    assertEquals(RealizationStatusEnum.DISABLED, createdCube.getStatus());

    cubeManager.updateCubeStatus(createdCube, RealizationStatusEnum.READY);

    assertEquals(RealizationStatusEnum.READY, cubeManager.getCube("a_whole_new_cube").getStatus());
}
 

private ProjectManager(KylinConfig config) throws IOException {
    logger.info("Initializing ProjectManager with metadata url " + config);
    this.config = config;
    this.projectMap = new CaseInsensitiveStringCache<ProjectInstance>(config, "project");
    this.l2Cache = new ProjectL2Cache(this);
    this.crud = new CachedCrudAssist<ProjectInstance>(getStore(), ResourceStore.PROJECT_RESOURCE_ROOT,
            ProjectInstance.class, projectMap) {
        @Override
        protected ProjectInstance initEntityAfterReload(ProjectInstance prj, String resourceName) {
            prj.init();
            return prj;
        }
    };

    // touch lower level metadata before registering my listener
    crud.reloadAll();
    Broadcaster.getInstance(config).registerListener(new ProjectSyncListener(), "project");
}
 

private CubeDescManager(KylinConfig cfg) throws IOException {
    logger.info("Initializing CubeDescManager with config {}", cfg);
    this.config = cfg;
    this.cubeDescMap = new CaseInsensitiveStringCache<>(config, "cube_desc");
    this.crud = new CachedCrudAssist<CubeDesc>(getStore(), ResourceStore.CUBE_DESC_RESOURCE_ROOT, CubeDesc.class,
            cubeDescMap) {
        @Override
        protected CubeDesc initEntityAfterReload(CubeDesc cubeDesc, String resourceName) {
            if (cubeDesc.isDraft())
                throw new IllegalArgumentException(String.format(Locale.ROOT, CUBE_SHOULD_NOT_BE_DRAFT_MSG, cubeDesc.getName()));

            try {
                cubeDesc.init(config);
            } catch (Exception e) {
                logger.warn(String.format(Locale.ROOT, BROKEN_CUBE_MSG, cubeDesc.resourceName()), e);
                cubeDesc.addError(e.toString());
            }
            return cubeDesc;
        }
    };

    // touch lower level metadata before registering my listener
    crud.reloadAll();
    Broadcaster.getInstance(config).registerListener(new CubeDescSyncListener(), "cube_desc");
}
 

public void migrate(KylinConfig kylinConfig) throws IOException {
    if (!checkIfNeedMigrate(kylinConfig)) {
        logger.info("Do not need to migrate acl table data");
        return;
    } else {
        if (!kylinConfig.getServerMode().equals("all")) {
            throw new IllegalStateException(
                    "Please make sure that you have config kylin.server.mode=all before migrating data");
        }
        logger.info("Start to migrate acl table data");
        ResourceStore store = ResourceStore.getStore(kylinConfig);
        String userTableName = kylinConfig.getMetadataUrlPrefix() + AclConstant.USER_TABLE_NAME;
        //System.out.println("user table name : " + userTableName);
        String aclTableName = kylinConfig.getMetadataUrlPrefix() + AclConstant.ACL_TABLE_NAME;
        if (needMigrateTable(aclTableName, store)) {
            logger.info("Migrate table : {}", aclTableName);
            migrate(store, AclConstant.ACL_TABLE_NAME, kylinConfig);
        }
        if (needMigrateTable(userTableName, store)) {
            logger.info("Migrate table : {}", userTableName);
            migrate(store, AclConstant.USER_TABLE_NAME, kylinConfig);
        }
    }
}
 

private void initSrcExt() throws IOException {
    this.srcExtMap = new CaseInsensitiveStringCache<>(config, "table_ext");
    this.srcExtCrud = new CachedCrudAssist<TableExtDesc>(getStore(), ResourceStore.TABLE_EXD_RESOURCE_ROOT,
            TableExtDesc.class, srcExtMap) {
        @Override
        protected TableExtDesc initEntityAfterReload(TableExtDesc t, String resourceName) {
            // convert old tableExt json to new one
            if (t.getIdentity() == null) {
                t = convertOldTableExtToNewer(resourceName);
            }

            String prj = TableDesc.parseResourcePath(resourceName).getProject();
            t.init(prj);
            return t;
        }
    };
    srcExtCrud.reloadAll();
    Broadcaster.getInstance(config).registerListener(new SrcTableExtSyncListener(), "table_ext");
}
 

public void migrate(KylinConfig kylinConfig) throws IOException {
    if (!checkIfNeedMigrate(kylinConfig)) {
        logger.info("Do not need to migrate acl table data");
        return;
    } else {
        if (!kylinConfig.getServerMode().equals("all")) {
            throw new IllegalStateException(
                    "Please make sure that you have config kylin.server.mode=all before migrating data");
        }
        logger.info("Start to migrate acl table data");
        ResourceStore store = ResourceStore.getStore(kylinConfig);
        String userTableName = kylinConfig.getMetadataUrlPrefix() + AclConstant.USER_TABLE_NAME;
        //System.out.println("user table name : " + userTableName);
        String aclTableName = kylinConfig.getMetadataUrlPrefix() + AclConstant.ACL_TABLE_NAME;
        if (needMigrateTable(aclTableName, store)) {
            logger.info("Migrate table : {}", aclTableName);
            migrate(store, AclConstant.ACL_TABLE_NAME, kylinConfig);
        }
        if (needMigrateTable(userTableName, store)) {
            logger.info("Migrate table : {}", userTableName);
            migrate(store, AclConstant.USER_TABLE_NAME, kylinConfig);
        }
    }
}
 

private String checkDupByInfo(SnapshotTable snapshot) throws IOException {
    ResourceStore store = getStore();
    String resourceDir = snapshot.getResourceDir();
    NavigableSet<String> existings = store.listResources(resourceDir);
    if (existings == null)
        return null;

    TableSignature sig = snapshot.getSignature();
    for (String existing : existings) {
        SnapshotTable existingTable = load(existing, false); // skip cache,
        // direct load from store
        if (existingTable != null && sig.equals(existingTable.getSignature()))
            return existing;
    }

    return null;
}
 

public void save(Draft draft) throws IOException {
    if (draft.getUuid() == null) {
        throw new IllegalArgumentException();
    }

    Draft youngerSelf = load(draft.getUuid());
    if (youngerSelf != null) {
        draft.setLastModified(youngerSelf.getLastModified());
    }

    ResourceStore store = getStore();
    store.checkAndPutResource(draft.getResourcePath(), draft, DRAFT_SERIALIZER);
    
    logger.trace("Saved " + draft);
}
 

public List<ExecutableOutputPO> getJobOutputs(long timeStart, long timeEndExclusive) throws PersistentException {
    try {
        return store.getAllResources(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT, false,
                new ResourceStore.VisitFilter(timeStart, timeEndExclusive),
                new ContentReader(JOB_OUTPUT_SERIALIZER));
    } catch (IOException e) {
        logger.error("error get all Jobs:", e);
        throw new PersistentException(e);
    }
}
 

private String checkDupByContent(SnapshotTable snapshot) throws IOException {
    ResourceStore store = getStore();
    String resourceDir = snapshot.getResourceDir();
    NavigableSet<String> existings = store.listResources(resourceDir);
    if (existings == null)
        return null;

    for (String existing : existings) {
        SnapshotTable existingTable = load(existing, true); // skip cache, direct load from store
        if (existingTable != null && existingTable.equals(snapshot))
            return existing;
    }

    return null;
}
 

public static void collectFiles(ResourceStore src, String path, List<String> ret) throws IOException {
    NavigableSet<String> children = src.listResources(path);

    if (children == null) {
        // case of resource (not a folder)

        ret.add(path);
    } else {
        // case of folder

        for (String child : children) {
            collectFiles(src, child, ret);
        }
    }
}
 

@Test
public void testBuildSameSnapshotSameTime() throws InterruptedException, IOException {
    final int threadCount = 3;
    final ExecutorService executorService = Executors.newFixedThreadPool(threadCount);
    final CountDownLatch countDownLatch = new CountDownLatch(threadCount);
    final TableDesc tableDesc = genTableDesc("TEST_TABLE");

    kylinConfig = KylinConfig.getInstanceFromEnv();
    snapshotManager = SnapshotManager.getInstance(kylinConfig);
    ResourceStore store = ResourceStore.getStore(kylinConfig);

    for (int i = 0; i < threadCount; ++i) {
        executorService.submit(new Runnable() {
            @Override
            public void run() {
                try {
                    snapshotManager.buildSnapshot(genTable("./origin", expect), tableDesc, kylinConfig);
                } catch (IOException e) {
                    Assert.fail();
                } finally {
                    countDownLatch.countDown();
                }
            }
        });
    }
    countDownLatch.await();
    Assert.assertEquals(1, store.listResources("/table_snapshot/NULL.TEST_TABLE").size());
}
 

private void upgradeJobInstance(String path) throws IOException {
    JobInstance job = getStore().getResource(path, JobInstance.class, new JsonSerializer<JobInstance>(JobInstance.class));
    CubingJob cubingJob = new CubingJob();
    cubingJob.setId(job.getId());
    cubingJob.setName(job.getName());
    cubingJob.setSubmitter(job.getSubmitter());
    for (JobInstance.JobStep step : job.getSteps()) {
        final AbstractExecutable executable = parseToExecutable(step);
        cubingJob.addTask(executable);
    }
    getExecutableManager().addJob(cubingJob);

    cubingJob.setStartTime(job.getExecStartTime());
    cubingJob.setEndTime(job.getExecEndTime());
    cubingJob.setMapReduceWaitTime(job.getMrWaiting());
    getExecutableManager().resetJobOutput(cubingJob.getId(), parseState(job.getStatus()), job.getStatus().toString());

    for (int i = 0, size = job.getSteps().size(); i < size; ++i) {
        final JobInstance.JobStep jobStep = job.getSteps().get(i);
        final InputStream inputStream = getStore().getResource(ResourceStore.JOB_OUTPUT_PATH_ROOT + "/" + job.getId() + "." + i);

        String output = null;
        if (inputStream != null) {
            HashMap<String, String> job_output = JsonUtil.readValue(inputStream, HashMap.class);

            if (job_output != null) {
                output = job_output.get("output");
            }
        }
        updateJobStepOutput(jobStep, output, cubingJob.getTasks().get(i));
    }
}
 

private String renameTableWithinProject(String srcItem) {
    if (dstProject != null && srcItem.contains(ResourceStore.TABLE_RESOURCE_ROOT)) {
        String tableIdentity = TableDesc.parseResourcePath(srcItem).getTable();
        if (srcItem.contains(ResourceStore.TABLE_EXD_RESOURCE_ROOT))
            return TableExtDesc.concatResourcePath(tableIdentity, dstProject);
        else
            return ResourceStore.TABLE_RESOURCE_ROOT + "/" + tableIdentity + "--" + dstProject + ".json";
    }
    return srcItem;
}
 

private void updateMetadataAfterMerge(String cubeId) {
    String buildStepUrl = getParam(MetadataConstants.P_OUTPUT_META_URL);
    KylinConfig buildConfig = KylinConfig.createKylinConfig(this.getConfig());
    buildConfig.setMetadataUrl(buildStepUrl);
    ResourceStore resourceStore = ResourceStore.getStore(buildConfig);
    String mergedSegmentId = getParam(CubingExecutableUtil.SEGMENT_ID);
    AfterMergeOrRefreshResourceMerger merger = new AfterMergeOrRefreshResourceMerger(buildConfig);
    merger.merge(cubeId, mergedSegmentId, resourceStore, getParam(MetadataConstants.P_JOB_TYPE));
}
 

private CubeDesc loadOldCubeDesc(String path) throws IOException {
    ResourceStore store = getStore();

    CubeDesc ndesc = store.getResource(path, CubeDesc.class, CUBE_DESC_SERIALIZER_V1);

    if (StringUtils.isBlank(ndesc.getName())) {
        throw new IllegalStateException("CubeDesc name must not be blank");
    }

    Map<String, TableDesc> tableMap = getMetadataManager().getAllTablesMap();
    Map<String, TableDesc> newMap = Maps.newHashMap();
    for (Entry<String, TableDesc> entry : tableMap.entrySet()) {
        String t = entry.getKey();

        if (t.indexOf(".") > 0) {
            newMap.put(t.substring(t.indexOf(".") + 1), entry.getValue());

        }
    }
    ndesc.init(KylinConfig.getInstanceFromEnv(), newMap);

    if (ndesc.getError().isEmpty() == false) {
        throw new IllegalStateException("Cube desc at " + path + " has issues: " + ndesc.getError());
    }

    return ndesc;
}
 

@Test
public void testGetAllCubes() throws Exception {
    final ResourceStore store = ResourceStore.getStore(getTestConfig());
    final NavigableSet<String> cubePath = store.listResources(ResourceStore.CUBE_RESOURCE_ROOT);
    assertTrue(cubePath.size() > 1);

    final List<CubeInstance> cubes = store.getAllResources(ResourceStore.CUBE_RESOURCE_ROOT,
            CubeManager.CUBE_SERIALIZER);
    assertEquals(cubePath.size(), cubes.size());
}
 

public void saveTableExt(TableExtDesc tableExt, String prj) throws IOException {
    try (AutoLock lock = srcExtMapLock.lockForWrite()) {
        if (tableExt.getUuid() == null || tableExt.getIdentity() == null) {
            throw new IllegalArgumentException();
        }

        // updating a legacy global table
        if (tableExt.getProject() == null) {
            if (getTableExt(tableExt.getIdentity(), prj).getProject() != null)
                throw new IllegalStateException(
                        "Updating a legacy global TableExtDesc while a project level version exists: "
                                + tableExt.getIdentity() + ", " + prj);
            prj = tableExt.getProject();
        }

        tableExt.init(prj);

        // what is this doing??
        String path = TableExtDesc.concatResourcePath(tableExt.getIdentity(), prj);
        ResourceStore store = getStore();
        TableExtDesc t = store.getResource(path, TABLE_EXT_SERIALIZER);
        if (t != null && t.getIdentity() == null)
            store.deleteResource(path);

        srcExtCrud.save(tableExt);
    }
}
 

@Override
public void merge(AbstractExecutable abstractExecutable) {
    String buildStepUrl = abstractExecutable.getParam(MetadataConstants.P_OUTPUT_META_URL);
    KylinConfig buildConfig = KylinConfig.createKylinConfig(this.getConfig());
    buildConfig.setMetadataUrl(buildStepUrl);
    ResourceStore resourceStore = ResourceStore.getStore(buildConfig);
    String cubeId = abstractExecutable.getParam(MetadataConstants.P_CUBE_ID);
    String segmentId = abstractExecutable.getParam(CubingExecutableUtil.SEGMENT_ID);
    merge(cubeId, segmentId, resourceStore, abstractExecutable.getParam(MetadataConstants.P_JOB_TYPE));
}
 

public void saveNewTableExtFromOld(String oldTableId, String prj, String newTableId) throws IOException {
    try (AutoLock lock = srcExtMapLock.lockForWrite()) {
        String path = TableExtDesc.concatResourcePath(oldTableId, prj);
        ResourceStore store = getStore();
        TableExtDesc newTableExt = store.getResource(path, TABLE_EXT_SERIALIZER);
        if (newTableExt != null) {
            newTableExt.setIdentity(newTableId);
            newTableExt.setLastModified(0L);

            newTableExt.init(prj);
            srcExtCrud.save(newTableExt);
        }
    }
}
 

private void attachSegmentMetadataWithDict(CubeSegment segment) throws IOException {
    Set<String> dumpList = new LinkedHashSet<>();
    dumpList.addAll(JobRelatedMetaUtil.collectCubeMetadata(segment.getCubeInstance()));
    dumpList.addAll(segment.getDictionaryPaths());
    ResourceStore rs = ResourceStore.getStore(segment.getConfig());
    if (rs.exists(segment.getStatisticsResourcePath())) {
        // cube statistics is not available for new segment
        dumpList.add(segment.getStatisticsResourcePath());
    }

    JobRelatedMetaUtil.dumpAndUploadKylinPropsAndMetadata(dumpList, (KylinConfigExt) segment.getConfig(),
            this.getParam(FlinkCubingByLayer.OPTION_META_URL.getOpt()));
}
 

private synchronized CubeInstance loadCubeInstance(String path) throws IOException {
    ResourceStore store = getStore();

    CubeInstance cubeInstance;
    try {
        cubeInstance = store.getResource(path, CubeInstance.class, CUBE_SERIALIZER);
        cubeInstance.setConfig(config);

        if (StringUtils.isBlank(cubeInstance.getName()))
            throw new IllegalStateException("CubeInstance name must not be blank, at " + path);

        if (cubeInstance.getDescriptor() == null)
            throw new IllegalStateException("CubeInstance desc not found '" + cubeInstance.getDescName() + "', at " + path);

        final String cubeName = cubeInstance.getName();
        cubeMap.putLocal(cubeName, cubeInstance);

        for (CubeSegment segment : cubeInstance.getSegments()) {
            usedStorageLocation.put(cubeName, segment.getStorageLocationIdentifier());
        }

        return cubeInstance;
    } catch (Exception e) {
        logger.error("Error during load cube instance " + path, e);
        return null;
    }
}
 

public CachedCrudAssist(ResourceStore store, String resourceRootPath, String resourcePathSuffix,
        Class<T> entityType, SingleValueCache<String, T> cache, boolean compact) {
    this.store = store;
    this.entityType = entityType;
    this.resRootPath = resourceRootPath;
    this.resPathSuffix = resourcePathSuffix;
    this.serializer = new JsonSerializer<T>(entityType, compact);
    this.cache = cache;

    this.checkCopyOnWrite = store.getConfig().isCheckCopyOnWrite();

    Preconditions.checkArgument(resRootPath.startsWith("/"));
    Preconditions.checkArgument(resRootPath.endsWith("/") == false);
}