Java Code Examples for org.apache.kylin.common.persistence.ResourceStore#getResource()

public static void dumpResources(KylinConfig kylinConfig, File metaDir, Set<String> dumpList) throws IOException {
    long startTime = System.currentTimeMillis();

    ResourceStore from = ResourceStore.getStore(kylinConfig);
    KylinConfig localConfig = KylinConfig.createInstanceFromUri(metaDir.getAbsolutePath());
    ResourceStore to = ResourceStore.getStore(localConfig);
    for (String path : dumpList) {
        RawResource res = from.getResource(path);
        if (res == null)
            throw new IllegalStateException("No resource found at -- " + path);
        to.putResource(path, res.content(), res.lastModified());
        res.content().close();
    }

    logger.debug("Dump resources to {} took {} ms", metaDir, System.currentTimeMillis() - startTime);
}
 

private ProjectInstance reloadProjectAt(String path) throws IOException {
    ResourceStore store = getStore();

    ProjectInstance projectInstance = store.getResource(path, ProjectInstance.class, PROJECT_SERIALIZER);
    if (projectInstance == null) {
        logger.warn("reload project at path:" + path + " not found, this:" + this.toString());
        return null;
    }

    projectInstance.init();

    projectMap.putLocal(projectInstance.getName(), projectInstance);
    clearL2Cache();

    return projectInstance;
}
 

/**
 * @param cuboidScheduler if it's null, part of it's functions will not be supported
 */
public CubeStatsReader(CubeSegment cubeSegment, CuboidScheduler cuboidScheduler, KylinConfig kylinConfig)
        throws IOException {
    ResourceStore store = ResourceStore.getStore(kylinConfig);
    String statsKey = cubeSegment.getStatisticsResourcePath();
    RawResource resource = store.getResource(statsKey);
    if (resource == null)
        throw new IllegalStateException("Missing resource at " + statsKey);

    File tmpSeqFile = writeTmpSeqFile(resource.content());
    Path path = new Path(HadoopUtil.fixWindowsPath("file://" + tmpSeqFile.getAbsolutePath()));

    CubeStatsResult cubeStatsResult = new CubeStatsResult(path, kylinConfig.getCubeStatsHLLPrecision());
    tmpSeqFile.delete();

    this.seg = cubeSegment;
    this.cuboidScheduler = cuboidScheduler;
    this.samplingPercentage = cubeStatsResult.getPercentage();
    this.mapperNumberOfFirstBuild = cubeStatsResult.getMapperNumber();
    this.mapperOverlapRatioOfFirstBuild = cubeStatsResult.getMapperOverlapRatio();
    this.cuboidRowEstimatesHLL = cubeStatsResult.getCounterMap();
    this.sourceRowCount = cubeStatsResult.getSourceRecordCount();
}
 

public static void dumpResources(KylinConfig kylinConfig, File metaDir, Set<String> dumpList) throws IOException {
    long startTime = System.currentTimeMillis();

    ResourceStore from = ResourceStore.getStore(kylinConfig);
    KylinConfig localConfig = KylinConfig.createInstanceFromUri(metaDir.getAbsolutePath());
    ResourceStore to = ResourceStore.getStore(localConfig);
    for (String path : dumpList) {
        RawResource res = from.getResource(path);
        if (res == null)
            throw new IllegalStateException("No resource found at -- " + path);
        to.putResource(path, res.content(), res.lastModified());
        res.content().close();
    }

    logger.debug("Dump resources to {} took {} ms", metaDir, System.currentTimeMillis() - startTime);
}
 

public List<Draft> list(String project) throws IOException {
    List<Draft> result = new ArrayList<>();
    ResourceStore store = getStore();
    NavigableSet<String> listPath = store.listResources(ResourceStore.DRAFT_RESOURCE_ROOT);
    if (listPath == null)
        return result;
    
    for (String path : listPath) {
        Draft draft = store.getResource(path, DRAFT_SERIALIZER);
        
        if (draft == null)
            continue;
        
        if (project == null || project.equals(draft.getProject()))
            result.add(draft);
    }
    return result;
}
 

private StreamingSourceConfig loadStreamingConfigAt(String path) throws IOException {
    ResourceStore store = getStore();
    StreamingSourceConfig streamingDesc = store.getResource(path, STREAMING_SERIALIZER);
    if (streamingDesc == null) {
        return null;
    }
    if (StringUtils.isBlank(streamingDesc.getName())) {
        throw new IllegalStateException("StreamingSourceConfig name must not be blank");
    }
    return streamingDesc;
}
 

private SnapshotTable load(String resourcePath, boolean loadData) throws IOException {
    logger.info("Loading snapshotTable from {}, with loadData: {}", resourcePath, loadData);
    ResourceStore store = getStore();

    SnapshotTable table = store.getResource(resourcePath,
            loadData ? SnapshotTableSerializer.FULL_SERIALIZER : SnapshotTableSerializer.INFO_SERIALIZER);

    if (loadData)
        logger.debug("Loaded snapshot at {}", resourcePath);

    return table;
}
 

private void reloadCubeMetadata(String cubeName) throws IOException {
    KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    ResourceStore resourceStore = ResourceStore.getStore(kylinConfig);
    CubeInstance rawCubeInstance = resourceStore.getResource(CubeInstance.concatResourcePath(cubeName),
             CubeManager.CUBE_SERIALIZER);
    CubeDesc rawCubeDesc = resourceStore.getResource(CubeDesc.concatResourcePath(rawCubeInstance.getDescName()),
            CubeDescManager.CUBE_DESC_SERIALIZER);
    DataModelDesc rawModel = resourceStore.getResource(
            DataModelDesc.concatResourcePath(rawCubeDesc.getModelName()),
            new JsonSerializer<>(DataModelDesc.class));
    ProjectManager projectManager = ProjectManager.getInstance(kylinConfig);
    List<ProjectInstance> projects = projectManager.findProjectsByModel(rawModel.getName());
    if (projects.isEmpty()) {
        projectManager.reloadAll();
        projects = projectManager.findProjectsByModel(rawModel.getName());
    }
    if (projects.size() != 1) {
        throw new IllegalArgumentException("the cube:" + cubeName + " is not in any project");
    }

    TableMetadataManager.getInstance(kylinConfig).reloadSourceTableQuietly(rawModel.getRootFactTableName(),
            projects.get(0).getName());
    DataModelManager.getInstance(kylinConfig).reloadDataModel(rawModel.getName());
    CubeDescManager.getInstance(kylinConfig).reloadCubeDescLocal(cubeName);
    CubeInstance cubeInstance = CubeManager.getInstance(kylinConfig).reloadCubeQuietly(cubeName);
    StreamingSourceConfigManager.getInstance(kylinConfig).reloadStreamingConfigLocal(
            cubeInstance.getRootFactTable());
}
 

public void saveNewTableExtFromOld(String oldTableId, String prj, String newTableId) throws IOException {
    try (AutoLock lock = srcExtMapLock.lockForWrite()) {
        String path = TableExtDesc.concatResourcePath(oldTableId, prj);
        ResourceStore store = getStore();
        TableExtDesc newTableExt = store.getResource(path, TABLE_EXT_SERIALIZER);
        if (newTableExt != null) {
            newTableExt.setIdentity(newTableId);
            newTableExt.setLastModified(0L);

            newTableExt.init(prj);
            srcExtCrud.save(newTableExt);
        }
    }
}
 

private DataModelDesc reloadDataModelDescAt(String path) {
    ResourceStore store = getStore();
    try {
        DataModelDesc dataModelDesc = store.getResource(path, DataModelDesc.class, MODELDESC_SERIALIZER);
        dataModelDesc.init(this.getAllTablesMap());
        dataModelDescMap.putLocal(dataModelDesc.getName(), dataModelDesc);
        return dataModelDesc;
    } catch (IOException e) {
        throw new IllegalStateException("Error to load" + path, e);
    }
}
 

private void dumpResources(KylinConfig kylinConfig, File metaDir, ArrayList<String> dumpList) throws IOException {
    ResourceStore from = ResourceStore.getStore(kylinConfig);
    KylinConfig localConfig = KylinConfig.createInstanceFromUri(metaDir.getAbsolutePath());
    ResourceStore to = ResourceStore.getStore(localConfig);
    for (String path : dumpList) {
        InputStream in = from.getResource(path);
        if (in == null)
            throw new IllegalStateException("No resource found at -- " + path);
        long ts = from.getResourceTimestamp(path);
        to.putResource(path, in, ts);
        //The following log is duplicate with in ResourceStore
        //log.info("Dumped resource " + path + " to " + metaDir.getAbsolutePath());
    }
}
 

private TableDesc reloadSourceTableAt(String path) throws IOException {
    ResourceStore store = getStore();
    TableDesc t = store.getResource(path, TableDesc.class, TABLE_SERIALIZER);
    if (t == null) {
        logger.error("Didn't load table at " + path);
        return null;
    }
    t.init();

    String tableIdentity = t.getIdentity();

    srcTableMap.putLocal(tableIdentity, t);

    return t;
}
 

private StreamingSourceConfig loadStreamingConfigAt(String path) throws IOException {
    ResourceStore store = getStore();
    StreamingSourceConfig streamingDesc = store.getResource(path, STREAMING_SERIALIZER);
    if (streamingDesc == null) {
        return null;
    }
    if (StringUtils.isBlank(streamingDesc.getName())) {
        throw new IllegalStateException("StreamingSourceConfig name must not be blank");
    }
    return streamingDesc;
}
 

private CubeDesc loadOldCubeDesc(String path) throws IOException {
    ResourceStore store = getStore();

    CubeDesc ndesc = store.getResource(path, CubeDesc.class, CUBE_DESC_SERIALIZER_V1);

    if (StringUtils.isBlank(ndesc.getName())) {
        throw new IllegalStateException("CubeDesc name must not be blank");
    }

    Map<String, TableDesc> tableMap = getMetadataManager().getAllTablesMap();
    Map<String, TableDesc> newMap = Maps.newHashMap();
    for (Entry<String, TableDesc> entry : tableMap.entrySet()) {
        String t = entry.getKey();

        if (t.indexOf(".") > 0) {
            newMap.put(t.substring(t.indexOf(".") + 1), entry.getValue());

        }
    }
    ndesc.init(KylinConfig.getInstanceFromEnv(), newMap);

    if (ndesc.getError().isEmpty() == false) {
        throw new IllegalStateException("Cube desc at " + path + " has issues: " + ndesc.getError());
    }

    return ndesc;
}
 

private synchronized CubeInstance loadCubeInstance(String path) throws IOException {
    ResourceStore store = getStore();

    CubeInstance cubeInstance;
    try {
        cubeInstance = store.getResource(path, CubeInstance.class, CUBE_SERIALIZER);
        cubeInstance.setConfig(config);

        if (StringUtils.isBlank(cubeInstance.getName()))
            throw new IllegalStateException("CubeInstance name must not be blank, at " + path);

        if (cubeInstance.getDescriptor() == null)
            throw new IllegalStateException("CubeInstance desc not found '" + cubeInstance.getDescName() + "', at " + path);

        final String cubeName = cubeInstance.getName();
        cubeMap.putLocal(cubeName, cubeInstance);

        for (CubeSegment segment : cubeInstance.getSegments()) {
            usedStorageLocation.put(cubeName, segment.getStorageLocationIdentifier());
        }

        return cubeInstance;
    } catch (Exception e) {
        logger.error("Error during load cube instance " + path, e);
        return null;
    }
}
 

public static void duplicateFactTableData(String factTableName, String joinType) throws IOException {
    // duplicate a copy of this fact table, with a naming convention with fact.csv.inner or fact.csv.left
    // so that later test cases can select different data files
    ResourceStore store = ResourceStore.getStore(config());
    InputStream in = store.getResource("/data/" + factTableName + ".csv");
    String factTablePathWithJoinType = "/data/" + factTableName + ".csv." + joinType.toLowerCase();
    store.deleteResource(factTablePathWithJoinType);
    store.putResource(factTablePathWithJoinType, in, System.currentTimeMillis());
    in.close();
}
 

private IIDesc loadIIDesc(String path) throws IOException {
    ResourceStore store = getStore();
    logger.info("Loading IIDesc " + store.getReadableResourcePath(path));

    IIDesc ndesc = store.getResource(path, IIDesc.class, II_DESC_SERIALIZER);

    if (StringUtils.isBlank(ndesc.getName())) {
        throw new IllegalStateException("IIDesc name must not be blank");
    }

    ndesc.init(getMetadataManager());

    return ndesc;
}
 

public void saveTableExt(TableExtDesc tableExt, String prj) throws IOException {
    try (AutoLock lock = srcExtMapLock.lockForWrite()) {
        if (tableExt.getUuid() == null || tableExt.getIdentity() == null) {
            throw new IllegalArgumentException();
        }

        // updating a legacy global table
        if (tableExt.getProject() == null) {
            if (getTableExt(tableExt.getIdentity(), prj).getProject() != null)
                throw new IllegalStateException(
                        "Updating a legacy global TableExtDesc while a project level version exists: "
                                + tableExt.getIdentity() + ", " + prj);
            prj = tableExt.getProject();
        }

        tableExt.init(prj);

        // what is this doing??
        String path = TableExtDesc.concatResourcePath(tableExt.getIdentity(), prj);
        ResourceStore store = getStore();
        TableExtDesc t = store.getResource(path, TABLE_EXT_SERIALIZER);
        if (t != null && t.getIdentity() == null)
            store.deleteResource(path);

        srcExtCrud.save(tableExt);
    }
}
 

private ExtTableSnapshotInfo load(String resourcePath) throws IOException {
    ResourceStore store = TableMetadataManager.getInstance(this.config).getStore();
    ExtTableSnapshotInfo snapshot = store.getResource(resourcePath, SNAPSHOT_SERIALIZER);

    return snapshot;
}
 

public Draft load(String uuid) throws IOException {
    ResourceStore store = getStore();
    Draft draft = store.getResource(Draft.concatResourcePath(uuid), DRAFT_SERIALIZER);
    return draft;
}