Java Code Examples for org.apache.kylin.common.KylinConfig#getMetadataUrl()

The following examples show how to use org.apache.kylin.common.KylinConfig#getMetadataUrl() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ResourceStore.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private static ResourceStore createResourceStore(KylinConfig kylinConfig) {
    StorageURL metadataUrl = kylinConfig.getMetadataUrl();
    logger.info("Using metadata url {} for resource store", metadataUrl);
    String clsName = kylinConfig.getResourceStoreImpls().get(metadataUrl.getScheme());
    try {
        Class<? extends ResourceStore> cls = ClassUtil.forName(clsName, ResourceStore.class);
        ResourceStore store = cls.getConstructor(KylinConfig.class).newInstance(kylinConfig);
        if (!store.exists(METASTORE_UUID_TAG)) {
            store.checkAndPutResource(METASTORE_UUID_TAG, new StringEntity(store.createMetaStoreUUID()), 0,
                    StringEntity.serializer);
        }
        return store;
    } catch (Throwable e) {
        throw new IllegalArgumentException("Failed to find metadata store by url: " + metadataUrl, e);
    }
}
Example 2
Source File: HBaseResourceStore.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
public HBaseResourceStore(KylinConfig kylinConfig) throws IOException {
    super(kylinConfig);

    String metadataUrl = kylinConfig.getMetadataUrl();
    // split TABLE@HBASE_URL
    int cut = metadataUrl.indexOf('@');
    tableNameBase = cut < 0 ? DEFAULT_TABLE_NAME : metadataUrl.substring(0, cut);
    hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);

    createHTableIfNeeded(getAllInOneTableName());

    //        tableNameMap = new LinkedHashMap<String, String>();
    //        for (Entry<String, String> entry : TABLE_SUFFIX_MAP.entrySet()) {
    //            String pathPrefix = entry.getKey();
    //            String tableName = tableNameBase + entry.getValue();
    //            tableNameMap.put(pathPrefix, tableName);
    //            createHTableIfNeeded(tableName);
    //        }

}
Example 3
Source File: ResourceStore.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private static ResourceStore createResourceStore(KylinConfig kylinConfig) {
    StorageURL metadataUrl = kylinConfig.getMetadataUrl();
    logger.info("Using metadata url {} for resource store", metadataUrl);
    String clsName = kylinConfig.getResourceStoreImpls().get(metadataUrl.getScheme());
    try {
        Class<? extends ResourceStore> cls = ClassUtil.forName(clsName, ResourceStore.class);
        ResourceStore store = cls.getConstructor(KylinConfig.class).newInstance(kylinConfig);
        if (!store.exists(METASTORE_UUID_TAG)) {
            store.checkAndPutResource(METASTORE_UUID_TAG, new StringEntity(store.createMetaStoreUUID()), 0,
                    StringEntity.serializer);
        }
        return store;
    } catch (Throwable e) {
        throw new IllegalArgumentException("Failed to find metadata store by url: " + metadataUrl, e);
    }
}
Example 4
Source File: ResourceStore.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
public static ResourceStore getStore(KylinConfig kylinConfig) {
    ResourceStore r = CACHE.get(kylinConfig);
    List<Throwable> es = new ArrayList<Throwable>();
    if (r == null) {
        logger.info("Using metadata url " + kylinConfig.getMetadataUrl() + " for resource store");
        for (Class<? extends ResourceStore> cls : knownImpl) {

            try {
                r = cls.getConstructor(KylinConfig.class).newInstance(kylinConfig);
            } catch (Exception e) {
                es.add(e);
            } catch (NoClassDefFoundError er) {
                // may throw NoClassDefFoundError
                es.add(er);
            }
            if (r != null) {
                break;
            }
        }
        if (r == null) {
            for (Throwable exceptionOrError : es) {
                logger.error("Create new store instance failed ", exceptionOrError);
            }
            throw new IllegalArgumentException("Failed to find metadata store by url: " + kylinConfig.getMetadataUrl());
        }

        CACHE.put(kylinConfig, r);
    }
    return r;
}
Example 5
Source File: FileResourceStore.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public FileResourceStore(KylinConfig kylinConfig) {
    super(kylinConfig);
    root = new File(getPath(kylinConfig)).getAbsoluteFile();
    if (!root.exists())
        throw new IllegalArgumentException(
                "File not exist by '" + kylinConfig.getMetadataUrl() + "': " + root.getAbsolutePath());
}
Example 6
Source File: JDBCConnectionManager.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private Map<String, String> initDbcpProps(KylinConfig config) {
    // metadataUrl is like "kylin_default_instance@jdbc,url=jdbc:mysql://localhost:3306/kylin,username=root,password=xxx"
    StorageURL metadataUrl = config.getMetadataUrl();
    JDBCResourceStore.checkScheme(metadataUrl);

    LinkedHashMap<String, String> ret = new LinkedHashMap<>(metadataUrl.getAllParameters());
    List<String> mandatoryItems = Arrays.asList("url", "username", PASSWORD);

    for (String item : mandatoryItems) {
        Preconditions.checkNotNull(ret.get(item),
                "Setting item \"" + item + "\" is mandatory for Jdbc connections.");
    }

    // Check whether password encrypted
    if ("true".equals(ret.get("passwordEncrypted"))) {
        String password = ret.get("password");
        ret.put("password", EncryptUtil.decrypt(password));
        ret.remove("passwordEncrypted");
    }

    logger.info("Connecting to Jdbc with url:{} by user {}", ret.get("url"), ret.get("username"));

    putIfMissing(ret, "driverClassName", "com.mysql.jdbc.Driver");
    putIfMissing(ret, "maxActive", "5");
    putIfMissing(ret, "maxIdle", "5");
    putIfMissing(ret, "maxWait", "1000");
    putIfMissing(ret, "removeAbandoned", "true");
    putIfMissing(ret, "removeAbandonedTimeout", "180");
    putIfMissing(ret, "testOnBorrow", "true");
    putIfMissing(ret, "testWhileIdle", "true");
    putIfMissing(ret, "validationQuery", "select 1");
    return ret;
}
Example 7
Source File: HBaseResourceStore.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private StorageURL buildMetadataUrl(KylinConfig kylinConfig) throws IOException {
    StorageURL url = kylinConfig.getMetadataUrl();
    if (!url.getScheme().equals("hbase"))
        throw new IOException("Cannot create HBaseResourceStore. Url not match. Url: " + url);

    // control timeout for prompt error report
    Map<String, String> newParams = new LinkedHashMap<>();
    newParams.put("hbase.client.scanner.timeout.period", kylinConfig.getHbaseClientScannerTimeoutPeriod());
    newParams.put("hbase.rpc.timeout", kylinConfig.getHbaseRpcTimeout());
    newParams.put("hbase.client.retries.number", kylinConfig.getHbaseClientRetriesNumber());
    newParams.putAll(url.getAllParameters());

    return url.copy(newParams);
}
Example 8
Source File: JDBCResourceStore.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public JDBCResourceStore(KylinConfig kylinConfig) throws SQLException, IOException {
    super(kylinConfig);
    StorageURL metadataUrl = kylinConfig.getMetadataUrl();
    checkScheme(metadataUrl);
    this.metadataIdentifier = metadataUrl.getIdentifier();
    this.tableNames[0] = metadataIdentifier;
    this.tableNames[1] = metadataIdentifier + "_log";
    this.connectionManager = JDBCConnectionManager.getConnectionManager();
    for (int i = 0; i < tableNames.length; i++) {
        createTableIfNeeded(tableNames[i]);
    }
}
Example 9
Source File: JDBCResourceStore.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public JDBCResourceStore(KylinConfig kylinConfig) throws SQLException, IOException {
    super(kylinConfig);
    StorageURL metadataUrl = kylinConfig.getMetadataUrl();
    checkScheme(metadataUrl);
    this.metadataIdentifier = metadataUrl.getIdentifier();
    this.tableNames[0] = metadataIdentifier;
    this.tableNames[1] = metadataIdentifier + "_log";
    this.connectionManager = JDBCConnectionManager.getConnectionManager();
    for (int i = 0; i < tableNames.length; i++) {
        createTableIfNeeded(tableNames[i]);
    }
}
Example 10
Source File: FileResourceStore.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public FileResourceStore(KylinConfig kylinConfig) {
    super(kylinConfig);
    root = new File(getPath(kylinConfig)).getAbsoluteFile();
    if (!root.exists())
        throw new IllegalArgumentException(
                "File not exist by '" + kylinConfig.getMetadataUrl() + "': " + root.getAbsolutePath());
}
Example 11
Source File: JDBCConnectionManager.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private Map<String, String> initDbcpProps(KylinConfig config) {
    // metadataUrl is like "kylin_default_instance@jdbc,url=jdbc:mysql://localhost:3306/kylin,username=root,password=xxx"
    StorageURL metadataUrl = config.getMetadataUrl();
    JDBCResourceStore.checkScheme(metadataUrl);

    LinkedHashMap<String, String> ret = new LinkedHashMap<>(metadataUrl.getAllParameters());
    List<String> mandatoryItems = Arrays.asList("url", "username", PASSWORD);

    for (String item : mandatoryItems) {
        Preconditions.checkNotNull(ret.get(item),
                "Setting item \"" + item + "\" is mandatory for Jdbc connections.");
    }

    // Check whether password encrypted
    if ("true".equals(ret.get("passwordEncrypted"))) {
        String password = ret.get("password");
        ret.put("password", EncryptUtil.decrypt(password));
        ret.remove("passwordEncrypted");
    }

    logger.info("Connecting to Jdbc with url:{} by user {}", ret.get("url"), ret.get("username"));

    putIfMissing(ret, "driverClassName", "com.mysql.jdbc.Driver");
    putIfMissing(ret, "maxActive", "5");
    putIfMissing(ret, "maxIdle", "5");
    putIfMissing(ret, "maxWait", "1000");
    putIfMissing(ret, "removeAbandoned", "true");
    putIfMissing(ret, "removeAbandonedTimeout", "180");
    putIfMissing(ret, "testOnBorrow", "true");
    putIfMissing(ret, "testWhileIdle", "true");
    putIfMissing(ret, "validationQuery", "select 1");
    return ret;
}
Example 12
Source File: HBaseResourceStore.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private StorageURL buildMetadataUrl(KylinConfig kylinConfig) throws IOException {
    StorageURL url = kylinConfig.getMetadataUrl();
    if (!url.getScheme().equals("hbase"))
        throw new IOException("Cannot create HBaseResourceStore. Url not match. Url: " + url);

    // control timeout for prompt error report
    Map<String, String> newParams = new LinkedHashMap<>();
    newParams.put("hbase.client.scanner.timeout.period", kylinConfig.getHbaseClientScannerTimeoutPeriod());
    newParams.put("hbase.rpc.timeout", kylinConfig.getHbaseRpcTimeout());
    newParams.put("hbase.client.retries.number", kylinConfig.getHbaseClientRetriesNumber());
    newParams.putAll(url.getAllParameters());

    return url.copy(newParams);
}
Example 13
Source File: HDFSResourceStore.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public HDFSResourceStore(KylinConfig kylinConfig) throws Exception {
    this(kylinConfig, kylinConfig.getMetadataUrl());
}
Example 14
Source File: FileResourceStore.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
public FileResourceStore(KylinConfig kylinConfig) {
    super(kylinConfig);
    root = new File(kylinConfig.getMetadataUrl()).getAbsoluteFile();
    if (root.exists() == false)
        throw new IllegalArgumentException("File not exist by '" + kylinConfig.getMetadataUrl() + "': " + root.getAbsolutePath());
}
Example 15
Source File: IdentifierFileResourceStore.java    From kylin with Apache License 2.0 4 votes vote down vote up 
protected String getPath(KylinConfig kylinConfig) {
    StorageURL metadataUrl = kylinConfig.getMetadataUrl();
    Preconditions.checkState(IFILE_SCHEME.equals(metadataUrl.getScheme()));
    return metadataUrl.getParameter("path");
}
Example 16
Source File: IdentifierFileResourceStore.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
protected String getPath(KylinConfig kylinConfig) {
    StorageURL metadataUrl = kylinConfig.getMetadataUrl();
    Preconditions.checkState(IFILE_SCHEME.equals(metadataUrl.getScheme()));
    return metadataUrl.getParameter("path");
}
Example 17
Source File: HDFSResourceStore.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public HDFSResourceStore(KylinConfig kylinConfig) throws Exception {
    this(kylinConfig, kylinConfig.getMetadataUrl());
}