Java Code Examples for org.apache.kylin.common.KylinConfig#createKylinConfig()
The following examples show how to use
org.apache.kylin.common.KylinConfig#createKylinConfig() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: JobRelatedMetaUtil.java From kylin with Apache License 2.0 | 6 votes |
|
public static void dumpAndUploadKylinPropsAndMetadata(Set<String> dumpList, KylinConfigExt kylinConfig, String metadataUrl)
throws IOException {
try (AutoDeleteDirectory tmpDir = new AutoDeleteDirectory("kylin_job_meta", "");
AutoDeleteDirectory metaDir = tmpDir.child("meta")) {
// dump metadata
JobRelatedMetaUtil.dumpResources(kylinConfig, metaDir.getFile(), dumpList);
// dump metadata
dumpResources(kylinConfig, metaDir.getFile(), dumpList);
// write kylin.properties
Properties props = kylinConfig.exportToProperties();
props.setProperty("kylin.metadata.url", metadataUrl);
File kylinPropsFile = new File(metaDir.getFile(), "kylin.properties");
try (FileOutputStream os = new FileOutputStream(kylinPropsFile)) {
props.store(os, kylinPropsFile.getAbsolutePath());
}
KylinConfig dstConfig = KylinConfig.createKylinConfig(props);
//upload metadata
new ResourceTool().copy(KylinConfig.createInstanceFromUri(metaDir.getAbsolutePath()), dstConfig);
}
}
Example 2
| Source File: JobRelatedMetaUtil.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
|
public static void dumpAndUploadKylinPropsAndMetadata(Set<String> dumpList, KylinConfigExt kylinConfig, String metadataUrl)
throws IOException {
try (AutoDeleteDirectory tmpDir = new AutoDeleteDirectory("kylin_job_meta", "");
AutoDeleteDirectory metaDir = tmpDir.child("meta")) {
// dump metadata
JobRelatedMetaUtil.dumpResources(kylinConfig, metaDir.getFile(), dumpList);
// dump metadata
dumpResources(kylinConfig, metaDir.getFile(), dumpList);
// write kylin.properties
Properties props = kylinConfig.exportToProperties();
props.setProperty("kylin.metadata.url", metadataUrl);
File kylinPropsFile = new File(metaDir.getFile(), "kylin.properties");
try (FileOutputStream os = new FileOutputStream(kylinPropsFile)) {
props.store(os, kylinPropsFile.getAbsolutePath());
}
KylinConfig dstConfig = KylinConfig.createKylinConfig(props);
//upload metadata
new ResourceTool().copy(KylinConfig.createInstanceFromUri(metaDir.getAbsolutePath()), dstConfig);
}
}
Example 3
| Source File: SCCreatorTest.java From kylin with Apache License 2.0 | 6 votes |
|
@Test
public void testExecute() throws Exception {
String metadataPath = tempMetadataDir.getPath();
String inputPath = "src/main/resources/SCSinkTools.json";
SCCreator cli = new SCCreator();
cli.execute("ADMIN", metadataPath, inputPath);
Assert.assertTrue(tempMetadataDir.isDirectory());
KylinConfig local = KylinConfig.createKylinConfig(KylinConfig.getInstanceFromEnv());
local.setMetadataUrl(metadataPath);
CubeManager cubeManager = CubeManager.getInstance(local);
List<CubeInstance> cubeList = cubeManager.listAllCubes();
System.out.println("System cubes: " + cubeList);
assertEquals(cubeList.size(), 10);
for (CubeInstance cube : cubeList) {
Assert.assertTrue(cube.getStatus() != RealizationStatusEnum.DESCBROKEN);
}
}
Example 4
| Source File: MetaDumpUtil.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
|
public static KylinConfig loadKylinConfigFromHdfs(String uri) {
if (uri == null)
throw new IllegalArgumentException("StorageUrl should not be null");
if (!uri.contains("@hdfs"))
throw new IllegalArgumentException("StorageUrl should like @hdfs schema");
logger.info("Ready to load KylinConfig from uri: {}", uri);
StorageURL url = StorageURL.valueOf(uri);
String metaDir = url.getParameter("path") + "/" + KylinConfig.KYLIN_CONF_PROPERTIES_FILE;
Path path = new Path(metaDir);
try (InputStream is = path.getFileSystem(HadoopUtil.getCurrentConfiguration()).open(new Path(metaDir))) {
Properties prop = KylinConfig.streamToProps(is);
return KylinConfig.createKylinConfig(prop);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
Example 5
| Source File: SCCreatorTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
|
@Test
public void testExecute() throws Exception {
String metadataPath = tempMetadataDir.getPath();
String inputPath = "src/main/resources/SCSinkTools.json";
SCCreator cli = new SCCreator();
cli.execute("ADMIN", metadataPath, inputPath);
Assert.assertTrue(tempMetadataDir.isDirectory());
KylinConfig local = KylinConfig.createKylinConfig(KylinConfig.getInstanceFromEnv());
local.setMetadataUrl(metadataPath);
CubeManager cubeManager = CubeManager.getInstance(local);
List<CubeInstance> cubeList = cubeManager.listAllCubes();
System.out.println("System cubes: " + cubeList);
assertEquals(cubeList.size(), 10);
for (CubeInstance cube : cubeList) {
Assert.assertTrue(cube.getStatus() != RealizationStatusEnum.DESCBROKEN);
}
}
Example 6
| Source File: AfterMergeOrRefreshResourceMerger.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
@Override
public void merge(AbstractExecutable abstractExecutable) {
String buildStepUrl = abstractExecutable.getParam(MetadataConstants.P_OUTPUT_META_URL);
KylinConfig buildConfig = KylinConfig.createKylinConfig(this.getConfig());
buildConfig.setMetadataUrl(buildStepUrl);
ResourceStore resourceStore = ResourceStore.getStore(buildConfig);
String cubeId = abstractExecutable.getParam(MetadataConstants.P_CUBE_ID);
String segmentId = abstractExecutable.getParam(CubingExecutableUtil.SEGMENT_ID);
merge(cubeId, segmentId, resourceStore, abstractExecutable.getParam(MetadataConstants.P_JOB_TYPE));
}
Example 7
| Source File: LookupTableTest.java From kylin with Apache License 2.0 | 5 votes |
|
@Test
public void testSnapshotOverVolume() throws IOException {
KylinConfig cubeConfig = KylinConfig.createKylinConfig(getTestConfig());
cubeConfig.setProperty("kylin.snapshot.max-mb", "0");
String tableName = "DEFAULT.TEST_KYLIN_FACT";
TableDesc tableDesc = TableMetadataManager.getInstance(getTestConfig()).getTableDesc(tableName, "default");
IReadableTable mockTable = new IReadableTable() {
@Override
public TableReader getReader() throws IOException {
return new TableReader() {
@Override
public boolean next() throws IOException {
return false;
}
@Override
public String[] getRow() {
return new String[0];
}
@Override
public void close() throws IOException {
}
};
}
@Override
public TableSignature getSignature() throws IOException {
return new IReadableTable.TableSignature("", 2 * 1024 * 1024,
System.currentTimeMillis());
}
@Override
public boolean exists() throws IOException {
return false;
}
};
SnapshotManager snapshotManager = getSnapshotManager();
try {
snapshotManager.buildSnapshot(mockTable, tableDesc, cubeConfig).getResourcePath();
} catch (IllegalStateException ex) {
Assert.assertTrue(ex.getLocalizedMessage().startsWith("Table snapshot should be no greater than 0 MB"));
}
}
Example 8
| Source File: ITJDBCResourceStoreTest.java From kylin with Apache License 2.0 | 5 votes |
|
@Before
public void setup() throws Exception {
this.createTestMetadata();
kylinConfig = KylinConfig.getInstanceFromEnv();
KylinConfig configBackup = KylinConfig.createKylinConfig(kylinConfig);
Statement statement = null;
Connection conn = null;
metadataUrlBackup = kylinConfig.getMetadataUrl();
kylinConfig.setMetadataUrl(mainIdentifier + jdbcMetadataUrlNoIdentifier);
JDBCSqlQueryFormat sqlQueryFormat = JDBCSqlQueryFormatProvider
.createJDBCSqlQueriesFormat(KylinConfig.getInstanceFromEnv().getMetadataDialect());
try {
connectionManager = JDBCConnectionManager.getConnectionManager();
conn = connectionManager.getConn();
statement = conn.createStatement();
String sql = new MessageFormat(sqlQueryFormat.getTestDropSql(), Locale.ROOT)
.format(mainIdentifier, new StringBuffer(), new FieldPosition(0)).toString();
statement.executeUpdate(sql);
sql = new MessageFormat(sqlQueryFormat.getTestDropSql(), Locale.ROOT)
.format(copyIdentifier, new StringBuffer(), new FieldPosition(0)).toString();
statement.executeUpdate(sql);
jdbcConnectable = true;
new ResourceTool().copy(configBackup, kylinConfig);
} catch (RuntimeException ex) {
logger.info("Init connection manager failed, skip test cases");
} finally {
JDBCConnectionManager.closeQuietly(statement);
JDBCConnectionManager.closeQuietly(conn);
}
}
Example 9
| Source File: ExampleServer.java From kylin with Apache License 2.0 | 5 votes |
|
public ExampleServer(String address) {
this.address = address;
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
KylinConfig kylinConfig1 = KylinConfig.createKylinConfig(kylinConfig);
kylinConfig1.setProperty("kylin.server.host-address", address);
discovery = new KylinServerDiscovery(kylinConfig1);
}
Example 10
| Source File: AbstractHadoopJob.java From kylin with Apache License 2.0 | 5 votes |
|
public static KylinConfig loadKylinConfigFromHdfs(String uri) {
if (uri == null)
throw new IllegalArgumentException("meta url should not be null");
if (!uri.contains("@hdfs"))
throw new IllegalArgumentException("meta url should like @hdfs schema");
if (kylinConfigCache.get(uri) != null) {
logger.info("KylinConfig cached for : {}", uri);
return kylinConfigCache.get(uri);
}
logger.info("Ready to load KylinConfig from uri: {}", uri);
KylinConfig config;
FileSystem fs;
String realHdfsPath = StorageURL.valueOf(uri).getParameter("path") + "/" + KylinConfig.KYLIN_CONF_PROPERTIES_FILE;
try {
fs = HadoopUtil.getFileSystem(realHdfsPath);
InputStream is = fs.open(new Path(realHdfsPath));
Properties prop = KylinConfig.streamToProps(is);
config = KylinConfig.createKylinConfig(prop);
} catch (IOException e) {
throw new RuntimeException(e);
}
kylinConfigCache.put(uri, config);
return config;
}
Example 11
| Source File: NSparkUpdateMetaAndCleanupAfterMergeStep.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
private void updateMetadataAfterMerge(String cubeId) {
String buildStepUrl = getParam(MetadataConstants.P_OUTPUT_META_URL);
KylinConfig buildConfig = KylinConfig.createKylinConfig(this.getConfig());
buildConfig.setMetadataUrl(buildStepUrl);
ResourceStore resourceStore = ResourceStore.getStore(buildConfig);
String mergedSegmentId = getParam(CubingExecutableUtil.SEGMENT_ID);
AfterMergeOrRefreshResourceMerger merger = new AfterMergeOrRefreshResourceMerger(buildConfig);
merger.merge(cubeId, mergedSegmentId, resourceStore, getParam(MetadataConstants.P_JOB_TYPE));
}
Example 12
| Source File: AbstractHadoopJob.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
public static KylinConfig loadKylinConfigFromHdfs(String uri) {
if (uri == null)
throw new IllegalArgumentException("meta url should not be null");
if (!uri.contains("@hdfs"))
throw new IllegalArgumentException("meta url should like @hdfs schema");
if (kylinConfigCache.get(uri) != null) {
logger.info("KylinConfig cached for : {}", uri);
return kylinConfigCache.get(uri);
}
logger.info("Ready to load KylinConfig from uri: {}", uri);
KylinConfig config;
FileSystem fs;
String realHdfsPath = StorageURL.valueOf(uri).getParameter("path") + "/" + KylinConfig.KYLIN_CONF_PROPERTIES_FILE;
try {
fs = HadoopUtil.getFileSystem(realHdfsPath);
InputStream is = fs.open(new Path(realHdfsPath));
Properties prop = KylinConfig.streamToProps(is);
config = KylinConfig.createKylinConfig(prop);
} catch (IOException e) {
throw new RuntimeException(e);
}
kylinConfigCache.put(uri, config);
return config;
}
Example 13
| Source File: LookupTableTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
@Test
public void testSnapshotOverVolume() throws IOException {
KylinConfig cubeConfig = KylinConfig.createKylinConfig(getTestConfig());
cubeConfig.setProperty("kylin.snapshot.max-mb", "0");
String tableName = "DEFAULT.TEST_KYLIN_FACT";
TableDesc tableDesc = TableMetadataManager.getInstance(getTestConfig()).getTableDesc(tableName, "default");
IReadableTable mockTable = new IReadableTable() {
@Override
public TableReader getReader() throws IOException {
return new TableReader() {
@Override
public boolean next() throws IOException {
return false;
}
@Override
public String[] getRow() {
return new String[0];
}
@Override
public void close() throws IOException {
}
};
}
@Override
public TableSignature getSignature() throws IOException {
return new IReadableTable.TableSignature("", 2 * 1024 * 1024,
System.currentTimeMillis());
}
@Override
public boolean exists() throws IOException {
return false;
}
};
SnapshotManager snapshotManager = getSnapshotManager();
try {
snapshotManager.buildSnapshot(mockTable, tableDesc, cubeConfig).getResourcePath();
} catch (IllegalStateException ex) {
Assert.assertTrue(ex.getLocalizedMessage().startsWith("Table snapshot should be no greater than 0 MB"));
}
}
Example 14
| Source File: ITJDBCResourceStoreTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
@Before
public void setup() throws Exception {
this.createTestMetadata();
kylinConfig = KylinConfig.getInstanceFromEnv();
KylinConfig configBackup = KylinConfig.createKylinConfig(kylinConfig);
Statement statement = null;
Connection conn = null;
metadataUrlBackup = kylinConfig.getMetadataUrl();
kylinConfig.setMetadataUrl(mainIdentifier + jdbcMetadataUrlNoIdentifier);
JDBCSqlQueryFormat sqlQueryFormat = JDBCSqlQueryFormatProvider
.createJDBCSqlQueriesFormat(KylinConfig.getInstanceFromEnv().getMetadataDialect());
try {
connectionManager = JDBCConnectionManager.getConnectionManager();
conn = connectionManager.getConn();
statement = conn.createStatement();
String sql = new MessageFormat(sqlQueryFormat.getTestDropSql(), Locale.ROOT)
.format(mainIdentifier, new StringBuffer(), new FieldPosition(0)).toString();
statement.executeUpdate(sql);
sql = new MessageFormat(sqlQueryFormat.getTestDropSql(), Locale.ROOT)
.format(copyIdentifier, new StringBuffer(), new FieldPosition(0)).toString();
statement.executeUpdate(sql);
jdbcConnectable = true;
new ResourceTool().copy(configBackup, kylinConfig);
} catch (RuntimeException ex) {
logger.info("Init connection manager failed, skip test cases");
} finally {
JDBCConnectionManager.closeQuietly(statement);
JDBCConnectionManager.closeQuietly(conn);
}
}
Example 15
| Source File: SignatureCalculatorTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
|
@Test
public void testFactTableRealizationSetCalculator() throws IOException {
KylinConfig config = KylinConfig.createKylinConfig(getTestConfig());
Map<String, String> overrides = Maps.newHashMap();
overrides.put("kylin.query.signature-class",
"org.apache.kylin.rest.signature.FactTableRealizationSetCalculator");
ProjectInstance projectInstance = ProjectManager.getInstance(config).getProject(projectName);
projectInstance.setConfig(KylinConfigExt.createInstance(config, overrides));
HybridManager hybridManager = HybridManager.getInstance(config);
HybridInstance hybrid1 = hybridManager.getHybridInstance("test_kylin_hybrid_ready");
CubeManager cubeManager = CubeManager.getInstance(config);
CubeInstance cube1 = cubeManager.getCube("test_kylin_cube_with_slr_ready_2_segments");
CubeInstance cube2 = cubeManager.getCube("test_kylin_cube_without_slr_ready");
CubeInstance cube2Clone = cloneCubeInstance(cubeManager, cube2, cube2.getName() + "_clone");
CubeInstance cube3 = cloneCubeInstance(cubeManager, cube2, cube2.getDescName());
//Related cubes:
// - test_kylin_cube_with_slr_ready
// - test_kylin_cube_with_slr_ready_2_segments
// - test_kylin_cube_without_slr_ready
String cubes = hybrid1.getCanonicalName() + "," + cube2Clone.getCanonicalName();
SQLResponse sqlResponse = new SQLResponse();
sqlResponse.setCube(cubes);
String signature = SQLResponseSignatureUtil.createSignature(config, sqlResponse, projectName);
sqlResponse.setSignature(signature);
Assert.assertTrue(SQLResponseSignatureUtil.checkSignature(config, sqlResponse, projectName));
{//Test the influence of related cubes status change
cube1 = cubeManager.updateCubeStatus(cube1, RealizationStatusEnum.DISABLED);
Assert.assertFalse(SQLResponseSignatureUtil.checkSignature(config, sqlResponse, projectName));
cube1 = cubeManager.updateCubeStatus(cube1, RealizationStatusEnum.READY);
Assert.assertTrue(SQLResponseSignatureUtil.checkSignature(config, sqlResponse, projectName));
}
{//Test the influence of cubes not in ${cubes} while share the same fact tables
cube3 = cubeManager.updateCubeStatus(cube3, RealizationStatusEnum.DISABLED);
Assert.assertFalse(SQLResponseSignatureUtil.checkSignature(config, sqlResponse, projectName));
cube3 = cubeManager.updateCubeStatus(cube3, RealizationStatusEnum.READY);
Assert.assertTrue(SQLResponseSignatureUtil.checkSignature(config, sqlResponse, projectName));
}
{//Test the influence of segment changes
cube2Clone = cubeManager.updateCubeDropSegments(cube2Clone, cube2Clone.getSegments().get(0));
Assert.assertFalse(SQLResponseSignatureUtil.checkSignature(config, sqlResponse, projectName));
}
}
Example 16
| Source File: SignatureCalculatorTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
|
@Test
public void testRealizationSetCalculator() throws IOException {
KylinConfig config = KylinConfig.createKylinConfig(getTestConfig());
Map<String, String> overrides = Maps.newHashMap();
overrides.put("kylin.query.signature-class", "org.apache.kylin.rest.signature.RealizationSetCalculator");
ProjectInstance projectInstance = ProjectManager.getInstance(config).getProject(projectName);
projectInstance.setConfig(KylinConfigExt.createInstance(config, overrides));
HybridManager hybridManager = HybridManager.getInstance(config);
HybridInstance hybrid1 = hybridManager.getHybridInstance("test_kylin_hybrid_ready");
CubeManager cubeManager = CubeManager.getInstance(config);
CubeInstance cube1 = cubeManager.getCube("test_kylin_cube_with_slr_ready_2_segments");
CubeInstance cube2 = cubeManager.getCube("test_kylin_cube_without_slr_ready");
CubeInstance cube2Clone = cloneCubeInstance(cubeManager, cube2, cube2.getName() + "_clone");
//Related cubes:
// - test_kylin_cube_with_slr_ready
// - test_kylin_cube_with_slr_ready_2_segments
// - test_kylin_cube_without_slr_ready
String cubes = hybrid1.getCanonicalName() + "," + cube2Clone.getCanonicalName();
SQLResponse sqlResponse = new SQLResponse();
sqlResponse.setCube(cubes);
String signature = SQLResponseSignatureUtil.createSignature(config, sqlResponse, projectName);
sqlResponse.setSignature(signature);
Assert.assertTrue(SQLResponseSignatureUtil.checkSignature(config, sqlResponse, projectName));
{//Test the influence of related cubes status change
cube1 = cubeManager.updateCubeStatus(cube1, RealizationStatusEnum.DISABLED);
Assert.assertFalse(SQLResponseSignatureUtil.checkSignature(config, sqlResponse, projectName));
cube1 = cubeManager.updateCubeStatus(cube1, RealizationStatusEnum.READY);
Assert.assertTrue(SQLResponseSignatureUtil.checkSignature(config, sqlResponse, projectName));
}
{//Test the influence of segment changes
cube2Clone = cubeManager.updateCubeDropSegments(cube2Clone, cube2Clone.getSegments().get(0));
Assert.assertFalse(SQLResponseSignatureUtil.checkSignature(config, sqlResponse, projectName));
}
}
Example 17
| Source File: MultiNodeManagerTestBase.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
|
@Before
public void setup() throws Exception {
staticCreateTestMetadata();
System.clearProperty("kylin.server.cluster-servers");
int port = CheckUtil.randomAvailablePort(40000, 50000);
logger.info("Chosen port for CacheServiceTest is " + port);
configA = KylinConfig.getInstanceFromEnv();
configA.setProperty("kylin.server.cluster-servers", "localhost:" + port);
configB = KylinConfig.createKylinConfig(configA);
configB.setProperty("kylin.server.cluster-servers", "localhost:" + port);
configB.setMetadataUrl("../examples/test_metadata");
server = new Server(port);
ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);
context.setContextPath("/");
server.setHandler(context);
final CacheService serviceA = new CacheService() {
@Override
public KylinConfig getConfig() {
return configA;
}
};
final CacheService serviceB = new CacheService() {
@Override
public KylinConfig getConfig() {
return configB;
}
};
context.addServlet(new ServletHolder(new BroadcasterReceiveServlet(new BroadcasterReceiveServlet.BroadcasterHandler() {
@Override
public void handle(String entity, String cacheKey, String event) {
Broadcaster.Event wipeEvent = Broadcaster.Event.getEvent(event);
final String log = "wipe cache type: " + entity + " event:" + wipeEvent + " name:" + cacheKey;
logger.info(log);
try {
serviceA.notifyMetadataChange(entity, wipeEvent, cacheKey);
serviceB.notifyMetadataChange(entity, wipeEvent, cacheKey);
} catch (Exception e) {
e.printStackTrace();
}
}
})), "/");
server.start();
}
Example 18
| Source File: SignatureCalculatorTest.java From kylin with Apache License 2.0 | 4 votes |
|
@Test
public void testFactTableRealizationSetCalculator() throws IOException {
KylinConfig config = KylinConfig.createKylinConfig(getTestConfig());
Map<String, String> overrides = Maps.newHashMap();
overrides.put("kylin.query.signature-class",
"org.apache.kylin.rest.signature.FactTableRealizationSetCalculator");
ProjectInstance projectInstance = ProjectManager.getInstance(config).getProject(projectName);
projectInstance.setConfig(KylinConfigExt.createInstance(config, overrides));
HybridManager hybridManager = HybridManager.getInstance(config);
HybridInstance hybrid1 = hybridManager.getHybridInstance("test_kylin_hybrid_ready");
CubeManager cubeManager = CubeManager.getInstance(config);
CubeInstance cube1 = cubeManager.getCube("test_kylin_cube_with_slr_ready_2_segments");
CubeInstance cube2 = cubeManager.getCube("test_kylin_cube_without_slr_ready");
CubeInstance cube2Clone = cloneCubeInstance(cubeManager, cube2, cube2.getName() + "_clone");
CubeInstance cube3 = cloneCubeInstance(cubeManager, cube2, cube2.getDescName());
//Related cubes:
// - test_kylin_cube_with_slr_ready
// - test_kylin_cube_with_slr_ready_2_segments
// - test_kylin_cube_without_slr_ready
String cubes = hybrid1.getCanonicalName() + "," + cube2Clone.getCanonicalName();
SQLResponse sqlResponse = new SQLResponse();
sqlResponse.setCube(cubes);
String signature = SQLResponseSignatureUtil.createSignature(config, sqlResponse, projectName);
sqlResponse.setSignature(signature);
Assert.assertTrue(SQLResponseSignatureUtil.checkSignature(config, sqlResponse, projectName));
{//Test the influence of related cubes status change
cube1 = cubeManager.updateCubeStatus(cube1, RealizationStatusEnum.DISABLED);
Assert.assertFalse(SQLResponseSignatureUtil.checkSignature(config, sqlResponse, projectName));
cube1 = cubeManager.updateCubeStatus(cube1, RealizationStatusEnum.READY);
Assert.assertTrue(SQLResponseSignatureUtil.checkSignature(config, sqlResponse, projectName));
}
{//Test the influence of cubes not in ${cubes} while share the same fact tables
cube3 = cubeManager.updateCubeStatus(cube3, RealizationStatusEnum.DISABLED);
Assert.assertFalse(SQLResponseSignatureUtil.checkSignature(config, sqlResponse, projectName));
cube3 = cubeManager.updateCubeStatus(cube3, RealizationStatusEnum.READY);
Assert.assertTrue(SQLResponseSignatureUtil.checkSignature(config, sqlResponse, projectName));
}
{//Test the influence of segment changes
cube2Clone = cubeManager.updateCubeDropSegments(cube2Clone, cube2Clone.getSegments().get(0));
Assert.assertFalse(SQLResponseSignatureUtil.checkSignature(config, sqlResponse, projectName));
}
}
Example 19
| Source File: ITJDBCResourceStoreTest.java From kylin with Apache License 2.0 | 4 votes |
|
@Test
public void testPerformanceWithResourceTool() throws Exception {
Assume.assumeTrue(jdbcConnectable);
KylinConfig tmpConfig = KylinConfig.createKylinConfig(KylinConfig.getInstanceFromEnv());
tmpConfig.setMetadataUrl(copyIdentifier + jdbcMetadataUrlNoIdentifier);
JDBCResourceStore store = (JDBCResourceStore) ResourceStore.getStore(kylinConfig);
NavigableSet<String> executes = store.listResources(ResourceStore.EXECUTE_RESOURCE_ROOT);
NavigableSet<String> executeOutputs = store.listResources(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT);
long startTs = System.currentTimeMillis();
for (String execute : executes) {
String uuid = StringUtils.substringAfterLast(execute, "/");
RawResource executeResource = store.getResource(execute);
Map<String, RawResource> executeOutputResourceMap = new HashMap<>();
for (String executeOutput : executeOutputs) {
if (executeOutput.contains(uuid)) {
RawResource executeOutputResource = store.getResource(executeOutput);
executeOutputResourceMap.put(executeOutput, executeOutputResource);
}
}
for (int i = 0; i < 200; i++) {
String newUuid = UUID.randomUUID().toString();
store.putResource(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + newUuid, executeResource.content(),
System.currentTimeMillis());
for (String key : executeOutputResourceMap.keySet()) {
String step = StringUtils.substringAfterLast(key, uuid);
store.putResource(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + newUuid + step,
executeOutputResourceMap.get(key).content(), System.currentTimeMillis());
}
}
}
long queryNumBeforeCopy = store.getQueriedSqlNum();
new ResourceTool().copy(kylinConfig, tmpConfig);
long endTs = System.currentTimeMillis();
long queryNumAfterCopy = store.getQueriedSqlNum();
JDBCResourceStore resourceStoreCopy = (JDBCResourceStore) ResourceStore.getStore(tmpConfig);
int executeNum = store.listResources("/execute").size();
int executeOutputNum = store.listResources("/execute_output").size();
assertEquals(executeNum, resourceStoreCopy.listResources("/execute").size());
assertEquals(executeOutputNum, resourceStoreCopy.listResources("/execute_output").size());
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT);
String startTime = sdf.format(new Date(Long.parseLong(String.valueOf(startTs))));
String endTime = sdf.format(new Date(Long.parseLong(String.valueOf(endTs))));
logger.info("Test performance with ResourceTool done during " + startTime + " to " + endTime);
logger.info("Now there is " + executeNum + " execute data and " + executeOutputNum
+ " execute_output data in resource store.");
logger.info("Resource store run " + queryNumBeforeCopy + " sqls for metadata generation, and "
+ (queryNumAfterCopy - queryNumBeforeCopy) + " sqls for copy with ResourceTool.");
assertTrue((queryNumAfterCopy - queryNumBeforeCopy) < queryNumBeforeCopy);
logger.info("This test is expected to be done in 10 mins.");
assertTrue((endTs - startTs) < 600000);
}
Example 20
| Source File: ITJDBCResourceStoreTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
|
@Test
public void testPerformanceWithResourceTool() throws Exception {
Assume.assumeTrue(jdbcConnectable);
KylinConfig tmpConfig = KylinConfig.createKylinConfig(KylinConfig.getInstanceFromEnv());
tmpConfig.setMetadataUrl(copyIdentifier + jdbcMetadataUrlNoIdentifier);
JDBCResourceStore store = (JDBCResourceStore) ResourceStore.getStore(kylinConfig);
NavigableSet<String> executes = store.listResources(ResourceStore.EXECUTE_RESOURCE_ROOT);
NavigableSet<String> executeOutputs = store.listResources(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT);
long startTs = System.currentTimeMillis();
for (String execute : executes) {
String uuid = StringUtils.substringAfterLast(execute, "/");
RawResource executeResource = store.getResource(execute);
Map<String, RawResource> executeOutputResourceMap = new HashMap<>();
for (String executeOutput : executeOutputs) {
if (executeOutput.contains(uuid)) {
RawResource executeOutputResource = store.getResource(executeOutput);
executeOutputResourceMap.put(executeOutput, executeOutputResource);
}
}
for (int i = 0; i < 200; i++) {
String newUuid = UUID.randomUUID().toString();
store.putResource(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + newUuid, executeResource.content(),
System.currentTimeMillis());
for (String key : executeOutputResourceMap.keySet()) {
String step = StringUtils.substringAfterLast(key, uuid);
store.putResource(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + newUuid + step,
executeOutputResourceMap.get(key).content(), System.currentTimeMillis());
}
}
}
long queryNumBeforeCopy = store.getQueriedSqlNum();
new ResourceTool().copy(kylinConfig, tmpConfig);
long endTs = System.currentTimeMillis();
long queryNumAfterCopy = store.getQueriedSqlNum();
JDBCResourceStore resourceStoreCopy = (JDBCResourceStore) ResourceStore.getStore(tmpConfig);
int executeNum = store.listResources("/execute").size();
int executeOutputNum = store.listResources("/execute_output").size();
assertEquals(executeNum, resourceStoreCopy.listResources("/execute").size());
assertEquals(executeOutputNum, resourceStoreCopy.listResources("/execute_output").size());
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT);
String startTime = sdf.format(new Date(Long.parseLong(String.valueOf(startTs))));
String endTime = sdf.format(new Date(Long.parseLong(String.valueOf(endTs))));
logger.info("Test performance with ResourceTool done during " + startTime + " to " + endTime);
logger.info("Now there is " + executeNum + " execute data and " + executeOutputNum
+ " execute_output data in resource store.");
logger.info("Resource store run " + queryNumBeforeCopy + " sqls for metadata generation, and "
+ (queryNumAfterCopy - queryNumBeforeCopy) + " sqls for copy with ResourceTool.");
assertTrue((queryNumAfterCopy - queryNumBeforeCopy) < queryNumBeforeCopy);
logger.info("This test is expected to be done in 10 mins.");
assertTrue((endTs - startTs) < 600000);
}
