Java Code Examples for org.apache.kylin.common.KylinConfig#setProperty()

public static void staticCreateTestMetadata(boolean useTestMeta, MetadataTestCaseHook hook) {
    try {
        KylinConfig.destroyInstance();

        FileUtils.deleteDirectory(new File(LOCALMETA_TEMP_DATA));
        if (useTestMeta) {
            FileUtils.copyDirectory(new File(LOCALMETA_TEST_DATA), new File(LOCALMETA_TEMP_DATA));
        }

        if (System.getProperty(KylinConfig.KYLIN_CONF) == null && System.getenv(KylinConfig.KYLIN_CONF) == null) {
            System.setProperty(KylinConfig.KYLIN_CONF, LOCALMETA_TEMP_DATA);
        }

        if (hook != null) {
            hook.hook();
        }
        KylinConfig config = KylinConfig.getInstanceFromEnv();
        config.setMetadataUrl(LOCALMETA_TEMP_DATA);
        config.setProperty("kylin.env.hdfs-working-dir", "file:///tmp/kylin");
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 

/**
 * generate the spark driver log hdfs path format, json path + timestamp + .log
 *
 * @param
 * @return
 */
/*public String getSparkDriverLogHdfsPath(KylinConfig config) {
    return String.format("%s.%s.log", config.getJobTmpOutputStorePath(getProject(), getId()),
            System.currentTimeMillis());
}*/

protected KylinConfig wrapConfig(ExecutableContext context) {
    KylinConfig originalConfig = context.getConfig();
    String project = getParam(MetadataConstants.P_PROJECT_NAME);
    Preconditions.checkState(StringUtils.isNotBlank(project), "job " + getId() + " project info is empty");

    String parentId = getParentId();
    originalConfig.setProperty("job.id", StringUtils.defaultIfBlank(parentId, getId()));
    originalConfig.setProperty("job.project", project);
    if (StringUtils.isNotBlank(parentId)) {
        originalConfig.setProperty("job.stepId", getId());
    }
    originalConfig.setProperty("user.timezone", KylinConfig.getInstanceFromEnv().getTimeZone());

    return originalConfig;
}
 

@Test
public void testSyntaxError() {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    config.setProperty("kylin.query.cache-enabled", "true");
    config.setProperty("kylin.query.lazy-query-enabled", "true");

    String badSql = "select with syntax error";

    SQLRequest request = new SQLRequest();
    request.setProject("default");
    request.setSql(badSql);

    try (SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(config)) {
        queryService.doQueryWithCache(request, false);
    } catch (Exception e) {
        // expected error
        Cache.ValueWrapper wrapper = cacheManager.getCache(QueryService.QUERY_CACHE).get(request.getCacheKey());
        Assert.assertTrue(wrapper == null || wrapper.get() == null);
    }
}
 

@Test
public void testCreateTableToWith() {
    String create_table1 = " create table tableId as select * from some_table1;";
    String create_table2 = "CREATE TABLE tableId2 AS select * FROM some_table2;";
    String select_table = "select * from tableId join tableId2 on tableId.a = tableId2.b;";

    KylinConfig config = KylinConfig.getInstanceFromEnv();
    config.setProperty("kylin.query.convert-create-table-to-with", "true");
    try (SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(config)) {

        SQLRequest request = new SQLRequest();
        request.setProject("default");
        request.setSql(create_table1);
        queryService.doQueryWithCache(request);

        request.setSql(create_table2);
        queryService.doQueryWithCache(request);

        request.setSql(select_table);
        SQLResponse response = queryService.doQueryWithCache(request, true);

        Assert.assertEquals(
                "WITH tableId as (select * from some_table1) , tableId2 AS (select * FROM some_table2) select * from tableId join tableId2 on tableId.a = tableId2.b;",
                response.getExceptionMessage());
    }
}
 

@Before
public void setUp() {
    this.createTestMetadata();
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    config.setProperty("kylin.storage.provider.2", MockQueryTimeoutStorage.class.getName());
    config.setProperty("kylin.storage.default", "2");
}
 

private void setupDebugEnv() {
    try {
        Properties props = new Properties();
        props.load(new FileInputStream("../build/conf/kylin-tools-log4j.properties"));
        PropertyConfigurator.configure(props);
        KylinConfig.setSandboxEnvIfPossible();
        KylinConfig config = KylinConfig.getInstanceFromEnv();
        config.setProperty("kylin.stream.settled.storage",
                "org.apache.kylin.stream.server.storage.LocalStreamStorage");
    } catch (Exception e) {
        logger.error("debug env setup fail", e);
    }
}
 

@Test
public void testSnapshotOverVolume() throws IOException {
    KylinConfig cubeConfig = KylinConfig.createKylinConfig(getTestConfig());
    cubeConfig.setProperty("kylin.snapshot.max-mb", "0");
    String tableName = "DEFAULT.TEST_KYLIN_FACT";
    TableDesc tableDesc = TableMetadataManager.getInstance(getTestConfig()).getTableDesc(tableName, "default");
    IReadableTable mockTable = new IReadableTable() {
        @Override
        public TableReader getReader() throws IOException {
            return new TableReader() {
                @Override
                public boolean next() throws IOException {
                    return false;
                }
                @Override
                public String[] getRow() {
                    return new String[0];
                }
                @Override
                public void close() throws IOException {
                }
            };
        }
        @Override
        public TableSignature getSignature() throws IOException {
            return new IReadableTable.TableSignature("", 2 * 1024 * 1024,
                    System.currentTimeMillis());
        }
        @Override
        public boolean exists() throws IOException {
            return false;
        }
    };
    SnapshotManager snapshotManager = getSnapshotManager();
    try {
        snapshotManager.buildSnapshot(mockTable, tableDesc, cubeConfig).getResourcePath();
    } catch (IllegalStateException ex) {
        Assert.assertTrue(ex.getLocalizedMessage().startsWith("Table snapshot should be no greater than 0 MB"));
    }
}
 

@Test
public void testFindFragmentsToMerge() {
    List<DataSegmentFragment> allFragments = Lists.newArrayList();
    KylinConfig config = getTestConfig();
    config.setProperty("kylin.stream.segment-min-fragments", "5");
    config.setProperty("kylin.stream.max-fragment-size-mb", "100");
    segmentStore.latestCheckpointFragment = 1000;
    List<DataSegmentFragment> result = segmentStore.chooseFragmentsToMerge(config, allFragments);
    assertTrue(result.isEmpty());
    for (int i = 0; i < 10; i++) {
        allFragments.add(new MockFragment(new FragmentId(i), 15));
    }
    result = segmentStore.chooseFragmentsToMerge(config, allFragments);
    assertEquals(6, result.size());

    allFragments.clear();
    for (int i = 0; i < 50; i++) {
        allFragments.add(new MockFragment(new FragmentId(i), 1));
    }
    result = segmentStore.chooseFragmentsToMerge(config, allFragments);
    assertEquals(46, result.size());

    allFragments.clear();
    allFragments.add(new MockFragment(new FragmentId(0), 100));
    for (int i = 1; i < 10; i++) {
        allFragments.add(new MockFragment(new FragmentId(i), 15));
    }
    result = segmentStore.chooseFragmentsToMerge(config, allFragments);
    assertEquals(6, result.size());

    allFragments.clear();
    allFragments.add(new MockFragment(new FragmentId(0, 5), 50));
    for (int i = 6; i < 20; i++) {
        allFragments.add(new MockFragment(new FragmentId(i), 15));
    }
    result = segmentStore.chooseFragmentsToMerge(config, allFragments);
    assertTrue(result.get(0).getFragmentId().equals(new FragmentId(6)));
    assertEquals(6, result.size());
}
 

public ExampleServer(String address) throws Exception {
    this.address = address;

    KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    KylinConfig kylinConfig1 = KylinConfig.createKylinConfig(kylinConfig);
    kylinConfig1.setProperty("kylin.server.host-address", address);

    CuratorFramework client = ZKUtil.newZookeeperClient(kylinConfig1);
    scheduler = new CuratorScheduler(client);
    scheduler.init(new JobEngineConfig(kylinConfig1), new MockJobLock());
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
}
 

@Test
public void testSnapshotOverVolume() throws IOException {
    KylinConfig cubeConfig = KylinConfig.createKylinConfig(getTestConfig());
    cubeConfig.setProperty("kylin.snapshot.max-mb", "0");
    String tableName = "DEFAULT.TEST_KYLIN_FACT";
    TableDesc tableDesc = TableMetadataManager.getInstance(getTestConfig()).getTableDesc(tableName, "default");
    IReadableTable mockTable = new IReadableTable() {
        @Override
        public TableReader getReader() throws IOException {
            return new TableReader() {
                @Override
                public boolean next() throws IOException {
                    return false;
                }
                @Override
                public String[] getRow() {
                    return new String[0];
                }
                @Override
                public void close() throws IOException {
                }
            };
        }
        @Override
        public TableSignature getSignature() throws IOException {
            return new IReadableTable.TableSignature("", 2 * 1024 * 1024,
                    System.currentTimeMillis());
        }
        @Override
        public boolean exists() throws IOException {
            return false;
        }
    };
    SnapshotManager snapshotManager = getSnapshotManager();
    try {
        snapshotManager.buildSnapshot(mockTable, tableDesc, cubeConfig).getResourcePath();
    } catch (IllegalStateException ex) {
        Assert.assertTrue(ex.getLocalizedMessage().startsWith("Table snapshot should be no greater than 0 MB"));
    }
}
 

@BeforeClass
public static void setupClass() throws SQLException {
    staticCreateTestMetadata();
    KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    kylinConfig.setProperty("kylin.source.jdbc.connection-url", "jdbc:vertica://fakehost:1433/database");
    kylinConfig.setProperty("kylin.source.jdbc.driver", "com.vertica.jdbc.Driver");
    kylinConfig.setProperty("kylin.source.jdbc.user", "user");
    kylinConfig.setProperty("kylin.source.jdbc.pass", "");
    kylinConfig.setProperty("kylin.source.jdbc.dialect", "vertica");
}
 

@Test
public void testAclDisabled() {
    KylinConfig testConfig = KylinConfig.getInstanceFromEnv();
    testConfig.setProperty("kylin.env.zookeeper-acl-enabled", "false");

    ZookeeperAclBuilder zookeeperAclBuilder = new ZookeeperAclBuilder().invoke();
    Assert.assertNotNull(zookeeperAclBuilder);
    Assert.assertFalse(zookeeperAclBuilder.isNeedAcl());

    Builder builder = zookeeperAclBuilder.setZKAclBuilder(CuratorFrameworkFactory.builder());
    Assert.assertNotNull(builder);
    Assert.assertEquals(ZooDefs.Ids.OPEN_ACL_UNSAFE, builder.getAclProvider().getDefaultAcl());
    Assert.assertNull(builder.getAuthInfos());
}
 

@Test
public void testGetConnection() throws SQLException, IllegalAccessException {
    Connection connection = QueryConnection.getConnection("default");
    assertNotNull(connection);
    Map<InternalProperty, Object> properties = dirtyReadProperties(connection);
    assertEquals(true, properties.get(InternalProperty.CASE_SENSITIVE));
    assertEquals(Casing.TO_UPPER, properties.get(InternalProperty.UNQUOTED_CASING));
    assertEquals(Quoting.DOUBLE_QUOTE, properties.get(InternalProperty.QUOTING));

    ResultSet resultSet = connection.prepareStatement(SQL_WITH_MIXED_CASE).executeQuery();
    String columnName = resultSet.getMetaData().getColumnName(1);
    assertEquals("VALUE_ALIAS", columnName);

    // override connection properties
    KylinConfig conf = KylinConfig.getInstanceFromEnv();
    conf.setProperty("kylin.query.calcite.extras-props.caseSensitive", "false");
    conf.setProperty("kylin.query.calcite.extras-props.unquotedCasing", "UNCHANGED");
    conf.setProperty("kylin.query.calcite.extras-props.quoting", "BRACKET");
    Connection conn2 = QueryConnection.getConnection("default");
    Map<InternalProperty, Object> props = dirtyReadProperties(conn2);
    assertEquals(false, props.get(InternalProperty.CASE_SENSITIVE));
    assertEquals(Casing.UNCHANGED, props.get(InternalProperty.UNQUOTED_CASING));
    assertEquals(Quoting.BRACKET, props.get(InternalProperty.QUOTING));

    ResultSet resultSet1 = conn2.prepareStatement(SQL_WITH_MIXED_CASE).executeQuery();
    assertEquals("value_ALIAS", resultSet1.getMetaData().getColumnName(1));
}
 

@BeforeClass
public static void setupClass() throws SQLException {
    staticCreateTestMetadata();
    KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    kylinConfig.setProperty("kylin.source.hive.quote-enabled", "true");
}
 

private void setupCubeConfig() {
    KylinConfig kylinConfig = cubeInstance.getConfig();
    kylinConfig.setProperty("kylin.stream.index.maxrows", "1000000");
}
 

private void setupCubeConfig() {
    KylinConfig kylinConfig = cubeInstance.getConfig();
    kylinConfig.setProperty("kylin.stream.index.maxrows", "1000000");
}
 

@BeforeClass
public static void setupClass() throws SQLException {
    staticCreateTestMetadata();
    KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    kylinConfig.setProperty("kylin.source.hive.quote-enabled", "true");
}
 

@Test
public void testAutoMergeWithMaxSegmentMergeSpan() throws Exception {
    KylinConfig kylinConfig = getTestConfig();
    kylinConfig.setProperty("kylin.cube.max-segment-merge.span", "4000");
    CubeManager mgr = CubeManager.getInstance(kylinConfig);
    CubeInstance cube = mgr.getCube("test_kylin_cube_with_slr_empty").latestCopyForWrite();

    cube.getDescriptor().setAutoMergeTimeRanges(new long[] { 2000 });
    mgr.updateCube(new CubeUpdate(cube).setStatus(RealizationStatusEnum.READY));
    assertEquals(RealizationStatusEnum.READY, cube.getStatus());
    assertTrue(cube.needAutoMerge());

    // no segment at first
    assertEquals(0, cube.getSegments().size());

    // append first
    CubeSegment seg1 = mgr.appendSegment(cube, new TSRange(0L, 1000L));
    mgr.updateCubeSegStatus(seg1, SegmentStatusEnum.READY);

    CubeSegment seg2 = mgr.appendSegment(cube, new TSRange(1000L, 5000L));
    mgr.updateCubeSegStatus(seg2, SegmentStatusEnum.READY);

    CubeSegment seg3 = mgr.appendSegment(cube, new TSRange(5000L, 6000L));
    mgr.updateCubeSegStatus(seg3, SegmentStatusEnum.READY);

    cube = mgr.getCube(cube.getName());
    assertEquals(3, cube.getSegments().size());

    SegmentRange mergedSeg = cube.autoMergeCubeSegments();

    assertTrue(mergedSeg == null);

    CubeSegment seg4 = mgr.appendSegment(cube, new TSRange(7000L, 8000L));
    mgr.updateCubeSegStatus(seg4, SegmentStatusEnum.READY);

    cube = mgr.getCube(cube.getName());
    assertEquals(4, cube.getSegments().size());
    mergedSeg = cube.autoMergeCubeSegments();
    assertTrue(mergedSeg == null);

    CubeSegment seg5 = mgr.appendSegment(cube, new TSRange(6000L, 7000L));
    mgr.updateCubeSegStatus(seg5, SegmentStatusEnum.READY);

    cube = mgr.getCube(cube.getName());
    assertEquals(5, cube.getSegments().size());

    mergedSeg = cube.autoMergeCubeSegments();
    assertTrue(mergedSeg != null);
    assertTrue((Long) mergedSeg.start.v == 5000 && (Long) mergedSeg.end.v == 7000);
}
 

@BeforeClass
public static void setup() throws Exception {
    staticCreateTestMetadata();

    new File(confDstPath1).getParentFile().mkdirs();
    new File(confDstPath2).getParentFile().mkdirs();
    KylinConfig srcConfig = KylinConfig.getInstanceFromEnv();

    localMetaDir = Files.createTempDir();
    backup = srcConfig.getMetadataUrl().toString();
    srcConfig.setProperty("kylin.metadata.url", localMetaDir.getAbsolutePath());
    srcConfig.exportToFile(new File(confDstPath1));
    srcConfig.exportToFile(new File(confDstPath2));

    kylinConfig1 = KylinConfig.createInstanceFromUri(new File(confDstPath1).getAbsolutePath());
    kylinConfig2 = KylinConfig.createInstanceFromUri(new File(confDstPath2).getAbsolutePath());

    initZk();
    
    ZookeeperDistributedLock.Factory factory = new ZookeeperDistributedLock.Factory(kylinConfig1);
    jobLock1 = (ZookeeperDistributedLock) factory.lockForClient(serverName1);
    jobLock2 = (ZookeeperDistributedLock) factory.lockForClient(serverName2);

    execMgr = ExecutableManager.getInstance(kylinConfig1);
    for (String jobId : execMgr.getAllJobIds()) {
        execMgr.deleteJob(jobId);
    }

    scheduler1 = DistributedScheduler.getInstance(kylinConfig1);
    scheduler1.init(new JobEngineConfig(kylinConfig1), jobLock1);
    if (!scheduler1.hasStarted()) {
        throw new RuntimeException("scheduler1 not started");
    }

    scheduler2 = DistributedScheduler.getInstance(kylinConfig2);
    scheduler2.init(new JobEngineConfig(kylinConfig2), jobLock2);
    if (!scheduler2.hasStarted()) {
        throw new RuntimeException("scheduler2 not started");
    }

    Thread.sleep(10000);
}
 

private void setupCubeConfig() {
    KylinConfig kylinConfig = cubeInstance.getConfig();
    kylinConfig.setProperty("kylin.stream.index.maxrows", "1000000");
}