org.apache.kylin.metadata.model.TableDesc Java Examples

public AbstractLookupRowEncoder(TableDesc tableDesc, String[] keyColumns) {
    this.columnsNum = tableDesc.getColumns().length;
    this.keyIndexes = new int[keyColumns.length];
    this.valueIndexes = new int[columnsNum - keyColumns.length];
    int keyIdx = 0;
    int valIdx = 0;
    for (int i = 0; i < columnsNum; i++) {
        boolean isKeyColumn = false;
        for (String keyColumn : keyColumns) {
            if (keyColumn.equals(tableDesc.getColumns()[i].getName())) {
                isKeyColumn = true;
                break;
            }
        }
        if (isKeyColumn) {
            keyIndexes[keyIdx] = i;
            keyIdx++;
        } else {
            valueIndexes[valIdx] = i;
            valIdx++;
        }
    }
}
 

private String[] generateCreateTableSql(TableDesc tableDesc) {

        String dropsql = "DROP TABLE IF EXISTS " + tableDesc.getIdentity();
        String dropsql2 = "DROP VIEW IF EXISTS " + tableDesc.getIdentity();

        StringBuilder ddl = new StringBuilder();
        ddl.append("CREATE TABLE " + tableDesc.getIdentity() + "\n");
        ddl.append("(" + "\n");

        for (int i = 0; i < tableDesc.getColumns().length; i++) {
            ColumnDesc col = tableDesc.getColumns()[i];
            if (i > 0) {
                ddl.append(",");
            }
            ddl.append(col.getName() + " " + getHiveDataType((col.getDatatype())) + "\n");
        }

        ddl.append(")" + "\n");
        ddl.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY ','" + "\n");
        ddl.append("STORED AS TEXTFILE");

        return new String[] { dropsql, dropsql2, ddl.toString() };
    }
 

public static TableDesc dealWithMappingForTable(TableDesc other, Map<String, TableSchemaUpdateMapping> mappings) {
    TableSchemaUpdateMapping mapping = getTableSchemaUpdateMapping(mappings, other.getIdentity());
    if (mapping == null) {
        return other;
    }

    TableDesc copy = new TableDesc(other);

    copy.setDatabase(mapping.getDatabase(other.getDatabase()));

    copy.setName(mapping.getTableName(other.getName()));

    // It will always be a new one
    copy.setLastModified(0L);

    return copy;
}
 

public static Set<String> collectCubeMetadata(CubeInstance cube) {
    // cube, model_desc, cube_desc, table
    Set<String> dumpList = new LinkedHashSet<>();
    dumpList.add(cube.getResourcePath());
    dumpList.add(cube.getDescriptor().getModel().getResourcePath());
    dumpList.add(cube.getDescriptor().getResourcePath());
    dumpList.add(cube.getProjectInstance().getResourcePath());

    for (TableRef tableRef : cube.getDescriptor().getModel().getAllTables()) {
        TableDesc table = tableRef.getTableDesc();
        dumpList.add(table.getResourcePath());
        dumpList.addAll(SourceManager.getMRDependentResources(table));
    }

    return dumpList;
}
 

private Map<String, Table> buildTableMap() {
    Map<String, Table> olapTables = new HashMap<String, Table>();

    Collection<TableDesc> projectTables = ProjectManager.getInstance(config).listExposedTables(projectName,
            exposeMore);

    for (TableDesc tableDesc : projectTables) {
        if (tableDesc.getDatabase().equals(schemaName)) {
            final String tableName = tableDesc.getName();//safe to use tableDesc.getName() here, it is in a DB context now
            final OLAPTable table = new OLAPTable(this, tableDesc, exposeMore);
            olapTables.put(tableName, table);
            //logger.debug("Project " + projectName + " exposes table " + tableName);
        }
    }

    return olapTables;
}
 

@Test
public void verifyConvertFilterConstants1() {
    GTInfo info = table.getInfo();

    TableDesc extTable = TableDesc.mockup("ext");
    TblColRef extColA = TblColRef.mockup(extTable, 1, "A", "timestamp");
    TblColRef extColB = TblColRef.mockup(extTable, 2, "B", "integer");

    CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14");
    CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.EQ, "10");
    LogicalTupleFilter filter = and(fComp1, fComp2);

    List<TblColRef> colMapping = Lists.newArrayList();
    colMapping.add(extColA);
    colMapping.add(extColB);

    TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
    assertEquals(
            "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 EQ [\\x00]]",
            newFilter.toString());
}
 

/**
 * Check the schema compatibility for table, model desc
 */
@RequestMapping(value = "/checkCompatibility", method = { RequestMethod.POST })
@ResponseBody
public void checkCompatibility(@RequestBody CompatibilityCheckRequest request) {
    try {
        List<TableDesc> tableDescList = deserializeTableDescList(request);
        for (TableDesc tableDesc : tableDescList) {
            logger.info("Schema compatibility check for table {}", tableDesc.getName());
            tableService.checkTableCompatibility(request.getProjectName(), tableDesc);
            logger.info("Pass schema compatibility check for table {}", tableDesc.getName());
        }
        DataModelDesc dataModelDesc = JsonUtil.readValue(request.getModelDescData(), DataModelDesc.class);
        logger.info("Schema compatibility check for model {}", dataModelDesc.getName());
        modelService.checkModelCompatibility(dataModelDesc, tableDescList);
        logger.info("Pass schema compatibility check for model {}", dataModelDesc.getName());
    } catch (Exception e) {
        logger.error(e.getMessage(), e);
        throw new ConflictException(e.getMessage(), e);
    }
}
 

@Test
public void testLoadTableMetadata() throws Exception {
    Pair<TableDesc, TableExtDesc> pair = explorer.loadTableMetadata("DEFAULT", "TEST_KYLIN_FACT", "DEFAULT");
    Assert.assertNotNull(pair.getFirst());
    Assert.assertNotNull(pair.getSecond());

    TableDesc tblDesc = pair.getFirst();
    TableExtDesc tblExtDesc = pair.getSecond();
    Assert.assertEquals("TEST_KYLIN_FACT", tblDesc.getName());
    Assert.assertEquals("TABLE", tblDesc.getTableType());
    Assert.assertEquals("DEFAULT.TEST_KYLIN_FACT", tblDesc.getIdentity());
    Assert.assertEquals("DEFAULT", tblDesc.getDatabase());
    Assert.assertEquals("DEFAULT", tblDesc.getProject());
    Assert.assertEquals(tblDesc.getIdentity(), tblExtDesc.getIdentity());
    Assert.assertEquals(tblDesc.getProject(), tblExtDesc.getProject());

    ColumnDesc[] columnDescs = tblDesc.getColumns();
    Assert.assertEquals(tblDesc.getColumnCount(), columnDescs.length);
    Assert.assertNotNull(columnDescs[0].getName());
    Assert.assertNotNull(columnDescs[0].getDatatype());
    Assert.assertNotNull(columnDescs[0].getType());
    Assert.assertNotNull(columnDescs[0].getId());
}
 

@Test
public void testLoadTableMetadata() throws Exception {
    Pair<TableDesc, TableExtDesc> pair = explorer.loadTableMetadata("DEFAULT", "TEST_KYLIN_FACT", "DEFAULT");
    Assert.assertNotNull(pair.getFirst());
    Assert.assertNotNull(pair.getSecond());

    TableDesc tblDesc = pair.getFirst();
    TableExtDesc tblExtDesc = pair.getSecond();
    Assert.assertEquals("TEST_KYLIN_FACT", tblDesc.getName());
    Assert.assertEquals("TABLE", tblDesc.getTableType());
    Assert.assertEquals("DEFAULT.TEST_KYLIN_FACT", tblDesc.getIdentity());
    Assert.assertEquals("DEFAULT", tblDesc.getDatabase());
    Assert.assertEquals("DEFAULT", tblDesc.getProject());
    Assert.assertEquals(tblDesc.getIdentity(), tblExtDesc.getIdentity());
    Assert.assertEquals(tblDesc.getProject(), tblExtDesc.getProject());

    ColumnDesc[] columnDescs = tblDesc.getColumns();
    Assert.assertEquals(tblDesc.getColumnCount(), columnDescs.length);
    Assert.assertNotNull(columnDescs[0].getName());
    Assert.assertNotNull(columnDescs[0].getDatatype());
    Assert.assertNotNull(columnDescs[0].getType());
    Assert.assertNotNull(columnDescs[0].getId());
}
 

private void initSrcExt() throws IOException {
    this.srcExtMap = new CaseInsensitiveStringCache<>(config, "table_ext");
    this.srcExtCrud = new CachedCrudAssist<TableExtDesc>(getStore(), ResourceStore.TABLE_EXD_RESOURCE_ROOT,
            TableExtDesc.class, srcExtMap) {
        @Override
        protected TableExtDesc initEntityAfterReload(TableExtDesc t, String resourceName) {
            // convert old tableExt json to new one
            if (t.getIdentity() == null) {
                t = convertOldTableExtToNewer(resourceName);
            }

            String prj = TableDesc.parseResourcePath(resourceName).getProject();
            t.init(prj);
            return t;
        }
    };
    srcExtCrud.reloadAll();
    Broadcaster.getInstance(config).registerListener(new SrcTableExtSyncListener(), "table_ext");
}
 

private String[] generateCreateTableSql(TableDesc tableDesc) {
    logger.info("Generate create table sql: {}", tableDesc);
    String tableIdentity = String
            .format(Locale.ROOT, "%s.%s", tableDesc.getDatabase().toUpperCase(Locale.ROOT), tableDesc.getName())
            .toUpperCase(Locale.ROOT);
    String dropsql = "DROP TABLE IF EXISTS " + tableIdentity;
    String dropsql2 = "DROP VIEW IF EXISTS " + tableIdentity;

    StringBuilder ddl = new StringBuilder();
    ddl.append("CREATE TABLE " + tableIdentity + "\n");
    ddl.append("(" + "\n");

    for (int i = 0; i < tableDesc.getColumns().length; i++) {
        ColumnDesc col = tableDesc.getColumns()[i];
        if (i > 0) {
            ddl.append(",");
        }
        ddl.append(col.getName() + " " + getSqlDataType((col.getDatatype())) + "\n");
    }

    ddl.append(")");

    return new String[] { dropsql, dropsql2, ddl.toString() };
}
 

@Override
public ILookupTable getCachedLookupTable(TableDesc tableDesc, ExtTableSnapshotInfo extTableSnapshotInfo,
        boolean buildIfNotExist) {
    String resourcePath = extTableSnapshotInfo.getResourcePath();
    if (inBuildingTables.containsKey(resourcePath)) {
        logger.info("cache is in building for snapshot:" + resourcePath);
        return null;
    }
    CachedTableInfo cachedTableInfo = tablesCache.getIfPresent(resourcePath);
    if (cachedTableInfo == null) {
        if (buildIfNotExist) {
            buildSnapshotCache(tableDesc, extTableSnapshotInfo,
                    getSourceLookupTable(tableDesc, extTableSnapshotInfo));
        }
        logger.info("no available cache ready for the table snapshot:" + extTableSnapshotInfo.getResourcePath());
        return null;
    }

    String[] keyColumns = extTableSnapshotInfo.getKeyColumns();
    String dbPath = getSnapshotStorePath(extTableSnapshotInfo.getTableName(), extTableSnapshotInfo.getId());
    return new RocksDBLookupTable(tableDesc, keyColumns, dbPath);
}
 

private boolean generateTable(TableDesc table) throws IOException {
    TableGenConfig config = new TableGenConfig(table, this);
    if (!config.needGen)
        return false;

    ByteArrayOutputStream bout = new ByteArrayOutputStream();
    PrintWriter pout = new PrintWriter(new OutputStreamWriter(bout, StandardCharsets.UTF_8));

    generateTableInternal(table, config, pout);

    pout.close();
    bout.close();

    saveResource(bout.toByteArray(), path(table));
    return true;
}
 

private Set<ColumnDesc> findUsedColumnsInFactTable(DataModelDesc usedModel, TableDesc factTable) {
    Set<ColumnDesc> usedColumns = Sets.newHashSet();
    // column in dimension
    for (ModelDimensionDesc dim : usedModel.getDimensions()) {
        if (dim.getTable().equalsIgnoreCase(factTable.getName())) {
            for (String col : dim.getColumns()) {
                usedColumns.add(mustGetColumnDesc(factTable, col));
            }
        }
    }

    // column in measure
    for (String columnInMeasure : usedModel.getMetrics()) {
        if (factTable.getName().equalsIgnoreCase(getTableName(columnInMeasure))) {
            usedColumns.add(mustGetColumnDesc(factTable, columnInMeasure));
        }
    }

    return usedColumns;
}
 

@Test
public void test() throws Exception {

    ISource source = SourceManager.getSource(this);
    ISourceMetadataExplorer explr = source.getSourceMetadataExplorer();
    Pair<TableDesc, TableExtDesc> pair;

    pair = explr.loadTableMetadata("DEFAULT", "TEST_KYLIN_FACT", ProjectInstance.DEFAULT_PROJECT_NAME);
    assertTrue(pair.getFirst().getIdentity().equals("DEFAULT.TEST_KYLIN_FACT"));

    pair = explr.loadTableMetadata("EDW", "TEST_CAL_DT", ProjectInstance.DEFAULT_PROJECT_NAME);
    assertTrue(pair.getFirst().getIdentity().equals("EDW.TEST_CAL_DT"));

}
 

public List<TableDesc> getTableDescByProject(String project, boolean withExt) throws IOException {
    aclEvaluate.checkProjectReadPermission(project);
    List<TableDesc> tables = getProjectManager().listDefinedTables(project);
    if (null == tables) {
        return Collections.emptyList();
    }
    if (withExt) {
        aclEvaluate.checkProjectWritePermission(project);
        tables = cloneTableDesc(tables, project);
    }
    return tables;
}
 

public HiveTable(TableDesc tableDesc) {
    this.database = tableDesc.getDatabase();
    this.hiveTable = tableDesc.getName();
    try {
        this.hiveTableMeta = getHiveClient().getHiveTableMeta(database, hiveTable);
    } catch (Exception e) {
        throw new RuntimeException("cannot get HiveTableMeta", e);
    }
}
 

@Test
public void test() throws Exception {
    ISource source = SourceManager.getDefaultSource();
    ISourceMetadataExplorer explr = source.getSourceMetadataExplorer();
    Pair<TableDesc, TableExtDesc> pair;
    
    pair = explr.loadTableMetadata("DEFAULT", "TEST_KYLIN_FACT", "default");
    assertTrue(pair.getFirst().getIdentity().equals("DEFAULT.TEST_KYLIN_FACT"));
    
    pair = explr.loadTableMetadata("EDW", "TEST_CAL_DT", "default");
    assertTrue(pair.getFirst().getIdentity().equals("EDW.TEST_CAL_DT"));
    
}
 

private void generateDDL(Set<TableDesc> tables) throws IOException {

        ByteArrayOutputStream bout = new ByteArrayOutputStream();
        PrintWriter pout = new PrintWriter(new OutputStreamWriter(bout, StandardCharsets.UTF_8));

        generateDatabaseDDL(tables, pout);
        generateCreateTableDDL(tables, pout);
        generateLoadDataDDL(tables, pout);

        pout.close();
        bout.close();

        saveResource(bout.toByteArray(), path(model));
    }
 

/**
 * Get available table list of the project
 *
 * @return Table metadata array
 * @throws IOException
 */
@RequestMapping(value = "", method = { RequestMethod.GET }, produces = { "application/json" })
@ResponseBody
public List<TableDesc> getTableDesc(@RequestParam(value = "ext", required = false) boolean withExt,
        @RequestParam(value = "project", required = true) String project) throws IOException {
    try {
        return tableService.getTableDescByProject(project, withExt);
    } catch (IOException e) {
        logger.error("Failed to get Hive Tables", e);
        throw new InternalErrorException(e.getLocalizedMessage(), e);
    }
}
 

private ILookupTable getExtLookupTable(CubeSegment cubeSegment, String tableName,
        SnapshotTableDesc snapshotTableDesc) {
    String snapshotResPath = getSnapshotResPath(cubeSegment, tableName, snapshotTableDesc);

    ExtTableSnapshotInfo extTableSnapshot = ExtTableSnapshotInfoManager.getInstance(config)
            .getSnapshot(snapshotResPath);
    TableDesc tableDesc = getMetadataManager().getTableDesc(tableName, cubeSegment.getProject());
    return LookupProviderFactory.getExtLookupTable(tableDesc, extTableSnapshot);
}
 

@Test
public void testBuildSameSnapshotSameTime() throws InterruptedException, IOException {
    final int threadCount = 3;
    final ExecutorService executorService = Executors.newFixedThreadPool(threadCount);
    final CountDownLatch countDownLatch = new CountDownLatch(threadCount);
    final TableDesc tableDesc = genTableDesc("TEST_TABLE");

    kylinConfig = KylinConfig.getInstanceFromEnv();
    snapshotManager = SnapshotManager.getInstance(kylinConfig);
    ResourceStore store = ResourceStore.getStore(kylinConfig);

    for (int i = 0; i < threadCount; ++i) {
        executorService.submit(new Runnable() {
            @Override
            public void run() {
                try {
                    snapshotManager.buildSnapshot(genTable("./origin", expect), tableDesc, kylinConfig);
                } catch (IOException e) {
                    Assert.fail();
                } finally {
                    countDownLatch.countDown();
                }
            }
        });
    }
    countDownLatch.await();
    Assert.assertEquals(1, store.listResources("/table_snapshot/NULL.TEST_TABLE").size());
}
 

private TableDesc genTableDesc(String tableName) {
    TableDesc table = TableDesc.mockup(tableName);
    ColumnDesc desc1 = new ColumnDesc("1", "id", "string", null, null, null, null);
    desc1.setId("1");
    desc1.setDatatype("long");
    ColumnDesc desc2 = new ColumnDesc("2", "country", "string", null, null, null, null);
    desc2.setId("2");
    desc2.setDatatype("string");
    ColumnDesc[] columns = { desc1, desc2 };
    table.setColumns(columns);
    table.init(kylinConfig, "default");
    return table;
}
 

public OLAPTable(OLAPSchema schema, TableDesc tableDesc, boolean exposeMore) {
    super(Object[].class);
    this.exposeMore = exposeMore;
    this.olapSchema = schema;
    this.sourceTable = tableDesc;
    this.rowType = null;
}
 

private Iterable<? extends DataModelDesc> findModelByTable(TableDesc newTableDesc, String prj) {
    List<DataModelDesc> usedModels = Lists.newArrayList();
    List<String> modelNames = dataModelManager.getModelsUsingTable(newTableDesc, prj);
    modelNames.stream()
            .map(mn -> dataModelManager.getDataModelDesc(mn))
            .filter(m -> null != m)
            .forEach(m -> usedModels.add(m));

    return usedModels;
}
 

public void validateModel(String project, DataModelDesc desc) throws IllegalArgumentException {
    String factTableName = desc.getRootFactTableName();
    TableDesc tableDesc = getTableManager().getTableDesc(factTableName, project);
    if ((tableDesc.getSourceType() == ISourceAware.ID_STREAMING || tableDesc.isStreamingTable())
            && (desc.getPartitionDesc() == null || desc.getPartitionDesc().getPartitionDateColumn() == null)) {
        throw new IllegalArgumentException("Must define a partition column.");
    }
}
 

public CubeInstance rebuildLookupSnapshot(CubeInstance cube, String segmentName, String lookupTable)
        throws IOException {
    aclEvaluate.checkProjectOperationPermission(cube);
    Message msg = MsgPicker.getMsg();
    TableDesc tableDesc = getTableManager().getTableDesc(lookupTable, cube.getProject());
    if (tableDesc.isView()) {
        throw new BadRequestException(
                String.format(Locale.ROOT, msg.getREBUILD_SNAPSHOT_OF_VIEW(), tableDesc.getName()));
    }
    CubeSegment seg = cube.getSegment(segmentName, SegmentStatusEnum.READY);
    getCubeManager().buildSnapshotTable(seg, lookupTable, null);

    return cube;
}
 

/**
 * @param context
 * @param cube
 * @param value
 */
private void validateColumnParameter(ValidateContext context, CubeDesc cube, String value) {
    String factTable = cube.getFactTable();
    if (StringUtils.isEmpty(factTable)) {
        context.addResult(ResultLevel.ERROR, "Fact table can not be null.");
        return;
    }
    TableDesc table = MetadataManager.getInstance(cube.getConfig()).getTableDesc(factTable);
    if (table == null) {
        context.addResult(ResultLevel.ERROR, "Fact table can not be found: " + cube);
        return;
    }
    // Prepare column set
    Set<String> set = new HashSet<String>();
    ColumnDesc[] cdesc = table.getColumns();
    for (int i = 0; i < cdesc.length; i++) {
        ColumnDesc columnDesc = cdesc[i];
        set.add(columnDesc.getName());
    }

    String[] items = value.split(",");
    for (int i = 0; i < items.length; i++) {
        String item = items[i].trim();
        if (StringUtils.isEmpty(item)) {
            continue;
        }
        if (!set.contains(item)) {
            context.addResult(ResultLevel.ERROR, "Column [" + item + "] does not exist in factable table" + factTable);
        }
    }

}
 

private boolean sanityCheck(ProjectCache prjCache, IRealization realization) {
    if (realization == null)
        return false;

    TableMetadataManager metaMgr = mgr.getTableManager();

    Set<TblColRef> allColumns = realization.getAllColumns();
    if (allColumns == null || allColumns.isEmpty()) {
        logger.error("Realization '" + realization.getCanonicalName() + "' does not report any columns");
        return false;
    }

    for (TblColRef col : allColumns) {
        TableDesc table = metaMgr.getTableDesc(col.getTable(), prjCache.project);
        if (table == null) {
            logger.error("Realization '" + realization.getCanonicalName() + "' reports column '" + col.getCanonicalName() + "', but its table is not found by MetadataManager");
            return false;
        }

        if (!col.getColumnDesc().isComputedColumn()) {
            ColumnDesc foundCol = table.findColumnByName(col.getName());
            if (col.getColumnDesc().equals(foundCol) == false) {
                logger.error("Realization '" + realization.getCanonicalName() + "' reports column '" + col.getCanonicalName() + "', but it is not equal to '" + foundCol + "' according to MetadataManager");
                return false;
            }
        } else {
            //computed column may not exit here
        }

        // auto-define table required by realization for some legacy test case
        if (prjCache.tables.get(table.getIdentity()) == null) {
            prjCache.tables.put(table.getIdentity(), new TableCache(table));
            logger.warn("Realization '" + realization.getCanonicalName() + "' reports column '" + col.getCanonicalName() + "' whose table is not defined in project '" + prjCache.project + "'");
        }
    }

    return true;
}
 

private List<TableDesc> cloneTableDesc(List<TableDesc> tables, String prj) throws IOException {
    List<TableDesc> descs = new ArrayList<TableDesc>();
    Iterator<TableDesc> it = tables.iterator();
    while (it.hasNext()) {
        TableDesc table = it.next();
        TableDescResponse rtableDesc = cloneTableDesc(table, prj);
        descs.add(rtableDesc);
    }

    return descs;
}