com.alibaba.otter.canal.parse.inbound.TableMeta Java Exaples

Source File: MemoryTableMeta_DDL_Test.java From canal with Apache License 2.0

6 votes

@Test
public void test_any() throws Throwable {
    MemoryTableMeta memoryTableMeta = new MemoryTableMeta();
    URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
    File dummyFile = new File(url.getFile());
    File create = new File(dummyFile.getParent() + "/ddl", "ddl_any.sql");
    String sql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n");
    memoryTableMeta.apply(null, "test", sql, null);
    
    List<String> tableNames = Lists.newArrayList();
    for (Schema schema : memoryTableMeta.getRepository().getSchemas()) {
        tableNames.addAll(schema.showTables());
    }

    for (String table : tableNames) {
        TableMeta sourceMeta = memoryTableMeta.find("test", table);
        System.out.println(sourceMeta.toString());
    }
}

Source File: MemoryTableMeta_DDL_Test.java From canal with Apache License 2.0

6 votes

@Test
public void test1() throws Throwable {
    MemoryTableMeta memoryTableMeta = new MemoryTableMeta();
    URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
    File dummyFile = new File(url.getFile());
    File create = new File(dummyFile.getParent() + "/ddl", "ddl_test1.sql");
    String sql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n");
    memoryTableMeta.apply(null, "test", sql, null);

    TableMeta meta = memoryTableMeta.find("yushitai_test", "card_record");
    System.out.println(meta);
    Assert.assertNotNull(meta.getFieldMetaByName("customization_id"));

    meta = memoryTableMeta.find("yushitai_test", "_card_record_gho");
    Assert.assertNull(meta);
}

Source File: MemoryTableMeta_DDL_Test.java From canal-1.1.3 with Apache License 2.0

6 votes

@Test
public void test1() throws Throwable {
    MemoryTableMeta memoryTableMeta = new MemoryTableMeta();
    URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
    File dummyFile = new File(url.getFile());
    File create = new File(dummyFile.getParent() + "/ddl", "ddl_test1.sql");
    String sql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n");
    memoryTableMeta.apply(null, "test", sql, null);

    TableMeta meta = memoryTableMeta.find("yushitai_test", "card_record");
    System.out.println(meta);
    Assert.assertNotNull(meta.getFieldMetaByName("customization_id"));

    meta = memoryTableMeta.find("yushitai_test", "_card_record_gho");
    Assert.assertNull(meta);
}

Source File: LogEventConvert.java From canal with Apache License 2.0

6 votes

private TableMeta getTableMeta(String dbName, String tbName, boolean useCache, EntryPosition position) {
    try {
        return tableMetaCache.getTableMeta(dbName, tbName, useCache, position);
    } catch (Throwable e) {
        String message = ExceptionUtils.getRootCauseMessage(e);
        if (filterTableError) {
            if (StringUtils.contains(message, "errorNumber=1146") && StringUtils.contains(message, "doesn't exist")) {
                return null;
            } else if (StringUtils.contains(message, "errorNumber=1142")
                       && StringUtils.contains(message, "command denied")) {
                return null;
            }
        }

        throw new CanalParseException(e);
    }
}

Source File: LogEventConvert.java From canal-1.1.3 with Apache License 2.0

6 votes

private TableMeta getTableMeta(String dbName, String tbName, boolean useCache, EntryPosition position) {
    try {
        return tableMetaCache.getTableMeta(dbName, tbName, useCache, position);
    } catch (Throwable e) {
        String message = ExceptionUtils.getRootCauseMessage(e);
        if (filterTableError) {
            if (StringUtils.contains(message, "errorNumber=1146") && StringUtils.contains(message, "doesn't exist")) {
                return null;
            } else if (StringUtils.contains(message, "errorNumber=1142")
                       && StringUtils.contains(message, "command denied")) {
                return null;
            }
        }

        throw new CanalParseException(e);
    }
}

Source File: LogEventConvert_old.java From DBus with Apache License 2.0

5 votes

private TableMeta getTableMeta(String dbName, String tbName, boolean useCache) {
    try {
        return tableMetaCache.getTableMeta(dbName, tbName, useCache);
    } catch (Exception e) {
        String message = ExceptionUtils.getRootCauseMessage(e);
        if (filterTableError) {
            if (StringUtils.contains(message, "errorNumber=1146") && StringUtils.contains(message, "doesn't exist")) {
                return null;
            }
        }

        throw new CanalParseException(e);
    }
}

Source File: LogEventConvert.java From DBus with Apache License 2.0

5 votes

private TableMeta getTableMeta(String dbName, String tbName, boolean useCache) {
    try {
        return tableMetaCache.getTableMeta(dbName, tbName, useCache);
    } catch (Exception e) {
        String message = ExceptionUtils.getRootCauseMessage(e);
        if (filterTableError) {
            if (StringUtils.contains(message, "errorNumber=1146") && StringUtils.contains(message, "doesn't exist")) {
                return null;
            }
        }

        throw new CanalParseException(e);
    }
}

Source File: DatabaseTableMeta.java From canal-1.1.3 with Apache License 2.0

5 votes

@Override
public TableMeta find(String schema, String table) {
    lock.readLock().lock();
    try {
        return memoryTableMeta.find(schema, table);
    } finally {
        lock.readLock().unlock();
    }
}

Source File: DatabaseTableMeta.java From canal with Apache License 2.0

5 votes

@Override
public TableMeta find(String schema, String table) {
    lock.readLock().lock();
    try {
        return memoryTableMeta.find(schema, table);
    } finally {
        lock.readLock().unlock();
    }
}

Source File: MemoryTableMeta.java From canal with Apache License 2.0

5 votes

@Override
public TableMeta find(String schema, String table) {
    List<String> keys = Arrays.asList(schema, table);
    TableMeta tableMeta = tableMetas.get(keys);
    if (tableMeta == null) {
        synchronized (this) {
            tableMeta = tableMetas.get(keys);
            if (tableMeta == null) {
                Schema schemaRep = repository.findSchema(schema);
                if (schemaRep == null) {
                    return null;
                }
                SchemaObject data = schemaRep.findTable(table);
                if (data == null) {
                    return null;
                }
                SQLStatement statement = data.getStatement();
                if (statement == null) {
                    return null;
                }
                if (statement instanceof SQLCreateTableStatement) {
                    tableMeta = parse((SQLCreateTableStatement) statement);
                }
                if (tableMeta != null) {
                    if (table != null) {
                        tableMeta.setTable(table);
                    }
                    if (schema != null) {
                        tableMeta.setSchema(schema);
                    }

                    tableMetas.put(keys, tableMeta);
                }
            }
        }
    }

    return tableMeta;
}

Source File: MemoryTableMeta.java From canal with Apache License 2.0

5 votes

private TableMeta parse(SQLCreateTableStatement statement) {
    int size = statement.getTableElementList().size();
    if (size > 0) {
        TableMeta tableMeta = new TableMeta();
        for (int i = 0; i < size; ++i) {
            SQLTableElement element = statement.getTableElementList().get(i);
            processTableElement(element, tableMeta);
        }
        return tableMeta;
    }

    return null;
}

Source File: TableMetaCache.java From canal with Apache License 2.0

5 votes

public static List<FieldMeta> parseTableMeta(String schema, String table, ResultSetPacket packet) {
    if (packet.getFieldValues().size() > 1) {
        String createDDL = packet.getFieldValues().get(1);
        MemoryTableMeta memoryTableMeta = new MemoryTableMeta();
        memoryTableMeta.apply(DatabaseTableMeta.INIT_POSITION, schema, createDDL, null);
        TableMeta tableMeta = memoryTableMeta.find(schema, table);
        return tableMeta.getFields();
    } else {
        return new ArrayList<FieldMeta>();
    }
}

Source File: TableMetaCache.java From canal with Apache License 2.0

5 votes

public TableMeta getTableMeta(String schema, String table, boolean useCache) {
    if (!useCache) {
        tableMetaDB.invalidate(getFullName(schema, table));
    }

    return tableMetaDB.getUnchecked(getFullName(schema, table));
}

Source File: MemoryTableMeta_Random_DDL_Test.java From canal with Apache License 2.0

5 votes

private void compareTableMeta(int num, MemoryTableMeta source, MemoryTableMeta target) {
    List<String> tableNames = Lists.newArrayList();
    for (Schema schema : source.getRepository().getSchemas()) {
        tableNames.addAll(schema.showTables());
    }

    for (String table : tableNames) {
        TableMeta sourceMeta = source.find("test", table);
        TableMeta targetMeta = target.find("test", table);
        boolean result = DatabaseTableMeta.compareTableMeta(sourceMeta, targetMeta);
        if (!result) {
            Assert.fail(sourceMeta.toString() + " vs " + targetMeta.toString());
        }
    }
}

Source File: MemoryTableMetaTest.java From canal with Apache License 2.0

5 votes

@Test
public void testSimple() throws Throwable {
    MemoryTableMeta memoryTableMeta = new MemoryTableMeta();
    URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
    File dummyFile = new File(url.getFile());
    File create = new File(dummyFile.getParent() + "/ddl", "create.sql");
    String sql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n");
    memoryTableMeta.apply(null, "test", sql, null);

    TableMeta meta = memoryTableMeta.find("test", "test");
    System.out.println(meta);
    Assert.assertTrue(meta.getFieldMetaByName("ID").isKey());
}

Source File: MemoryTableMeta_DDL_Test.java From canal with Apache License 2.0

5 votes

@Test
public void test2() throws Throwable {
    MemoryTableMeta memoryTableMeta = new MemoryTableMeta();
    URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
    File dummyFile = new File(url.getFile());
    File create = new File(dummyFile.getParent() + "/ddl", "ddl_test2.sql");
    String sql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n");
    memoryTableMeta.apply(null, "test", sql, null);

    TableMeta meta = memoryTableMeta.find("yushitai_test", "card_record");
    System.out.println(meta);
    Assert.assertEquals(meta.getFieldMetaByName("id").isKey(), true);
    Assert.assertEquals(meta.getFieldMetaByName("name").isUnique(), true);
}

Source File: MemoryTableMeta_DDL_Test.java From canal with Apache License 2.0

5 votes

@Test
public void test3() throws Throwable {
    MemoryTableMeta memoryTableMeta = new MemoryTableMeta();
    URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
    File dummyFile = new File(url.getFile());
    File create = new File(dummyFile.getParent() + "/ddl", "ddl_test3.sql");
    String sql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n");
    memoryTableMeta.apply(null, "test", sql, null);

    TableMeta meta = memoryTableMeta.find("test", "quniya4");
    System.out.println(meta);
    Assert.assertTrue(meta.getFields().get(0).getColumnName().equalsIgnoreCase("id"));
}

Source File: MemoryTableMeta_DDL_Test.java From canal-1.1.3 with Apache License 2.0

5 votes

@Test
public void test3() throws Throwable {
    MemoryTableMeta memoryTableMeta = new MemoryTableMeta();
    URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
    File dummyFile = new File(url.getFile());
    File create = new File(dummyFile.getParent() + "/ddl", "ddl_test3.sql");
    String sql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n");
    memoryTableMeta.apply(null, "test", sql, null);

    TableMeta meta = memoryTableMeta.find("test", "quniya4");
    System.out.println(meta);
    Assert.assertTrue(meta.getFields().get(0).getColumnName().equalsIgnoreCase("id"));
}

Source File: MemoryTableMeta_DDL_Test.java From canal-1.1.3 with Apache License 2.0

5 votes

@Test
public void test2() throws Throwable {
    MemoryTableMeta memoryTableMeta = new MemoryTableMeta();
    URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
    File dummyFile = new File(url.getFile());
    File create = new File(dummyFile.getParent() + "/ddl", "ddl_test2.sql");
    String sql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n");
    memoryTableMeta.apply(null, "test", sql, null);

    TableMeta meta = memoryTableMeta.find("yushitai_test", "card_record");
    System.out.println(meta);
    Assert.assertEquals(meta.getFieldMetaByName("id").isKey(), true);
    Assert.assertEquals(meta.getFieldMetaByName("name").isUnique(), true);
}

Source File: MemoryTableMetaTest.java From canal-1.1.3 with Apache License 2.0

5 votes

@Test
public void testSimple() throws Throwable {
    MemoryTableMeta memoryTableMeta = new MemoryTableMeta();
    URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
    File dummyFile = new File(url.getFile());
    File create = new File(dummyFile.getParent() + "/ddl", "create.sql");
    String sql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n");
    memoryTableMeta.apply(null, "test", sql, null);

    TableMeta meta = memoryTableMeta.find("test", "test");
    System.out.println(meta);
    Assert.assertTrue(meta.getFieldMetaByName("ID").isKey());
}

Source File: LogEventConvert.java From DBus with Apache License 2.0

5 votes

private TableMeta getTableMeta(String dbName, String tbName, boolean useCache) {
    try {
        return tableMetaCache.getTableMeta(dbName, tbName, useCache);
    } catch (Exception e) {
        String message = ExceptionUtils.getRootCauseMessage(e);
        if (filterTableError) {
            if (StringUtils.contains(message, "errorNumber=1146") && StringUtils.contains(message, "doesn't exist")) {
                return null;
            }
        }

        throw new CanalParseException(e);
    }
}

Source File: MemoryTableMeta.java From canal-1.1.3 with Apache License 2.0

5 votes

@Override
public TableMeta find(String schema, String table) {
    List<String> keys = Arrays.asList(schema, table);
    TableMeta tableMeta = tableMetas.get(keys);
    if (tableMeta == null) {
        synchronized (this) {
            tableMeta = tableMetas.get(keys);
            if (tableMeta == null) {
                Schema schemaRep = repository.findSchema(schema);
                if (schemaRep == null) {
                    return null;
                }
                SchemaObject data = schemaRep.findTable(table);
                if (data == null) {
                    return null;
                }
                SQLStatement statement = data.getStatement();
                if (statement == null) {
                    return null;
                }
                if (statement instanceof SQLCreateTableStatement) {
                    tableMeta = parse((SQLCreateTableStatement) statement);
                }
                if (tableMeta != null) {
                    if (table != null) {
                        tableMeta.setTable(table);
                    }
                    if (schema != null) {
                        tableMeta.setSchema(schema);
                    }

                    tableMetas.put(keys, tableMeta);
                }
            }
        }
    }

    return tableMeta;
}

Source File: MemoryTableMeta.java From canal-1.1.3 with Apache License 2.0

5 votes

private TableMeta parse(SQLCreateTableStatement statement) {
    int size = statement.getTableElementList().size();
    if (size > 0) {
        TableMeta tableMeta = new TableMeta();
        for (int i = 0; i < size; ++i) {
            SQLTableElement element = statement.getTableElementList().get(i);
            processTableElement(element, tableMeta);
        }
        return tableMeta;
    }

    return null;
}

Source File: TableMetaCache.java From canal-1.1.3 with Apache License 2.0

5 votes

public static List<FieldMeta> parseTableMeta(String schema, String table, ResultSetPacket packet) {
    if (packet.getFieldValues().size() > 1) {
        String createDDL = packet.getFieldValues().get(1);
        MemoryTableMeta memoryTableMeta = new MemoryTableMeta();
        memoryTableMeta.apply(DatabaseTableMeta.INIT_POSITION, schema, createDDL, null);
        TableMeta tableMeta = memoryTableMeta.find(schema, table);
        return tableMeta.getFields();
    } else {
        return new ArrayList<FieldMeta>();
    }
}

Source File: TableMetaCache.java From canal-1.1.3 with Apache License 2.0

5 votes

public TableMeta getTableMeta(String schema, String table, boolean useCache) {
    if (!useCache) {
        tableMetaDB.invalidate(getFullName(schema, table));
    }

    return tableMetaDB.getUnchecked(getFullName(schema, table));
}

Source File: DatabaseTableMeta.java From canal-1.1.3 with Apache License 2.0

4 votes

private boolean compareTableMeta(TableMeta source, TableMeta target) {
    if (!StringUtils.equalsIgnoreCase(source.getSchema(), target.getSchema())) {
        return false;
    }

    if (!StringUtils.equalsIgnoreCase(source.getTable(), target.getTable())) {
        return false;
    }

    List<FieldMeta> sourceFields = source.getFields();
    List<FieldMeta> targetFields = target.getFields();
    if (sourceFields.size() != targetFields.size()) {
        return false;
    }

    for (int i = 0; i < sourceFields.size(); i++) {
        FieldMeta sourceField = sourceFields.get(i);
        FieldMeta targetField = targetFields.get(i);
        if (!StringUtils.equalsIgnoreCase(sourceField.getColumnName(), targetField.getColumnName())) {
            return false;
        }

        // if (!StringUtils.equalsIgnoreCase(sourceField.getColumnType(),
        // targetField.getColumnType())) {
        // return false;
        // }

        // https://github.com/alibaba/canal/issues/1100
        // 支持一下 int vs int(10)
        if ((sourceField.isUnsigned() && !targetField.isUnsigned())
            || (!sourceField.isUnsigned() && targetField.isUnsigned())) {
            return false;
        }

        String sign = sourceField.isUnsigned() ? "unsigned" : "signed";
        String sourceColumnType = StringUtils.removeEndIgnoreCase(sourceField.getColumnType(), sign).trim();
        String targetColumnType = StringUtils.removeEndIgnoreCase(targetField.getColumnType(), sign).trim();

        boolean columnTypeCompare = false;
        columnTypeCompare |= StringUtils.containsIgnoreCase(sourceColumnType, targetColumnType);
        columnTypeCompare |= StringUtils.containsIgnoreCase(targetColumnType, sourceColumnType);
        if (!columnTypeCompare) {
            return false;
        }

        // if (!StringUtils.equalsIgnoreCase(sourceField.getDefaultValue(),
        // targetField.getDefaultValue())) {
        // return false;
        // }

        if (sourceField.isNullable() != targetField.isNullable()) {
            return false;
        }

        // mysql会有一种处理,针对show create只有uk没有pk时，会在desc默认将uk当做pk
        boolean isSourcePkOrUk = sourceField.isKey() || sourceField.isUnique();
        boolean isTargetPkOrUk = targetField.isKey() || targetField.isUnique();
        if (isSourcePkOrUk != isTargetPkOrUk) {
            return false;
        }
    }

    return true;
}

Source File: TableMetaCache.java From canal-1.1.3 with Apache License 2.0

4 votes

public TableMeta getTableMeta(String schema, String table) {
    return getTableMeta(schema, table, true);
}

Source File: MysqlMultiStageCoprocessor.java From canal with Apache License 2.0

4 votes

public void setTable(TableMeta table) {
    this.table = table;
}

Source File: MysqlMultiStageCoprocessor.java From canal with Apache License 2.0

4 votes

public TableMeta getTable() {
    return table;
}

Source File: MysqlMultiStageCoprocessor.java From canal with Apache License 2.0

4 votes

public void onEvent(MessageEvent event, long sequence, boolean endOfBatch) throws Exception {
    try {
        LogEvent logEvent = event.getEvent();
        if (logEvent == null) {
            LogBuffer buffer = event.getBuffer();
            logEvent = decoder.decode(buffer, context);
            event.setEvent(logEvent);
        }

        int eventType = logEvent.getHeader().getType();
        TableMeta tableMeta = null;
        boolean needDmlParse = false;
        switch (eventType) {
            case LogEvent.WRITE_ROWS_EVENT_V1:
            case LogEvent.WRITE_ROWS_EVENT:
                tableMeta = logEventConvert.parseRowsEventForTableMeta((WriteRowsLogEvent) logEvent);
                needDmlParse = true;
                break;
            case LogEvent.UPDATE_ROWS_EVENT_V1:
            case LogEvent.PARTIAL_UPDATE_ROWS_EVENT:
            case LogEvent.UPDATE_ROWS_EVENT:
                tableMeta = logEventConvert.parseRowsEventForTableMeta((UpdateRowsLogEvent) logEvent);
                needDmlParse = true;
                break;
            case LogEvent.DELETE_ROWS_EVENT_V1:
            case LogEvent.DELETE_ROWS_EVENT:
                tableMeta = logEventConvert.parseRowsEventForTableMeta((DeleteRowsLogEvent) logEvent);
                needDmlParse = true;
                break;
            case LogEvent.ROWS_QUERY_LOG_EVENT:
                needDmlParse = true;
                break;
            default:
                CanalEntry.Entry entry = logEventConvert.parse(event.getEvent(), false);
                event.setEntry(entry);
        }

        // 记录一下DML的表结构
        event.setNeedDmlParse(needDmlParse);
        event.setTable(tableMeta);
    } catch (Throwable e) {
        exception = new CanalParseException(e);
        throw exception;
    }
}

com.alibaba.otter.canal.parse.inbound.TableMeta Java Examples