com.alibaba.otter.canal.protocol.CanalEntry.EventType#INSERT

Source File: Support.java From DBus with Apache License 2.0

5 votes

public static List<Column> getFinalColumns(EventType type, RowData rowData) {
    List<Column> Columns = null;
    if (type == EventType.INSERT || type == EventType.UPDATE) {
        Columns = rowData.getAfterColumnsList();
    } else if (type == EventType.DELETE) {
        Columns = rowData.getBeforeColumnsList();
    }
    return Columns;
}

Source File: LogEventConvert.java From canal-1.1.3 with Apache License 2.0

4 votes

private boolean processFilter(String queryString, DdlResult result) {
    String schemaName = result.getSchemaName();
    String tableName = result.getTableName();
    // fixed issue https://github.com/alibaba/canal/issues/58
    // 更新下table meta cache
    if (tableMetaCache != null
        && (result.getType() == EventType.ALTER || result.getType() == EventType.ERASE || result.getType() == EventType.RENAME)) {
        // 对外返回，保证兼容，还是返回QUERY类型，这里暂不解析tableName，所以无法支持过滤
        for (DdlResult renameResult = result; renameResult != null; renameResult = renameResult.getRenameTableResult()) {
            String schemaName0 = renameResult.getSchemaName();
            String tableName0 = renameResult.getTableName();
            if (StringUtils.isNotEmpty(tableName0)) {
                // 如果解析到了正确的表信息，则根据全名进行清除
                tableMetaCache.clearTableMeta(schemaName0, tableName0);
            } else {
                // 如果无法解析正确的表信息，则根据schema进行清除
                tableMetaCache.clearTableMetaWithSchemaName(schemaName0);
            }
        }
    }

    // fixed issue https://github.com/alibaba/canal/issues/58
    if (result.getType() == EventType.ALTER || result.getType() == EventType.ERASE
        || result.getType() == EventType.CREATE || result.getType() == EventType.TRUNCATE
        || result.getType() == EventType.RENAME || result.getType() == EventType.CINDEX
        || result.getType() == EventType.DINDEX) { // 针对DDL类型

        if (filterQueryDdl) {
            return true;
        }

        if (StringUtils.isEmpty(tableName)
            || (result.getType() == EventType.RENAME && StringUtils.isEmpty(result.getOriTableName()))) {
            // 如果解析不出tableName,记录一下日志，方便bugfix，目前直接抛出异常，中断解析
            throw new CanalParseException("SimpleDdlParser process query failed. pls submit issue with this queryString: "
                                          + queryString + " , and DdlResult: " + result.toString());
            // return null;
        } else {
            // check name filter
            String name = schemaName + "." + tableName;
            if (nameFilter != null && !nameFilter.filter(name)) {
                if (result.getType() == EventType.RENAME) {
                    // rename校验只要源和目标满足一个就进行操作
                    if (nameFilter != null
                        && !nameFilter.filter(result.getOriSchemaName() + "." + result.getOriTableName())) {
                        return true;
                    }
                } else {
                    // 其他情况返回null
                    return true;
                }
            }

            if (nameBlackFilter != null && nameBlackFilter.filter(name)) {
                if (result.getType() == EventType.RENAME) {
                    // rename校验只要源和目标满足一个就进行操作
                    if (nameBlackFilter != null
                        && nameBlackFilter.filter(result.getOriSchemaName() + "." + result.getOriTableName())) {
                        return true;
                    }
                } else {
                    // 其他情况返回null
                    return true;
                }
            }
        }
    } else if (result.getType() == EventType.INSERT || result.getType() == EventType.UPDATE
               || result.getType() == EventType.DELETE) {
        // 对外返回，保证兼容，还是返回QUERY类型，这里暂不解析tableName，所以无法支持过滤
        if (filterQueryDml) {
            return true;
        }
    } else if (filterQueryDcl) {
        return true;
    }

    return false;
}

Source File: EventTransactionBuffer.java From canal-1.1.3 with Apache License 2.0

4 votes

private boolean isDml(EventType eventType) {
    return eventType == EventType.INSERT || eventType == EventType.UPDATE || eventType == EventType.DELETE;
}

Source File: EntryHeader.java From DBus with Apache License 2.0

4 votes

public boolean isInsert() {
    return header.getEventType() == EventType.INSERT;
}

Source File: MysqlInitialLoadProcessor.java From DBus with Apache License 2.0

4 votes

/**
 * 处理mysql 拉全量请求
 */
@Override
public void process(final DBusConsumerRecord<String, byte[]> consumerRecord, Object... args) {
    try {
        List<MessageEntry> msgEntryLst = parser.getEntry(consumerRecord.value());
        if (msgEntryLst.isEmpty())
            return;
        EntryHeader entryHeader = msgEntryLst.get(0).getEntryHeader();
        EventType operType = entryHeader.getOperType();

        //TODO 暂时放弃拉全量表的update/delete等消息
        if (operType != EventType.INSERT) {
            listener.reduceFlowSize(consumerRecord.serializedValueSize());
            consumerListener.syncOffset(consumerRecord);
            return;
        }

        // 判断是否处理过该消息
        String msgPos = entryHeader.getPos();
        Object processed = cache.getIfPresent(msgPos);
        if (processed != null) {
            logger.info("Data have bean processed, the data position is [{}]", msgPos);
            listener.reduceFlowSize(consumerRecord.serializedValueSize());
            consumerListener.syncOffset(consumerRecord);
            return;
        }

        logger.info("Received FULL DATA PULL REQUEST message");
        ControlMessage message = Convertor.mysqlFullPullMessage(msgEntryLst.get(0), listener.getListenerId(), consumerRecord);
        String schemaName = getStringValue("SCHEMA_NAME", message);
        String tableName = getStringValue("TABLE_NAME", message);
        DataTable table = ThreadLocalCache.get(Constants.CacheNames.DATA_TABLES, Utils.buildDataTableCacheKey(schemaName, tableName));
        if (table == null) {
            logger.warn("Table {}.{} is not supported,please configure it in dbus database.", schemaName, tableName);
            return;
        }

        for (String controlTopic : controlTopics) {
            String json = message.toJSONString();
            ProducerRecord<String, byte[]> producerRecord = new ProducerRecord<>(controlTopic, message.getType(), json.getBytes());

            Future<RecordMetadata> future = listener.sendRecord(producerRecord);
            future.get();

            logger.info("write initial load request message to kafka: {}", json);
        }

        // 暂停接收 topic 的数据
        TopicPartition tp = new TopicPartition(consumerRecord.topic(), consumerRecord.partition());
        consumerListener.pauseTopic(tp, consumerRecord.offset(), message);

        // emit FULL_DATA_PULL_REQ 通知给bolt
        EmitData emitData = new EmitData();
        emitData.add(EmitData.DB_SCHEMA, schemaName);
        emitData.add(EmitData.DATA_TABLE, tableName);
        listener.emitData(emitData, Command.FULL_DATA_PULL_REQ, consumerRecord);

        consumerListener.syncOffset(consumerRecord);
        cache.put(msgPos, msgEntryLst.get(0).toString());
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Source File: LogEventConvert.java From canal with Apache License 2.0

4 votes

private boolean processFilter(String queryString, DdlResult result) {
    String schemaName = result.getSchemaName();
    String tableName = result.getTableName();
    // fixed issue https://github.com/alibaba/canal/issues/58
    // 更新下table meta cache
    if (tableMetaCache != null
        && (result.getType() == EventType.ALTER || result.getType() == EventType.ERASE || result.getType() == EventType.RENAME)) {
        // 对外返回，保证兼容，还是返回QUERY类型，这里暂不解析tableName，所以无法支持过滤
        for (DdlResult renameResult = result; renameResult != null; renameResult = renameResult.getRenameTableResult()) {
            String schemaName0 = renameResult.getSchemaName();
            String tableName0 = renameResult.getTableName();
            if (StringUtils.isNotEmpty(tableName0)) {
                // 如果解析到了正确的表信息，则根据全名进行清除
                tableMetaCache.clearTableMeta(schemaName0, tableName0);
            } else {
                // 如果无法解析正确的表信息，则根据schema进行清除
                tableMetaCache.clearTableMetaWithSchemaName(schemaName0);
            }
        }
    }

    // fixed issue https://github.com/alibaba/canal/issues/58
    if (result.getType() == EventType.ALTER || result.getType() == EventType.ERASE
        || result.getType() == EventType.CREATE || result.getType() == EventType.TRUNCATE
        || result.getType() == EventType.RENAME || result.getType() == EventType.CINDEX
        || result.getType() == EventType.DINDEX) { // 针对DDL类型

        if (filterQueryDdl) {
            return true;
        }

        if (StringUtils.isEmpty(tableName)
            || (result.getType() == EventType.RENAME && StringUtils.isEmpty(result.getOriTableName()))) {
            // 如果解析不出tableName,记录一下日志，方便bugfix，目前直接抛出异常，中断解析
            throw new CanalParseException("SimpleDdlParser process query failed. pls submit issue with this queryString: "
                                          + queryString + " , and DdlResult: " + result.toString());
            // return null;
        } else {
            // check name filter
            String name = schemaName + "." + tableName;
            if (nameFilter != null && !nameFilter.filter(name)) {
                if (result.getType() == EventType.RENAME) {
                    // rename校验只要源和目标满足一个就进行操作
                    if (nameFilter != null
                        && !nameFilter.filter(result.getOriSchemaName() + "." + result.getOriTableName())) {
                        return true;
                    }
                } else {
                    // 其他情况返回null
                    return true;
                }
            }

            if (nameBlackFilter != null && nameBlackFilter.filter(name)) {
                if (result.getType() == EventType.RENAME) {
                    // rename校验只要源和目标满足一个就进行操作
                    if (nameBlackFilter != null
                        && nameBlackFilter.filter(result.getOriSchemaName() + "." + result.getOriTableName())) {
                        return true;
                    }
                } else {
                    // 其他情况返回null
                    return true;
                }
            }
        }
    } else if (result.getType() == EventType.INSERT || result.getType() == EventType.UPDATE
               || result.getType() == EventType.DELETE) {
        // 对外返回，保证兼容，还是返回QUERY类型，这里暂不解析tableName，所以无法支持过滤
        if (filterQueryDml) {
            return true;
        }
    } else if (filterQueryDcl) {
        return true;
    }

    return false;
}

Source File: EventTransactionBuffer.java From canal with Apache License 2.0

4 votes

private boolean isDml(EventType eventType) {
    return eventType == EventType.INSERT || eventType == EventType.UPDATE || eventType == EventType.DELETE;
}

Java Code Examples for com.alibaba.otter.canal.protocol.CanalEntry.EventType#INSERT