com.alibaba.otter.canal.protocol.CanalEntry Java Examples

private void printSummary(Message message, long batchId, int size) {
    long memsize = 0;
    for (CanalEntry.Entry entry : message.getEntries()) {
        memsize += entry.getHeader().getEventLength();
    }

    String startPosition = null;
    String endPosition = null;
    if (!CollectionUtils.isEmpty(message.getEntries())) {
        startPosition = buildPositionForDump(message.getEntries().get(0));
        endPosition = buildPositionForDump(message.getEntries().get(message.getEntries().size() - 1));
    }

    logger.info(context_format, new Object[]{batchId, size, memsize, format.format(new Date()), startPosition,
            endPosition});
}
 

@Override
public int getRowCount() {
    //int proCount = entry.getHeader().getPropsCount();
    //if (proCount == 1) {
    //    CanalEntry.Pair pair = entry.getHeader().getProps(0);
    //    if (pair.getKey().equals("rowCount"))
    //        return Integer.parseInt(pair.getValue());
    //}
    for (CanalEntry.Pair pair : entry.getHeader().getPropsList()) {
        if (pair.getKey().equals("rowsCount")) {
            return Integer.parseInt(pair.getValue());
        }
    }

    return 1;
}
 

protected CanalEntry.Entry parseAndProfilingIfNecessary(EVENT bod, boolean isSeek) throws Exception {
    long startTs = -1;
    boolean enabled = getProfilingEnabled();
    if (enabled) {
        startTs = System.currentTimeMillis();
    }
    CanalEntry.Entry event = binlogParser.parse(bod, isSeek);
    if (enabled) {
        this.parsingInterval = System.currentTimeMillis() - startTs;
    }

    if (parsedEventCount.incrementAndGet() < 0) {
        parsedEventCount.set(0);
    }
    return event;
}
 

protected boolean doFilter(CanalEntry.Entry entry) {
    if (filter != null && entry.getEntryType() == EntryType.ROWDATA) {
        String name = getSchemaNameAndTableName(entry);
        boolean need = filter.filter(name);
        if (!need) {
            logger.debug("filter name[{}] entry : {}:{}",
                name,
                entry.getHeader().getLogfileName(),
                entry.getHeader().getLogfileOffset());
        }

        return need;
    } else {
        return true;
    }
}
 

/**
 * {@link Schema.Table#columns}有值, 则对于UPDATE操作过滤更改的字段是否包含在{@link Schema.Table#columns}
 * DELETE, INSERT事件执行条件过滤, 对于UPDATE的过滤不在这做, 比较复杂, 由子类自己实现过滤
 *
 * @param rowChange 更改的数据
 * @return 解析结果
 */
@SuppressWarnings({"rawtypes", "unchecked"})
@Override
protected final List<RowChangedData> changedDataParse(CanalEntry.RowChange rowChange) {
    List<RowChangedData> dataList = RowChangedData.build(rowChange, currentTable.getRowDataColumns());
    if (CommonsUtils.isEmpty(dataList)) return null;
    ConditionContainer columnCondition;
    if (currentEventType != CanalEntry.EventType.UPDATE
            && (columnCondition = currentTable.getColumnCondition()) != null) {
        //对于INSERT类型的记录更新, 如果条件判断没有通过, 可以认为该更新事件没有发生~~~~
        //对于DELETE类型的记录更新, 如果条件判断没有通过, 可以认为该数据删除之前就不关心, 那这次删除我们更不关心了~~~
        Iterator<RowChangedData> it = dataList.iterator();
        while (it.hasNext()) {
            if (!columnCondition.verify(it.next())) {
                it.remove();
            }
        }
    }
    return dataList;
}
 

public void deleteData(String schemaName, String tableName, DBObject obj) {
    int i = 0;
    String path = "/" + schemaName + "/" + tableName + "/" + CanalEntry.EventType.DELETE.getNumber();
    DBObject newObj = (DBObject) ObjectUtils.clone(obj);
    DBObject logObj = (DBObject) ObjectUtils.clone(obj);
    //保存原始数据
    try {
        i++;
        if (obj.containsField("id")) {
            naiveMongoTemplate.getCollection(tableName).remove(new BasicDBObject("_id", obj.get("id")));
        }
        i++;
        SpringUtil.doEvent(path, newObj);
    } catch (MongoClientException | MongoSocketException clientException) {
        //客户端连接异常抛出，阻塞同步，防止mongodb宕机
        throw clientException;
    } catch (Exception e) {
        logError(schemaName, tableName, 3, i, logObj, e);
    }
}
 

protected CanalEntry.Entry parseAndProfilingIfNecessary(EVENT bod) throws Exception {
    long startTs = -1;
    boolean enabled = getProfilingEnabled();
    if (enabled) {
        startTs = System.currentTimeMillis();
    }
    CanalEntry.Entry event = binlogParser.parse(bod);
    if (enabled) {
        this.parsingInterval = System.currentTimeMillis() - startTs;
    }

    if (parsedEventCount.incrementAndGet() < 0) {
        parsedEventCount.set(0);
    }
    return event;
}
 

@Override
public void onEvent(MessageEvent event) throws Exception {
    try {
        if (event.isNeedDmlParse()) {
            int eventType = event.getEvent().getHeader().getType();
            CanalEntry.Entry entry = null;
            switch (eventType) {
                case LogEvent.ROWS_QUERY_LOG_EVENT:
                    entry = logEventConvert.parse(event.getEvent(), false);
                    break;
                default:
                    // 单独解析dml事件
                    entry = logEventConvert.parseRowsEvent((RowsLogEvent) event.getEvent(), event.getTable());
            }

            event.setEntry(entry);
        }
    } catch (Throwable e) {
        exception = new CanalParseException(e);
        throw exception;
    }
}
 

protected boolean consumeTheEventAndProfilingIfNecessary(List<CanalEntry.Entry> entrys) throws CanalSinkException,
                                                                                       InterruptedException {
    long startTs = -1;
    boolean enabled = getProfilingEnabled();
    if (enabled) {
        startTs = System.currentTimeMillis();
    }

    boolean result = eventSink.sink(entrys, (runningInfo == null) ? null : runningInfo.getAddress(), destination);

    if (enabled) {
        this.processingInterval = System.currentTimeMillis() - startTs;
    }

    if (consumedEventCount.incrementAndGet() < 0) {
        consumedEventCount.set(0);
    }

    return result;
}
 

public void insertData(String schemaName, String tableName, DBObject naive, DBObject complete) {
    int i = 0;
    DBObject logObj = (DBObject) ObjectUtils.clone(complete);
    //保存原始数据
    try {
        String path = "/" + schemaName + "/" + tableName + "/" + CanalEntry.EventType.INSERT.getNumber();
        i++;
        naiveMongoTemplate.getCollection(tableName).insert(naive);
        i++;
        SpringUtil.doEvent(path, complete);
        i++;
    } catch (MongoClientException | MongoSocketException clientException) {
        //客户端连接异常抛出，阻塞同步，防止mongodb宕机
        throw clientException;
    } catch (Exception e) {
        logError(schemaName, tableName, 1, i, logObj, e);
    }
}
 

public void update(List<CanalEntry.Column> data, String schemaName, String tableName) {
    DBObject obj = DBConvertUtil.columnToJson(data);
    logger.debug("update：{}", obj.toString());
    //订单库单独处理
    if (schemaName.equals("order")) {
        if (tableName.startsWith("order_base_info")) {
            tableName = "order_base_info";
        } else if (tableName.startsWith("order_detail_info")) {
            tableName = "order_detail_info";
        } else {
            logger.info("unknown data：{}.{}:{}", schemaName, tableName, obj);
        }
        updateData(schemaName, tableName, new BasicDBObject("orderId", obj.get("orderId")), obj);
    } else {
        if (obj.containsField("id")) {
            updateData(schemaName, tableName, new BasicDBObject("_id", obj.get("id")), obj);
        } else {
            logger.info("unknown data structure");
        }
    }
}
 

public void insert(List<CanalEntry.Column> data, String schemaName, String tableName) {
    DBObject obj = DBConvertUtil.columnToJson(data);
    logger.debug("insert ：{}", obj.toString());
    //订单库单独处理
    if (schemaName.equals("order")) {
        //保存原始数据
        if (tableName.startsWith("order_base_info")) {
            tableName = "order_base_info";
        } else if (tableName.startsWith("order_detail_info")) {
            tableName = "order_detail_info";
        } else {
            logger.info("unknown data ：{}.{}:{}", schemaName, tableName, obj);
            return;
        }
        insertData(schemaName, tableName, obj, obj);
    } else {
        DBObject newObj = (DBObject) ObjectUtils.clone(obj);
        if (newObj.containsField("id")) {
            newObj.put("_id", newObj.get("id"));
            newObj.removeField("id");
        }
        insertData(schemaName, tableName, newObj, obj);
    }
}
 

@SuppressWarnings({"rawtypes", "unchecked"})
private void runLastEventTypeOfAction(int eventType, List<? extends RowChangedData> dataList) {
    if (log.isDebugEnabled()) {
        log.debug("canal instance: " + instanceName + " need handle data size: " + dataList.size() + ", eventType: " + eventType
                + " table: " + lastTable);
    }
    if (eventType == CanalEntry.EventType.UPDATE_VALUE) {
        lastTable.getAction().onUpdateAction(Collections.unmodifiableList((List<RowChangedData.Update>) dataList));
    } else if (eventType == CanalEntry.EventType.INSERT_VALUE) {
        lastTable.getAction().onInsertAction(Collections.unmodifiableList((List<RowChangedData.Insert>) dataList));
    } else {
        lastTable.getAction().onDeleteAction(Collections.unmodifiableList((List<RowChangedData.Delete>) dataList));
    }
    //这儿主动调用clear, 数据无效掉, 避免调用Action时保留引用导致无法回收
    for (RowChangedData data : dataList) {
        data.close();
    }
    //这儿清楚掉
    dataList.clear();
}
 

public AbstractEventParser(){
    // 初始化一下
    transactionBuffer = new EventTransactionBuffer(new TransactionFlushCallback() {

        public void flush(List<CanalEntry.Entry> transaction) throws InterruptedException {
            boolean successed = consumeTheEventAndProfilingIfNecessary(transaction);
            if (!running) {
                return;
            }

            if (!successed) {
                throw new CanalParseException("consume failed!");
            }

            LogPosition position = buildLastTransactionPosition(transaction);
            if (position != null) { // 可能position为空
                logPositionManager.persistLogPosition(AbstractEventParser.this.destination, position);
            }
        }
    });
}
 

protected boolean consumeTheEventAndProfilingIfNecessary(List<CanalEntry.Entry> entrys) throws CanalSinkException,
                                                                                       InterruptedException {
    long startTs = -1;
    boolean enabled = getProfilingEnabled();
    if (enabled) {
        startTs = System.currentTimeMillis();
    }

    boolean result = eventSink.sink(entrys, (runningInfo == null) ? null : runningInfo.getAddress(), destination);

    if (enabled) {
        this.processingInterval = System.currentTimeMillis() - startTs;
    }

    if (consumedEventCount.incrementAndGet() < 0) {
        consumedEventCount.set(0);
    }

    return result;
}
 

protected CanalEntry.Entry parseAndProfilingIfNecessary(EVENT bod, boolean isSeek) throws Exception {
    long startTs = -1;
    boolean enabled = getProfilingEnabled();
    if (enabled) {
        startTs = System.currentTimeMillis();
    }
    CanalEntry.Entry event = binlogParser.parse(bod, isSeek);
    if (enabled) {
        this.parsingInterval = System.currentTimeMillis() - startTs;
    }

    if (parsedEventCount.incrementAndGet() < 0) {
        parsedEventCount.set(0);
    }
    return event;
}
 

@Override
public byte[] wrapMessages(List<IGenericMessage> list) throws IOException {
    CanalPacket.Messages.Builder builder = CanalPacket.Messages.newBuilder();
    builder.setBatchId(0);

    for (IGenericMessage obj : list) {
        MysqlGenericMessage message = (MysqlGenericMessage) obj;
        CanalEntry.Entry ent = message.getEntry();

        builder.addMessages(ent.toByteString());
    }
    CanalPacket.Messages cMessage = builder.build();
    return cMessage.toByteArray();
}
 

/**
 * 不做筛选
 */
@Override
public boolean startHandle(CanalEntry.Header header) {
    currentEventType = header.getEventType();
    //事件类型排除处理
    if ((forbidEventType & RowChangedData.getEventTypeFlag(currentEventType)) != 0) return false;
    currentHandleSchema = header.getSchemaName();
    currentHandleTable = header.getTableName();
    return true;
}
 

@Override
public List<IGenericMessage> unwrapMessages(byte[] data) throws IOException {
    // canal-1.0.24
    //List<IGenericMessage> list = new ArrayList<>();
    //
    //CanalPacket.Messages cMessage = CanalPacket.Messages.parseFrom(data);
    //List<ByteString> strings = cMessage.getMessagesList();
    //for (ByteString str : strings) {
    //    CanalEntry.Entry ent = CanalEntry.Entry.parseFrom(str);
    //    MysqlGenericMessage message = new MysqlGenericMessage(ent);
    //
    //    list.add(message);
    //}
    // canal-1.1.4
    Message message = CanalMessageDeserializer.deserializer(data);
    List<IGenericMessage> list = new ArrayList<>();
    for (CanalEntry.Entry entry : message.getEntries()) {
        if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
            logger.debug("the entry type is transaction begin or transaction end.");
            continue;
        }
        //处理带正则的表名
        String schemaName = entry.getHeader().getSchemaName();
        String tableName = entry.getHeader().getTableName();
        String localTable = getLocalTable(schemaName, tableName);
        logger.debug("收到表数据{}.{}(基准表名{})数据", schemaName, tableName, localTable);
        if (!StringUtils.equals(tableName, localTable)) {
            String finalTable = StringUtils.join(new String[]{localTable, tableName}, ".");
            CanalEntry.Header header = CanalEntry.Header.newBuilder(entry.getHeader()).setTableName(finalTable).build();
            entry = CanalEntry.Entry.newBuilder(entry).setHeader(header).build();
            logger.debug("重新生成entry");
        }
        MysqlGenericMessage mysqlGenericMessage = new MysqlGenericMessage(entry);
        list.add(mysqlGenericMessage);
    }
    return list;
}
 

private void handlerMap() {
    if (instanceMap.size() <= 0)
        return;
    for (Map.Entry<String, Object> entry : instanceMap.entrySet()) {
        if (entry.getValue().getClass().isAnnotationPresent(Schema.class)) {
            Schema schema = entry.getValue().getClass().getAnnotation(Schema.class);
            String schemeName = schema.value();
            Method[] methods = entry.getValue().getClass().getMethods();
            for (Method method : methods) {
                if (method.isAnnotationPresent(Table.class)) {
                    Table table = method.getAnnotation(Table.class);
                    String tName = table.value();
                    CanalEntry.EventType[] events = table.event();
                    //未标明数据事件类型的方法不做映射
                    if (events.length < 1) {
                        continue;
                    }
                    //同一个方法可以映射多张表
                    for (int i = 0; i < events.length; i++) {
                        String path = "/" + schemeName + "/" + tName + "/" + events[i].getNumber();
                        handlerMap.put(path, method);
                        logger.info("handlerMap [{}:{}]", path, method.getName());
                    }
                } else {
                    continue;
                }
            }
        } else {
            continue;
        }

    }
}
 

/**
 * 获取当前Entry的 GTID Sequence No信息示例
 * 
 * @param header
 * @return
 */
public static String getCurrentGtidSn(CanalEntry.Header header) {
    List<CanalEntry.Pair> props = header.getPropsList();
    if (props != null && props.size() > 0) {
        for (CanalEntry.Pair pair : props) {
            if ("curtGtidSn".equals(pair.getKey())) {
                return pair.getValue();
            }
        }
    }
    return "";
}
 

@Override
public ElasticsearchMetadata trans(Message message) {
    List<CanalEntry.Entry> entries = message.getEntries();
    ElasticsearchMetadata elasticsearchMetadata = TotoroObjectPool.esMetadata();
    elasticsearchMetadata.setBatchId(message.getId());
    elasticsearchMetadata.setTransForm(this);
    if (entries != null && entries.size() > 0) {
        EsEntryArrayList esEntryList = TotoroObjectPool.esEntryArrayList();

        entries.forEach(entry -> {
            if (messageFilterChain.filter(entry)) {
                try {
                    ElasticsearchMetadata.EsEntry esEntry = getElasticsearchMetadata(entry);
                    esEntryList.add(esEntry);
                } catch (InvalidProtocolBufferException e) {
                    e.printStackTrace();
                }
            }
        });
        elasticsearchMetadata.setEsEntries(esEntryList);
    }

    logger.info("Trans form complete message id =====> {}", message.getId());

    if (logger.isDebugEnabled()) {
        logger.debug("Trans form complete elasticsearch metadata  =====> {}", elasticsearchMetadata.toString());
    }
    return elasticsearchMetadata;
}
 

private Map<String,Object> processColumnList(List<CanalEntry.Column> columnList) {
    Map<String,Object> map = new HashMap<>();
    for (CanalEntry.Column column : columnList) {
        map.put(column.getName(), column.getValue());
    }
    return map;
}
 

protected LogPosition buildLastTransactionPosition(List<CanalEntry.Entry> entries) { // 初始化一下
    for (int i = entries.size() - 1; i > 0; i--) {
        CanalEntry.Entry entry = entries.get(i);
        if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {// 尽量记录一个事务做为position
            return buildLastPosition(entry);
        }
    }

    return null;
}
 

private ElasticsearchMetadata.EsEntry getElasticsearchMetadata(CanalEntry.Entry entry) throws InvalidProtocolBufferException {

        final String database = entry.getHeader().getSchemaName(); // => index
        final String table = entry.getHeader().getTableName();// => type
        final CanalEntry.RowChange change = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
        final List<CanalEntry.RowData> rowDataList = change.getRowDatasList();

        CanalEntry.EventType eventType = entry.getHeader().getEventType();
        final int esEventType = esAdapter.getEsEventType(eventType);


        EsRowDataArrayList esRowDataList = TotoroObjectPool.esRowDataArrayList();

        for (CanalEntry.RowData rowData : rowDataList) {
            List<CanalEntry.Column> columnList = esAdapter.getColumnList(esEventType, rowData);
            ElasticsearchMetadata.EsRowData esRowData = TotoroObjectPool.esRowData();
            EsColumnHashMap columnMap = TotoroObjectPool.esColumnHashMap();
            columnList.forEach(column -> columnMap.put(column.getName(), column.getValue()));
            esRowData.setRowData(columnMap);
            esRowData.setIdColumn(esAdapter.getEsIdColumn(database, table));//获取es对应的id Column
            esRowDataList.add(esRowData);
        }

        ElasticsearchMetadata.EsEntry esEntry = TotoroObjectPool.esEntry();

        esEntry.setIndex(database)
                .setType(table)
                .setEsRowDatas(esRowDataList)
                .setEventType(esEventType);

        return esEntry;
    }
 

/**
 * 获取当前Entry的 GTID信息示例
 * 
 * @param header
 * @return
 */
public static String getCurrentGtid(CanalEntry.Header header) {
    List<CanalEntry.Pair> props = header.getPropsList();
    if (props != null && props.size() > 0) {
        for (CanalEntry.Pair pair : props) {
            if ("curtGtid".equals(pair.getKey())) {
                return pair.getValue();
            }
        }
    }
    return "";
}
 

public void add(CanalEntry.Entry entry) throws InterruptedException {
    switch (entry.getEntryType()) {
        case TRANSACTIONBEGIN:
            flush();// 刷新上一次的数据
            put(entry);
            break;
        case TRANSACTIONEND:
            put(entry);
            flush();
            break;
        case ROWDATA:
            put(entry);
            // 针对非DML的数据，直接输出，不进行buffer控制
            EventType eventType = entry.getHeader().getEventType();
            if (eventType != null && !isDml(eventType)) {
                flush();
            }
            break;
        case HEARTBEAT:
            // master过来的heartbeat，说明binlog已经读完了，是idle状态
            put(entry);
            flush();
            break;
        default:
            break;
    }
}
 

private CanalEntry.RowChange parseRowChange(CanalEntry.Entry entry) {
    CanalEntry.RowChange rowChange = null;
    try {
        rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
    } catch (Exception e) {
        logger.error("ERROR ## parser of eromanga-event has an error , data:" + entry.toString());
    }
    return rowChange;
}
 

/**
 * 获取当前Entry的 GTID信息示例
 * 
 * @param header
 * @return
 */
public static String getCurrentGtid(CanalEntry.Header header) {
    List<CanalEntry.Pair> props = header.getPropsList();
    if (props != null && props.size() > 0) {
        for (CanalEntry.Pair pair : props) {
            if ("curtGtid".equals(pair.getKey())) {
                return pair.getValue();
            }
        }
    }
    return "";
}
 

public static Message deserializer(byte[] data, boolean lazyParseEntry) {
    try {
        if (data == null) {
            return null;
        } else {
            CanalPacket.Packet p = CanalPacket.Packet.parseFrom(data);
            switch (p.getType()) {
                case MESSAGES: {
                    if (!p.getCompression().equals(CanalPacket.Compression.NONE)
                        && !p.getCompression().equals(CanalPacket.Compression.COMPRESSIONCOMPATIBLEPROTO2)) {
                        throw new CanalClientException("compression is not supported in this connector");
                    }

                    CanalPacket.Messages messages = CanalPacket.Messages.parseFrom(p.getBody());
                    Message result = new Message(messages.getBatchId());
                    if (lazyParseEntry) {
                        // byteString
                        result.setRawEntries(messages.getMessagesList());
                        result.setRaw(true);
                    } else {
                        for (ByteString byteString : messages.getMessagesList()) {
                            result.addEntry(CanalEntry.Entry.parseFrom(byteString));
                        }
                        result.setRaw(false);
                    }
                    return result;
                }
                case ACK: {
                    CanalPacket.Ack ack = CanalPacket.Ack.parseFrom(p.getBody());
                    throw new CanalClientException("something goes wrong with reason: " + ack.getErrorMessage());
                }
                default: {
                    throw new CanalClientException("unexpected packet type: " + p.getType());
                }
            }
        }
    } catch (Exception e) {
        throw new CanalClientException("deserializer failed", e);
    }
}