Java Code Examples for com.alibaba.otter.canal.protocol.CanalEntry#RowChange

/**
 * {@link Schema.Table#columns}有值, 则对于UPDATE操作过滤更改的字段是否包含在{@link Schema.Table#columns}
 * DELETE, INSERT事件执行条件过滤, 对于UPDATE的过滤不在这做, 比较复杂, 由子类自己实现过滤
 *
 * @param rowChange 更改的数据
 * @return 解析结果
 */
@SuppressWarnings({"rawtypes", "unchecked"})
@Override
protected final List<RowChangedData> changedDataParse(CanalEntry.RowChange rowChange) {
    List<RowChangedData> dataList = RowChangedData.build(rowChange, currentTable.getRowDataColumns());
    if (CommonsUtils.isEmpty(dataList)) return null;
    ConditionContainer columnCondition;
    if (currentEventType != CanalEntry.EventType.UPDATE
            && (columnCondition = currentTable.getColumnCondition()) != null) {
        //对于INSERT类型的记录更新, 如果条件判断没有通过, 可以认为该更新事件没有发生~~~~
        //对于DELETE类型的记录更新, 如果条件判断没有通过, 可以认为该数据删除之前就不关心, 那这次删除我们更不关心了~~~
        Iterator<RowChangedData> it = dataList.iterator();
        while (it.hasNext()) {
            if (!columnCondition.verify(it.next())) {
                it.remove();
            }
        }
    }
    return dataList;
}
 

/**
 * 消费当前消息
 */
private void consumerMessage(Message message) {
    log.debug("canal instance: " + handle.instanceName() + " get message entry size " + message.getEntries().size());
    try {
        for (CanalEntry.Entry e : message.getEntries()) {
            if (e.getEntryType() != CanalEntry.EntryType.ROWDATA || !e.hasStoreValue()) continue;
            CanalEntry.Header header = e.getHeader();
            if (header.getExecuteTime() < startRtTime
                    || header.getEventType().getNumber() > CanalEntry.EventType.DELETE_VALUE
                    || !handle.startHandle(header)) continue;
            try {
                CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(e.getStoreValue());
                if (rowChange.getIsDdl()) continue;
                handle.rowChangeHandle(rowChange);
            } catch (InvalidProtocolBufferException e1) {
                log.error("canal instance: " + handle.instanceName() + " parse store value have exception: ", e1);
            }
        }
        handle.ack(message.getId());
    } finally {
        handle.finishMessageHandle();
    }
}
 

@Override
public boolean sink(List<CanalEntry.Entry> entries, InetSocketAddress inetSocketAddress, String s) throws CanalSinkException, InterruptedException {
    for (CanalEntry.Entry entry : entries) {
        CanalEntry.EntryType entryType = entry.getEntryType();
        if (entryType != CanalEntry.EntryType.ROWDATA) {
            continue;
        }

        CanalEntry.RowChange rowChange = parseRowChange(entry);

        if(rowChange == null) {
            return false;
        }

        CanalEntry.Header header = entry.getHeader();
        long ts = header.getExecuteTime();
        String schema = header.getSchemaName();
        String table = header.getTableName();
        processRowChange(rowChange, schema, table, ts);
    }

    return true;
}
 

private void processRowChange(CanalEntry.RowChange rowChange, String schema, String table, long ts) {
    CanalEntry.EventType eventType = rowChange.getEventType();

    if(!binlog.accept(eventType.toString())) {
        return;
    }

    for(CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
        Map<String,Object> event = new HashMap<>();
        Map<String,Object> message = new HashMap<>();
        message.put("type", eventType.toString());
        message.put("schema", schema);
        message.put("table", table);
        message.put("ts", ts);
        message.put("before", processColumnList(rowData.getBeforeColumnsList()));
        message.put("after", processColumnList(rowData.getAfterColumnsList()));
        event.put("message", message);
        binlog.process(event);
    }

}
 

@Override
protected void distributeEvent(Message message) {
    List<CanalEntry.Entry> entries = message.getEntries();
    for (CanalEntry.Entry entry : entries) {
        //ignore the transaction operations
        List<CanalEntry.EntryType> ignoreEntryTypes = getIgnoreEntryTypes();
        if (ignoreEntryTypes != null
                && ignoreEntryTypes.stream().anyMatch(t -> entry.getEntryType() == t)) {
            continue;
        }
        CanalEntry.RowChange rowChange;
        try {
            rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
        } catch (Exception e) {
            throw new CanalClientException("ERROR ## parser of event has an error , data:" + entry.toString(),
                    e);
        }
        //ignore the ddl operation
        if (rowChange.hasIsDdl() && rowChange.getIsDdl()) {
            processDdl(rowChange);
            continue;
        }
        for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
            //distribute to listener interfaces
            distributeByImpl(rowChange.getEventType(), rowData);
            //distribute to annotation listener interfaces
            distributeByAnnotation(destination,
                    entry.getHeader().getSchemaName(),
                    entry.getHeader().getTableName(),
                    rowChange.getEventType(),
                    rowData);
        }
    }
}
 

private ElasticsearchMetadata.EsEntry getElasticsearchMetadata(CanalEntry.Entry entry) throws InvalidProtocolBufferException {

        final String database = entry.getHeader().getSchemaName(); // => index
        final String table = entry.getHeader().getTableName();// => type
        final CanalEntry.RowChange change = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
        final List<CanalEntry.RowData> rowDataList = change.getRowDatasList();

        CanalEntry.EventType eventType = entry.getHeader().getEventType();
        final int esEventType = esAdapter.getEsEventType(eventType);


        EsRowDataArrayList esRowDataList = TotoroObjectPool.esRowDataArrayList();

        for (CanalEntry.RowData rowData : rowDataList) {
            List<CanalEntry.Column> columnList = esAdapter.getColumnList(esEventType, rowData);
            ElasticsearchMetadata.EsRowData esRowData = TotoroObjectPool.esRowData();
            EsColumnHashMap columnMap = TotoroObjectPool.esColumnHashMap();
            columnList.forEach(column -> columnMap.put(column.getName(), column.getValue()));
            esRowData.setRowData(columnMap);
            esRowData.setIdColumn(esAdapter.getEsIdColumn(database, table));//获取es对应的id Column
            esRowDataList.add(esRowData);
        }

        ElasticsearchMetadata.EsEntry esEntry = TotoroObjectPool.esEntry();

        esEntry.setIndex(database)
                .setType(table)
                .setEsRowDatas(esRowDataList)
                .setEventType(esEventType);

        return esEntry;
    }
 

@Override
public final void rowChangeHandle(CanalEntry.RowChange rowChange) {
    try {
        List<RowChangedData> changedData = changedDataParse(rowChange);
        if (!CommonsUtils.isEmpty(changedData)) doRowChangeHandle(changedData);
    } catch (RuntimeException e) {
        if (!exceptionHandle(e, false)) {
            throw e;
        } else {
            log.warn("canal instance: " + instanceName + " ignore handle exception", e);
        }
    }
}
 

private CanalEntry.RowChange parseRowChange(CanalEntry.Entry entry) {
    CanalEntry.RowChange rowChange = null;
    try {
        rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
    } catch (Exception e) {
        logger.error("ERROR ## parser of eromanga-event has an error , data:" + entry.toString());
    }
    return rowChange;
}
 

@Override
public int processFullPullerMessage(Map<String, List<IGenericMessage>> map, IGenericMessage obj) throws
        IOException {
    MysqlGenericMessage message = (MysqlGenericMessage) obj;
    CanalEntry.Entry entry = message.getEntry();

    CanalEntry.EventType eventType = entry.getHeader().getEventType();
    if (eventType != CanalEntry.EventType.INSERT) {
        //skip it.
        logger.info("Skipped a FULL_PULL_REQUESTS message which is not INSERT Type! :" + eventType.toString());
        return -1;
    }

    CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
    List<CanalEntry.RowData> dataList = rowChange.getRowDatasList();
    if (dataList.size() != 1) {
        throw new RuntimeException(String.format("解压FULL_PULL_REQUESTS 发现 %d 条bach数据，应该只有一条", dataList.size()));
    }

    //为了向下兼容，使用PULL_REMARK 保存dsName
    String dsName = null;
    String schema = null;
    String table = null;
    List<CanalEntry.Column> columns = dataList.get(0).getAfterColumnsList();
    for (CanalEntry.Column column : columns) {
        if (column.getName().equalsIgnoreCase("PULL_REMARK")) {
            dsName = column.getValue();
        } else if (column.getName().equalsIgnoreCase("SCHEMA_NAME")) {
            schema = column.getValue();
        } else if (column.getName().equalsIgnoreCase("TABLE_NAME")) {
            table = column.getValue();
        }
    }

    if (dsName == null || schema == null || table == null) {
        throw new RuntimeException("解压FULL_PULL_REQUESTS 发现 dsName 或 schema 或 table为空.");
    }

    if (!dsName.equalsIgnoreCase(dsInfo.getDbSourceName())) {
        logger.info("Skipped other datasource FULL_PULL_REQUESTS! : {}.{}.{}", dsName, schema, table);
        return -1;
    }

    logger.info(String.format("Get FULL_PULL_REQUESTS message : %s.%s.%s", dsName, schema, table));

    //单独发送一条 拉全量的消息
    List<IGenericMessage> subList = new ArrayList<>();
    subList.add(message);
    map.put(schema, subList);
    return 0;
}
 

@Override
    public int processHeartBeatMessage(Map<String, List<IGenericMessage>> map, IGenericMessage obj) throws
            IOException {
        MysqlGenericMessage message = (MysqlGenericMessage) obj;
        CanalEntry.Entry entry = message.getEntry();

        CanalEntry.EventType eventType = entry.getHeader().getEventType();
        if (eventType != CanalEntry.EventType.INSERT) {
            //skip it.
            logger.info("Skipped a DB_HEARTBEAT_MONITOR message which is not INSERT Type! :" + eventType.toString());
            return -1;
        }

        CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
        List<CanalEntry.RowData> dataList = rowChange.getRowDatasList();
        /**
         * 以前代码dataList.size() 必须是1条数据，发现生产中有奇怪数据，心跳表数据居然不止一个心跳数据，推测是 insert ... select ...
         * 这种情况我们不处理，打算直接将这个心跳包抛弃。
         */
//        if (dataList.size() != 1) {
//            throw new RuntimeException(String.format("DB_HEARTBEAT_MONITOR 发现 %d 条bach数据，应该只有一条", dataList.size()));
//        }
        if (dataList.size() != 1) {
            logger.error(String.format("Skipped a DB_HEARTBEAT_MONITOR message. DB_HEARTBEAT_MONITOR 发现 %d 条bach数据，应该只有一条, 语句是%s",
                    dataList.size(), rowChange.getSql()));
            return -1;
        }

        String dsName = null;
        String schemaName = null;
        String tableName = null;
        String packetJson = null;
        List<CanalEntry.Column> columns = dataList.get(0).getAfterColumnsList();
        for (CanalEntry.Column column : columns) {
            if (column.getName().equalsIgnoreCase("DS_NAME")) {
                dsName = column.getValue();
            } else if (column.getName().equalsIgnoreCase("SCHEMA_NAME")) {
                schemaName = column.getValue();
            } else if (column.getName().equalsIgnoreCase("TABLE_NAME")) {
                tableName = column.getValue();
            } else if (column.getName().equalsIgnoreCase("PACKET"))
                packetJson = column.getValue();
        }

        if (dsName == null || schemaName == null || tableName == null || packetJson == null) {
            throw new RuntimeException("DB_HEARTBEAT_MONITOR 发现 dsName 或 schema 或 table, 或 packetJson 为空.");
        }

        if (!dsName.equalsIgnoreCase(dsInfo.getDbSourceName())) {
            logger.info("Skipped other datasource HeartBeat! : {}.{}.{}", dsName, schemaName, tableName);
            return -1;
        }

        logger.debug(String.format("Get DB_HEARTBEAT_MONITOR message : %s.%s, packetJson: %s", schemaName, tableName, packetJson));
        if (packetJson.indexOf("checkpoint") >= 0) {
            logger.debug(String.format("Get DB_HEARTBEAT_MONITOR message, prepare set stat message. "));
            HeartBeatPacket packet = HeartBeatPacket.parse(packetJson);
            statMeter(schemaName, tableName, packet.getTime(), packet.getTxtime());
        }

        List<IGenericMessage> subList = map.get(schemaName);
        if (subList != null) {
            subList.add(message);
        } else {
            subList = new ArrayList<>();
            subList.add(message);
            map.put(schemaName, subList);
        }
        return 0;
    }
 

@Override
protected List<RowChangedData> changedDataParse(CanalEntry.RowChange rowChange) {
    return RowChangedData.build(rowChange, null);
}
 

/**
 * process the ddl event
 * @param rowChange rowChange
 */
protected void processDdl(CanalEntry.RowChange rowChange) {}
 

/**
 * 记录更新处理, 不断调用该方法, 数据更新处理完成, 通过{@link #finishMessageHandle()} 结束处理
 *
 * @see #finishMessageHandle()
 */
void rowChangeHandle(CanalEntry.RowChange rowChange);
 

/**
 * 更改记录的诗句转换成 {@link RowChangedData} list
 *
 * @param rowChange 更改的数据
 * @return 构建的{@link RowChangedData} list
 */
protected abstract List<RowChangedData> changedDataParse(CanalEntry.RowChange rowChange);