Java Code Examples for com.alibaba.otter.canal.protocol.CanalEntry#Entry

protected LogPosition buildLastPosition(CanalEntry.Entry entry) { // 初始化一下
    LogPosition logPosition = new LogPosition();
    EntryPosition position = new EntryPosition();
    position.setJournalName(entry.getHeader().getLogfileName());
    position.setPosition(entry.getHeader().getLogfileOffset());
    position.setTimestamp(entry.getHeader().getExecuteTime());
    // add serverId at 2016-06-28
    position.setServerId(entry.getHeader().getServerId());
    // set gtid
    position.setGtid(entry.getHeader().getGtid());

    logPosition.setPostion(position);

    LogIdentity identity = new LogIdentity(runningInfo.getAddress(), -1L);
    logPosition.setIdentity(identity);
    return logPosition;
}
 

/**
 * 消费当前消息
 */
private void consumerMessage(Message message) {
    log.debug("canal instance: " + handle.instanceName() + " get message entry size " + message.getEntries().size());
    try {
        for (CanalEntry.Entry e : message.getEntries()) {
            if (e.getEntryType() != CanalEntry.EntryType.ROWDATA || !e.hasStoreValue()) continue;
            CanalEntry.Header header = e.getHeader();
            if (header.getExecuteTime() < startRtTime
                    || header.getEventType().getNumber() > CanalEntry.EventType.DELETE_VALUE
                    || !handle.startHandle(header)) continue;
            try {
                CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(e.getStoreValue());
                if (rowChange.getIsDdl()) continue;
                handle.rowChangeHandle(rowChange);
            } catch (InvalidProtocolBufferException e1) {
                log.error("canal instance: " + handle.instanceName() + " parse store value have exception: ", e1);
            }
        }
        handle.ack(message.getId());
    } finally {
        handle.finishMessageHandle();
    }
}
 

@Override
public void onEvent(MessageEvent event) throws Exception {
    try {
        if (event.isNeedDmlParse()) {
            int eventType = event.getEvent().getHeader().getType();
            CanalEntry.Entry entry = null;
            switch (eventType) {
                case LogEvent.ROWS_QUERY_LOG_EVENT:
                    entry = logEventConvert.parse(event.getEvent(), false);
                    break;
                default:
                    // 单独解析dml事件
                    entry = logEventConvert.parseRowsEvent((RowsLogEvent) event.getEvent(), event.getTable());
            }

            event.setEntry(entry);
        }
    } catch (Throwable e) {
        exception = new CanalParseException(e);
        throw exception;
    }
}
 

protected CanalEntry.Entry parseAndProfilingIfNecessary(EVENT bod) throws Exception {
    long startTs = -1;
    boolean enabled = getProfilingEnabled();
    if (enabled) {
        startTs = System.currentTimeMillis();
    }
    CanalEntry.Entry event = binlogParser.parse(bod);
    if (enabled) {
        this.parsingInterval = System.currentTimeMillis() - startTs;
    }

    if (parsedEventCount.incrementAndGet() < 0) {
        parsedEventCount.set(0);
    }
    return event;
}
 

public AbstractEventParser(){
    // 初始化一下
    transactionBuffer = new EventTransactionBuffer(new TransactionFlushCallback() {

        public void flush(List<CanalEntry.Entry> transaction) throws InterruptedException {
            boolean successed = consumeTheEventAndProfilingIfNecessary(transaction);
            if (!running) {
                return;
            }

            if (!successed) {
                throw new CanalParseException("consume failed!");
            }

            LogPosition position = buildLastTransactionPosition(transaction);
            if (position != null) { // 可能position为空
                logPositionManager.persistLogPosition(AbstractEventParser.this.destination, position);
            }
        }
    });
}
 

protected boolean consumeTheEventAndProfilingIfNecessary(List<CanalEntry.Entry> entrys) throws CanalSinkException,
                                                                                       InterruptedException {
    long startTs = -1;
    boolean enabled = getProfilingEnabled();
    if (enabled) {
        startTs = System.currentTimeMillis();
    }

    boolean result = eventSink.sink(entrys, (runningInfo == null) ? null : runningInfo.getAddress(), destination);

    if (enabled) {
        this.processingInterval = System.currentTimeMillis() - startTs;
    }

    if (consumedEventCount.incrementAndGet() < 0) {
        consumedEventCount.set(0);
    }

    return result;
}
 

private void flush() throws InterruptedException {
    long start = this.flushSequence.get() + 1;
    long end = this.putSequence.get();

    if (start <= end) {
        List<CanalEntry.Entry> transaction = new ArrayList<CanalEntry.Entry>();
        for (long next = start; next <= end; next++) {
            transaction.add(this.entries[getIndex(next)]);
        }

        flushCallback.flush(transaction);
        flushSequence.set(end);// flush成功后，更新flush位置
    }
}
 

@Override
public List<IGenericMessage> unwrapMessages(byte[] data) throws IOException {
    // canal-1.0.24
    //List<IGenericMessage> list = new ArrayList<>();
    //
    //CanalPacket.Messages cMessage = CanalPacket.Messages.parseFrom(data);
    //List<ByteString> strings = cMessage.getMessagesList();
    //for (ByteString str : strings) {
    //    CanalEntry.Entry ent = CanalEntry.Entry.parseFrom(str);
    //    MysqlGenericMessage message = new MysqlGenericMessage(ent);
    //
    //    list.add(message);
    //}
    // canal-1.1.4
    Message message = CanalMessageDeserializer.deserializer(data);
    List<IGenericMessage> list = new ArrayList<>();
    for (CanalEntry.Entry entry : message.getEntries()) {
        if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
            logger.debug("the entry type is transaction begin or transaction end.");
            continue;
        }
        //处理带正则的表名
        String schemaName = entry.getHeader().getSchemaName();
        String tableName = entry.getHeader().getTableName();
        String localTable = getLocalTable(schemaName, tableName);
        logger.debug("收到表数据{}.{}(基准表名{})数据", schemaName, tableName, localTable);
        if (!StringUtils.equals(tableName, localTable)) {
            String finalTable = StringUtils.join(new String[]{localTable, tableName}, ".");
            CanalEntry.Header header = CanalEntry.Header.newBuilder(entry.getHeader()).setTableName(finalTable).build();
            entry = CanalEntry.Entry.newBuilder(entry).setHeader(header).build();
            logger.debug("重新生成entry");
        }
        MysqlGenericMessage mysqlGenericMessage = new MysqlGenericMessage(entry);
        list.add(mysqlGenericMessage);
    }
    return list;
}
 

@Override
protected void distributeEvent(Message message) {
    List<CanalEntry.Entry> entries = message.getEntries();
    for (CanalEntry.Entry entry : entries) {
        //ignore the transaction operations
        List<CanalEntry.EntryType> ignoreEntryTypes = getIgnoreEntryTypes();
        if (ignoreEntryTypes != null
                && ignoreEntryTypes.stream().anyMatch(t -> entry.getEntryType() == t)) {
            continue;
        }
        CanalEntry.RowChange rowChange;
        try {
            rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
        } catch (Exception e) {
            throw new CanalClientException("ERROR ## parser of event has an error , data:" + entry.toString(),
                    e);
        }
        //ignore the ddl operation
        if (rowChange.hasIsDdl() && rowChange.getIsDdl()) {
            processDdl(rowChange);
            continue;
        }
        for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
            //distribute to listener interfaces
            distributeByImpl(rowChange.getEventType(), rowData);
            //distribute to annotation listener interfaces
            distributeByAnnotation(destination,
                    entry.getHeader().getSchemaName(),
                    entry.getHeader().getTableName(),
                    rowChange.getEventType(),
                    rowData);
        }
    }
}
 

public void add(CanalEntry.Entry entry) throws InterruptedException {
    switch (entry.getEntryType()) {
        case TRANSACTIONBEGIN:
            flush();// 刷新上一次的数据
            put(entry);
            break;
        case TRANSACTIONEND:
            put(entry);
            flush();
            break;
        case ROWDATA:
            put(entry);
            // 针对非DML的数据，直接输出，不进行buffer控制
            EventType eventType = entry.getHeader().getEventType();
            if (eventType != null && !isDml(eventType)) {
                flush();
            }
            break;
        case HEARTBEAT:
            // master过来的heartbeat，说明binlog已经读完了，是idle状态
            put(entry);
            flush();
            break;
        default:
            break;
    }
}
 

private void put(CanalEntry.Entry data) throws InterruptedException {
    // 首先检查是否有空位
    if (checkFreeSlotAt(putSequence.get() + 1)) {
        long current = putSequence.get();
        long next = current + 1;

        // 先写数据，再更新对应的cursor,并发度高的情况，putSequence会被get请求可见，拿出了ringbuffer中的老的Entry值
        entries[getIndex(next)] = data;
        putSequence.set(next);
    } else {
        flush();// buffer区满了，刷新一下
        put(data);// 继续加一下新数据
    }
}
 

public void add(CanalEntry.Entry entry) throws InterruptedException {
    switch (entry.getEntryType()) {
        case TRANSACTIONBEGIN:
            flush();// 刷新上一次的数据
            put(entry);
            break;
        case TRANSACTIONEND:
            put(entry);
            flush();
            break;
        case ROWDATA:
            put(entry);
            // 针对非DML的数据，直接输出，不进行buffer控制
            EventType eventType = entry.getHeader().getEventType();
            if (eventType != null && !isDml(eventType)) {
                flush();
            }
            break;
        case HEARTBEAT:
            // master过来的heartbeat，说明binlog已经读完了，是idle状态
            put(entry);
            flush();
            break;
        default:
            break;
    }
}
 

protected LogPosition buildLastTransactionPosition(List<CanalEntry.Entry> entries) { // 初始化一下
    for (int i = entries.size() - 1; i > 0; i--) {
        CanalEntry.Entry entry = entries.get(i);
        if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {// 尽量记录一个事务做为position
            return buildLastPosition(entry);
        }
    }

    return null;
}
 

private CanalEntry.RowChange parseRowChange(CanalEntry.Entry entry) {
    CanalEntry.RowChange rowChange = null;
    try {
        rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
    } catch (Exception e) {
        logger.error("ERROR ## parser of eromanga-event has an error , data:" + entry.toString());
    }
    return rowChange;
}
 

public boolean filter(CanalEntry.Entry entry) throws CanalFilterException {
    if (StringUtils.isEmpty(expression)) {
        return true;
    }

    Map<String, Object> env = new HashMap<String, Object>();
    env.put(ROOT_KEY, entry);
    return (Boolean) AviatorEvaluator.execute(expression, env);
}
 

private void put(CanalEntry.Entry data) throws InterruptedException {
    // 首先检查是否有空位
    if (checkFreeSlotAt(putSequence.get() + 1)) {
        long current = putSequence.get();
        long next = current + 1;

        // 先写数据，再更新对应的cursor,并发度高的情况，putSequence会被get请求可见，拿出了ringbuffer中的老的Entry值
        entries[getIndex(next)] = data;
        putSequence.set(next);
    } else {
        flush();// buffer区满了，刷新一下
        put(data);// 继续加一下新数据
    }
}
 

@Override
public int processFullPullerMessage(Map<String, List<IGenericMessage>> map, IGenericMessage obj) throws
        IOException {
    MysqlGenericMessage message = (MysqlGenericMessage) obj;
    CanalEntry.Entry entry = message.getEntry();

    CanalEntry.EventType eventType = entry.getHeader().getEventType();
    if (eventType != CanalEntry.EventType.INSERT) {
        //skip it.
        logger.info("Skipped a FULL_PULL_REQUESTS message which is not INSERT Type! :" + eventType.toString());
        return -1;
    }

    CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
    List<CanalEntry.RowData> dataList = rowChange.getRowDatasList();
    if (dataList.size() != 1) {
        throw new RuntimeException(String.format("解压FULL_PULL_REQUESTS 发现 %d 条bach数据，应该只有一条", dataList.size()));
    }

    //为了向下兼容，使用PULL_REMARK 保存dsName
    String dsName = null;
    String schema = null;
    String table = null;
    List<CanalEntry.Column> columns = dataList.get(0).getAfterColumnsList();
    for (CanalEntry.Column column : columns) {
        if (column.getName().equalsIgnoreCase("PULL_REMARK")) {
            dsName = column.getValue();
        } else if (column.getName().equalsIgnoreCase("SCHEMA_NAME")) {
            schema = column.getValue();
        } else if (column.getName().equalsIgnoreCase("TABLE_NAME")) {
            table = column.getValue();
        }
    }

    if (dsName == null || schema == null || table == null) {
        throw new RuntimeException("解压FULL_PULL_REQUESTS 发现 dsName 或 schema 或 table为空.");
    }

    if (!dsName.equalsIgnoreCase(dsInfo.getDbSourceName())) {
        logger.info("Skipped other datasource FULL_PULL_REQUESTS! : {}.{}.{}", dsName, schema, table);
        return -1;
    }

    logger.info(String.format("Get FULL_PULL_REQUESTS message : %s.%s.%s", dsName, schema, table));

    //单独发送一条 拉全量的消息
    List<IGenericMessage> subList = new ArrayList<>();
    subList.add(message);
    map.put(schema, subList);
    return 0;
}
 

private String getSchemaNameAndTableName(CanalEntry.Entry entry) {
    return entry.getHeader().getSchemaName() + "." + entry.getHeader().getTableName();
}
 

public boolean sink(List<CanalEntry.Entry> entrys, InetSocketAddress remoteAddress, String destination)
                                                                                                       throws CanalSinkException,
                                                                                                       InterruptedException {
    return sinkData(entrys, remoteAddress);
}
 

CanalEntry.Entry parse(T event, boolean isSeek) throws CanalParseException;