com.alibaba.otter.canal.protocol.Message#getEntries

Source File: BaseCanalClientTest.java From canal-1.1.3 with Apache License 2.0

6 votes

protected void printSummary(Message message, long batchId, int size) {
    long memsize = 0;
    for (Entry entry : message.getEntries()) {
        memsize += entry.getHeader().getEventLength();
    }

    String startPosition = null;
    String endPosition = null;
    if (!CollectionUtils.isEmpty(message.getEntries())) {
        startPosition = buildPositionForDump(message.getEntries().get(0));
        endPosition = buildPositionForDump(message.getEntries().get(message.getEntries().size() - 1));
    }

    SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
    logger.info(context_format, new Object[] { batchId, size, memsize, format.format(new Date()), startPosition,
            endPosition });
}

Source File: CanalReaderMessageDumper.java From DataLink with Apache License 2.0

6 votes

public static void dumpMessages(Message message, long batchId, int size) {
    long memsize = 0;
    for (CanalEntry.Entry entry : message.getEntries()) {
        memsize += entry.getHeader().getEventLength();
    }

    String startPosition = null;
    String endPosition = null;
    if (message.getEntries()!=null && message.getEntries().size()>0) {
        startPosition = buildPositionForDump(message.getEntries()
                .get(0));
        endPosition = buildPositionForDump(message.getEntries().get(
                message.getEntries().size() - 1));
    }

    SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
    logger.info(context_format, new Object[] { batchId, size, memsize,
            format.format(new Date()), startPosition, endPosition });
}

Source File: SelectorTask.java From canal-elasticsearch with Apache License 2.0

6 votes

private void printSummary(Message message, long batchId, int size) {
    long memsize = 0;
    for (CanalEntry.Entry entry : message.getEntries()) {
        memsize += entry.getHeader().getEventLength();
    }

    String startPosition = null;
    String endPosition = null;
    if (!CollectionUtils.isEmpty(message.getEntries())) {
        startPosition = buildPositionForDump(message.getEntries().get(0));
        endPosition = buildPositionForDump(message.getEntries().get(message.getEntries().size() - 1));
    }

    logger.info(context_format, new Object[]{batchId, size, memsize, format.format(new Date()), startPosition,
            endPosition});
}

Source File: CanalExecutor.java From search-commons with Apache License 2.0

6 votes

/**
 * 消费当前消息
 */
private void consumerMessage(Message message) {
    log.debug("canal instance: " + handle.instanceName() + " get message entry size " + message.getEntries().size());
    try {
        for (CanalEntry.Entry e : message.getEntries()) {
            if (e.getEntryType() != CanalEntry.EntryType.ROWDATA || !e.hasStoreValue()) continue;
            CanalEntry.Header header = e.getHeader();
            if (header.getExecuteTime() < startRtTime
                    || header.getEventType().getNumber() > CanalEntry.EventType.DELETE_VALUE
                    || !handle.startHandle(header)) continue;
            try {
                CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(e.getStoreValue());
                if (rowChange.getIsDdl()) continue;
                handle.rowChangeHandle(rowChange);
            } catch (InvalidProtocolBufferException e1) {
                log.error("canal instance: " + handle.instanceName() + " parse store value have exception: ", e1);
            }
        }
        handle.ack(message.getId());
    } finally {
        handle.finishMessageHandle();
    }
}

Source File: BaseCanalClientTest.java From canal with Apache License 2.0

6 votes

protected void printSummary(Message message, long batchId, int size) {
    long memsize = 0;
    for (Entry entry : message.getEntries()) {
        memsize += entry.getHeader().getEventLength();
    }

    String startPosition = null;
    String endPosition = null;
    if (!CollectionUtils.isEmpty(message.getEntries())) {
        startPosition = buildPositionForDump(message.getEntries().get(0));
        endPosition = buildPositionForDump(message.getEntries().get(message.getEntries().size() - 1));
    }

    SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
    logger.info(context_format, new Object[] { batchId, size, memsize, format.format(new Date()), startPosition,
            endPosition });
}

Source File: AbstractBasicMessageTransponder.java From spring-boot-starter-canal with MIT License

5 votes

@Override
protected void distributeEvent(Message message) {
    List<CanalEntry.Entry> entries = message.getEntries();
    for (CanalEntry.Entry entry : entries) {
        //ignore the transaction operations
        List<CanalEntry.EntryType> ignoreEntryTypes = getIgnoreEntryTypes();
        if (ignoreEntryTypes != null
                && ignoreEntryTypes.stream().anyMatch(t -> entry.getEntryType() == t)) {
            continue;
        }
        CanalEntry.RowChange rowChange;
        try {
            rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
        } catch (Exception e) {
            throw new CanalClientException("ERROR ## parser of event has an error , data:" + entry.toString(),
                    e);
        }
        //ignore the ddl operation
        if (rowChange.hasIsDdl() && rowChange.getIsDdl()) {
            processDdl(rowChange);
            continue;
        }
        for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
            //distribute to listener interfaces
            distributeByImpl(rowChange.getEventType(), rowData);
            //distribute to annotation listener interfaces
            distributeByAnnotation(destination,
                    entry.getHeader().getSchemaName(),
                    entry.getHeader().getTableName(),
                    rowChange.getEventType(),
                    rowData);
        }
    }
}

Source File: MysqlMessageProcessor.java From DBus with Apache License 2.0

5 votes

@Override
public List<IGenericMessage> unwrapMessages(byte[] data) throws IOException {
    // canal-1.0.24
    //List<IGenericMessage> list = new ArrayList<>();
    //
    //CanalPacket.Messages cMessage = CanalPacket.Messages.parseFrom(data);
    //List<ByteString> strings = cMessage.getMessagesList();
    //for (ByteString str : strings) {
    //    CanalEntry.Entry ent = CanalEntry.Entry.parseFrom(str);
    //    MysqlGenericMessage message = new MysqlGenericMessage(ent);
    //
    //    list.add(message);
    //}
    // canal-1.1.4
    Message message = CanalMessageDeserializer.deserializer(data);
    List<IGenericMessage> list = new ArrayList<>();
    for (CanalEntry.Entry entry : message.getEntries()) {
        if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
            logger.debug("the entry type is transaction begin or transaction end.");
            continue;
        }
        //处理带正则的表名
        String schemaName = entry.getHeader().getSchemaName();
        String tableName = entry.getHeader().getTableName();
        String localTable = getLocalTable(schemaName, tableName);
        logger.debug("收到表数据{}.{}(基准表名{})数据", schemaName, tableName, localTable);
        if (!StringUtils.equals(tableName, localTable)) {
            String finalTable = StringUtils.join(new String[]{localTable, tableName}, ".");
            CanalEntry.Header header = CanalEntry.Header.newBuilder(entry.getHeader()).setTableName(finalTable).build();
            entry = CanalEntry.Entry.newBuilder(entry).setHeader(header).build();
            logger.debug("重新生成entry");
        }
        MysqlGenericMessage mysqlGenericMessage = new MysqlGenericMessage(entry);
        list.add(mysqlGenericMessage);
    }
    return list;
}

Source File: TotoroTransForm.java From canal-elasticsearch with Apache License 2.0

5 votes

@Override
public ElasticsearchMetadata trans(Message message) {
    List<CanalEntry.Entry> entries = message.getEntries();
    ElasticsearchMetadata elasticsearchMetadata = TotoroObjectPool.esMetadata();
    elasticsearchMetadata.setBatchId(message.getId());
    elasticsearchMetadata.setTransForm(this);
    if (entries != null && entries.size() > 0) {
        EsEntryArrayList esEntryList = TotoroObjectPool.esEntryArrayList();

        entries.forEach(entry -> {
            if (messageFilterChain.filter(entry)) {
                try {
                    ElasticsearchMetadata.EsEntry esEntry = getElasticsearchMetadata(entry);
                    esEntryList.add(esEntry);
                } catch (InvalidProtocolBufferException e) {
                    e.printStackTrace();
                }
            }
        });
        elasticsearchMetadata.setEsEntries(esEntryList);
    }

    logger.info("Trans form complete message id =====> {}", message.getId());

    if (logger.isDebugEnabled()) {
        logger.debug("Trans form complete elasticsearch metadata  =====> {}", elasticsearchMetadata.toString());
    }
    return elasticsearchMetadata;
}

Source File: CanalReader.java From DataLink with Apache License 2.0

3 votes

/**
 * <p>
 * 设计之初，firstEntryTime的获取方式为：message.getEntries().get(0).getHeader().getExecuteTime()
 * 这样获取不够精确，在有大事务或者长事务时，按此取到的firstEntryTime和事务最终提交的时间会有比较大的差距，
 * 此时，我们通过firstEntryTime去计算延迟时间的时候，算出来的值就会比较大而触发报警。
 * <p>
 * 为什么按此取到的firstEntryTime和事务最终提交的时间会有比较大的差距呢？是因为mysql对于每个操作事件，
 * 以其真正发生的时间作为ExecuteTime，不会因为在一个事务里，而以提交时间为准，如下所示：
 * TRANSACTIONBEGIN   2018-01-01 10:00:00
 * INSERT             2018-01-01 10:01:00
 * UPDATE             2018-01-01 10:02:00
 * TRANSACTIONEND     2018-01-01 10:03:00
 * </p>
 * <p>
 * <p>
 * 而应该以第一个TRANSACTIONEND的时间为准，因为TRANSACTIONEND对应的是最终的commit，代表了实际发生
 * 如果此批次没有TRANSACTIONEND，说明该批次所有数据都在一个事务里，则以最后一条的时间为准
 * 这么改，主要是解决了长事务的问题，此处所说的长事务，是指一个事务从begin到end的跨度比较长，但中间不一定
 * 有非常多的操做，并且这些操作的间隔时间比较长。
 * </p>
 *
 * @param message
 * @return
 */
private long calcFirstEntryTime(Message message) {
    CanalEntry.Entry lastEntry = null;
    for (CanalEntry.Entry entry : message.getEntries()) {
        lastEntry = entry;
        if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
            break;
        }
    }

    return lastEntry.getHeader().getExecuteTime();
}

Java Code Examples for com.alibaba.otter.canal.protocol.Message#getEntries()