com.alibaba.otter.canal.protocol.Message Java Exaples

Source File: BaseCanalClientTest.java From canal-1.1.3 with Apache License 2.0

6 votes

protected void printSummary(Message message, long batchId, int size) {
    long memsize = 0;
    for (Entry entry : message.getEntries()) {
        memsize += entry.getHeader().getEventLength();
    }

    String startPosition = null;
    String endPosition = null;
    if (!CollectionUtils.isEmpty(message.getEntries())) {
        startPosition = buildPositionForDump(message.getEntries().get(0));
        endPosition = buildPositionForDump(message.getEntries().get(message.getEntries().size() - 1));
    }

    SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
    logger.info(context_format, new Object[] { batchId, size, memsize, format.format(new Date()), startPosition,
            endPosition });
}

Source File: KafkaOffsetCanalConnector.java From canal-1.1.3 with Apache License 2.0

6 votes

/**
 * 获取Kafka消息，不确认
 *
 * @param timeout
 * @param unit
 * @param offset  消息偏移地址（-1为不偏移）
 * @return
 * @throws CanalClientException
 */
public List<KafkaMessage> getListWithoutAck(Long timeout, TimeUnit unit, long offset) throws CanalClientException {
    waitClientRunning();
    if (!running) {
        return Lists.newArrayList();
    }

    if (offset > -1) {
        TopicPartition tp = new TopicPartition(topic, partition == null ? 0 : partition);
        kafkaConsumer.seek(tp, offset);
    }

    ConsumerRecords<String, Message> records = kafkaConsumer.poll(unit.toMillis(timeout));

    if (!records.isEmpty()) {
        List<KafkaMessage> messages = new ArrayList<>();
        for (ConsumerRecord<String, Message> record : records) {
            KafkaMessage message = new KafkaMessage(record.value(), record.offset());
            messages.add(message);
        }
        return messages;
    }
    return Lists.newArrayList();
}

Source File: KafkaOffsetCanalConnector.java From canal with Apache License 2.0

6 votes

/**
 * 获取Kafka消息，不确认
 *
 * @param timeout
 * @param unit
 * @param offset  消息偏移地址（-1为不偏移）
 * @return
 * @throws CanalClientException
 */
public List<KafkaMessage> getListWithoutAck(Long timeout, TimeUnit unit, long offset) throws CanalClientException {
    waitClientRunning();
    if (!running) {
        return Lists.newArrayList();
    }

    if (offset > -1) {
        TopicPartition tp = new TopicPartition(topic, partition == null ? 0 : partition);
        kafkaConsumer.seek(tp, offset);
    }

    ConsumerRecords<String, Message> records = kafkaConsumer.poll(unit.toMillis(timeout));

    if (!records.isEmpty()) {
        List<KafkaMessage> messages = new ArrayList<>();
        for (ConsumerRecord<String, Message> record : records) {
            KafkaMessage message = new KafkaMessage(record.value(), record.offset());
            messages.add(message);
        }
        return messages;
    }
    return Lists.newArrayList();
}

Source File: ClusterCanalConnector.java From canal-1.1.3 with Apache License 2.0

6 votes

public Message getWithoutAck(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
    int times = 0;
    while (times < retryTimes) {
        try {
            Message msg = currentConnector.getWithoutAck(batchSize, timeout, unit);
            return msg;
        } catch (Throwable t) {
            logger.warn(String.format("something goes wrong when getWithoutAck data from server:%s",
                currentConnector != null ? currentConnector.getAddress() : "null"), t);
            times++;
            restart();
            logger.info("restart the connector for next round retry.");
        }
    }
    throw new CanalClientException("failed to fetch the data after " + times + " times retry");
}

Source File: ClusterCanalConnector.java From canal-1.1.3 with Apache License 2.0

6 votes

public Message getWithoutAck(int batchSize) throws CanalClientException {
    int times = 0;
    while (times < retryTimes) {
        try {
            Message msg = currentConnector.getWithoutAck(batchSize);
            return msg;
        } catch (Throwable t) {
            logger.warn(String.format("something goes wrong when getWithoutAck data from server:%s",
                currentConnector != null ? currentConnector.getAddress() : "null"), t);
            times++;
            restart();
            logger.info("restart the connector for next round retry.");
        }
    }
    throw new CanalClientException("failed to fetch the data after " + times + " times retry");
}

Source File: ClusterCanalConnector.java From canal-1.1.3 with Apache License 2.0

6 votes

public Message get(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
    int times = 0;
    while (times < retryTimes) {
        try {
            Message msg = currentConnector.get(batchSize, timeout, unit);
            return msg;
        } catch (Throwable t) {
            logger.warn(String.format("something goes wrong when getting data from server:%s",
                currentConnector != null ? currentConnector.getAddress() : "null"), t);
            times++;
            restart();
            logger.info("restart the connector for next round retry.");
        }
    }
    throw new CanalClientException("failed to fetch the data after " + times + " times retry");
}

Source File: ClusterCanalConnector.java From canal-1.1.3 with Apache License 2.0

6 votes

public Message get(int batchSize) throws CanalClientException {
    int times = 0;
    while (times < retryTimes) {
        try {
            Message msg = currentConnector.get(batchSize);
            return msg;
        } catch (Throwable t) {
            logger.warn(String.format("something goes wrong when getting data from server:%s",
                currentConnector != null ? currentConnector.getAddress() : "null"), t);
            times++;
            restart();
            logger.info("restart the connector for next round retry.");
        }
    }
    throw new CanalClientException("failed to fetch the data after " + times + " times retry");
}

Source File: RocketMQCanalConnector.java From canal with Apache License 2.0

6 votes

@Override
public List<Message> getListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException {
    try {
        if (this.lastGetBatchMessage != null) {
            throw new CanalClientException("mq get/ack not support concurrent & async ack");
        }

        ConsumerBatchMessage batchMessage = messageBlockingQueue.poll(timeout, unit);
        if (batchMessage != null) {
            this.lastGetBatchMessage = batchMessage;
            return batchMessage.getData();
        }
    } catch (InterruptedException ex) {
        logger.warn("Get message timeout", ex);
        throw new CanalClientException("Failed to fetch the data after: " + timeout);
    }
    return Lists.newArrayList();
}

Source File: BaseCanalServerWithEmbededTest.java From canal-1.1.3 with Apache License 2.0

6 votes

@Test
public void testGet() {
    int maxEmptyCount = 10;
    int emptyCount = 0;
    int totalCount = 0;
    server.subscribe(clientIdentity);
    while (emptyCount < maxEmptyCount) {
        Message message = server.get(clientIdentity, 11);
        if (CollectionUtils.isEmpty(message.getEntries())) {
            emptyCount++;
            try {
                Thread.sleep(emptyCount * 300L);
            } catch (InterruptedException e) {
                Assert.fail();
            }

            System.out.println("empty count : " + emptyCount);
        } else {
            emptyCount = 0;
            totalCount += message.getEntries().size();
        }
    }

    System.out.println("!!!!!! testGet totalCount : " + totalCount);
    server.unsubscribe(clientIdentity);
}

Source File: CanalReaderMessageDumper.java From DataLink with Apache License 2.0

6 votes

public static void dumpMessages(Message message, long batchId, int size) {
    long memsize = 0;
    for (CanalEntry.Entry entry : message.getEntries()) {
        memsize += entry.getHeader().getEventLength();
    }

    String startPosition = null;
    String endPosition = null;
    if (message.getEntries()!=null && message.getEntries().size()>0) {
        startPosition = buildPositionForDump(message.getEntries()
                .get(0));
        endPosition = buildPositionForDump(message.getEntries().get(
                message.getEntries().size() - 1));
    }

    SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
    logger.info(context_format, new Object[] { batchId, size, memsize,
            format.format(new Date()), startPosition, endPosition });
}

Source File: CanalReader.java From DataLink with Apache License 2.0

6 votes

void dump() {
    if (logger.isInfoEnabled()) {
        Message message = (Message) latestReaderMsg.getMetaData().get(MESSAGE_KEY);
        String startPosition = null;
        String endPosition = null;
        if (!CollectionUtils.isEmpty(message.getEntries())) {
            startPosition = CanalReaderRecordsDumper.buildPositionForDump(message.getEntries().get(0));
            endPosition = CanalReaderRecordsDumper.buildPositionForDump(message.getEntries().get(message.getEntries().size() - 1));
        }

        //dump for troubleshoot problems
        CanalReaderMessageDumper.dumpMessages(message, latestReaderMsg.getBatchId(), message.getEntries().size());
        CanalReaderRecordsDumper.dumpRecords(latestReaderMsg.getBatchId(), latestReaderMsg.getRecords(),
                startPosition, endPosition, message.getEntries().size(), parameter.isDumpDetail());
    }
}

Source File: SelectorTask.java From canal-elasticsearch with Apache License 2.0

6 votes

private void printSummary(Message message, long batchId, int size) {
    long memsize = 0;
    for (CanalEntry.Entry entry : message.getEntries()) {
        memsize += entry.getHeader().getEventLength();
    }

    String startPosition = null;
    String endPosition = null;
    if (!CollectionUtils.isEmpty(message.getEntries())) {
        startPosition = buildPositionForDump(message.getEntries().get(0));
        endPosition = buildPositionForDump(message.getEntries().get(message.getEntries().size() - 1));
    }

    logger.info(context_format, new Object[]{batchId, size, memsize, format.format(new Date()), startPosition,
            endPosition});
}

Source File: ClusterCanalConnector.java From canal with Apache License 2.0

6 votes

public Message get(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
    int times = 0;
    while (times < retryTimes) {
        try {
            Message msg = currentConnector.get(batchSize, timeout, unit);
            return msg;
        } catch (Throwable t) {
            logger.warn(String.format("something goes wrong when getting data from server:%s",
                currentConnector != null ? currentConnector.getAddress() : "null"), t);
            times++;
            restart();
            logger.info("restart the connector for next round retry.");
        }
    }
    throw new CanalClientException("failed to fetch the data after " + times + " times retry");
}

Source File: ClusterCanalConnector.java From canal with Apache License 2.0

6 votes

public Message getWithoutAck(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
    int times = 0;
    while (times < retryTimes) {
        try {
            Message msg = currentConnector.getWithoutAck(batchSize, timeout, unit);
            return msg;
        } catch (Throwable t) {
            logger.warn(String.format("something goes wrong when getWithoutAck data from server:%s",
                currentConnector != null ? currentConnector.getAddress() : "null"), t);
            times++;
            restart();
            logger.info("restart the connector for next round retry.");
        }
    }
    throw new CanalClientException("failed to fetch the data after " + times + " times retry");
}

Source File: CanalExecutor.java From search-commons with Apache License 2.0

6 votes

/**
 * 消费当前消息
 */
private void consumerMessage(Message message) {
    log.debug("canal instance: " + handle.instanceName() + " get message entry size " + message.getEntries().size());
    try {
        for (CanalEntry.Entry e : message.getEntries()) {
            if (e.getEntryType() != CanalEntry.EntryType.ROWDATA || !e.hasStoreValue()) continue;
            CanalEntry.Header header = e.getHeader();
            if (header.getExecuteTime() < startRtTime
                    || header.getEventType().getNumber() > CanalEntry.EventType.DELETE_VALUE
                    || !handle.startHandle(header)) continue;
            try {
                CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(e.getStoreValue());
                if (rowChange.getIsDdl()) continue;
                handle.rowChangeHandle(rowChange);
            } catch (InvalidProtocolBufferException e1) {
                log.error("canal instance: " + handle.instanceName() + " parse store value have exception: ", e1);
            }
        }
        handle.ack(message.getId());
    } finally {
        handle.finishMessageHandle();
    }
}

Source File: CanalKafkaProducer.java From canal-1.1.3 with Apache License 2.0

5 votes

@Override
public void init(MQProperties kafkaProperties) {
    this.kafkaProperties = kafkaProperties;
    Properties properties = new Properties();
    properties.put("bootstrap.servers", kafkaProperties.getServers());
    properties.put("acks", kafkaProperties.getAcks());
    properties.put("compression.type", kafkaProperties.getCompressionType());
    properties.put("batch.size", kafkaProperties.getBatchSize());
    properties.put("linger.ms", kafkaProperties.getLingerMs());
    properties.put("max.request.size", kafkaProperties.getMaxRequestSize());
    properties.put("buffer.memory", kafkaProperties.getBufferMemory());
    properties.put("key.serializer", StringSerializer.class.getName());
    properties.put("max.in.flight.requests.per.connection", 1);

    if (!kafkaProperties.getProperties().isEmpty()) {
        properties.putAll(kafkaProperties.getProperties());
    }

    if (kafkaProperties.getTransaction()) {
        properties.put("transactional.id", "canal-transactional-id");
    } else {
        properties.put("retries", kafkaProperties.getRetries());
    }
    if (!kafkaProperties.getFlatMessage()) {
        properties.put("value.serializer", MessageSerializer.class.getName());
        producer = new KafkaProducer<String, Message>(properties);
    } else {
        properties.put("value.serializer", StringSerializer.class.getName());
        producer2 = new KafkaProducer<String, String>(properties);
    }
    if (kafkaProperties.getTransaction()) {
        if (!kafkaProperties.getFlatMessage()) {
            producer.initTransactions();
        } else {
            producer2.initTransactions();
        }
    }
}

Source File: CanalMessageDeserializer.java From canal-1.1.3 with Apache License 2.0

5 votes

public static Message deserializer(byte[] data, boolean lazyParseEntry) {
    try {
        if (data == null) {
            return null;
        } else {
            CanalPacket.Packet p = CanalPacket.Packet.parseFrom(data);
            switch (p.getType()) {
                case MESSAGES: {
                    if (!p.getCompression().equals(Compression.NONE)
                        && !p.getCompression().equals(Compression.COMPRESSIONCOMPATIBLEPROTO2)) {
                        throw new CanalClientException("compression is not supported in this connector");
                    }

                    CanalPacket.Messages messages = CanalPacket.Messages.parseFrom(p.getBody());
                    Message result = new Message(messages.getBatchId());
                    if (lazyParseEntry) {
                        // byteString
                        result.setRawEntries(messages.getMessagesList());
                        result.setRaw(true);
                    } else {
                        for (ByteString byteString : messages.getMessagesList()) {
                            result.addEntry(CanalEntry.Entry.parseFrom(byteString));
                        }
                        result.setRaw(false);
                    }
                    return result;
                }
                case ACK: {
                    Ack ack = Ack.parseFrom(p.getBody());
                    throw new CanalClientException("something goes wrong with reason: " + ack.getErrorMessage());
                }
                default: {
                    throw new CanalClientException("unexpected packet type: " + p.getType());
                }
            }
        }
    } catch (Exception e) {
        throw new CanalClientException("deserializer failed", e);
    }
}

Source File: BaseCanalServerWithEmbededTest.java From canal with Apache License 2.0

5 votes

@Test
public void testGetWithoutAck() {
    int maxEmptyCount = 10;
    int emptyCount = 0;
    int totalCount = 0;
    server.subscribe(clientIdentity);
    while (emptyCount < maxEmptyCount) {
        Message message = server.getWithoutAck(clientIdentity, 11);
        if (CollectionUtils.isEmpty(message.getEntries())) {
            emptyCount++;
            try {
                Thread.sleep(emptyCount * 300L);
            } catch (InterruptedException e) {
                Assert.fail();
            }

            System.out.println("empty count : " + emptyCount);
        } else {
            emptyCount = 0;
            totalCount += message.getEntries().size();
            server.ack(clientIdentity, message.getId());
        }
    }

    System.out.println("!!!!!! testGetWithoutAck totalCount : " + totalCount);
    server.unsubscribe(clientIdentity);
}

Source File: AbstractCanalClientTest.java From canal-1.1.3 with Apache License 2.0

5 votes

protected void process() {
    int batchSize = 5 * 1024;
    while (running) {
        try {
            MDC.put("destination", destination);
            connector.connect();
            connector.subscribe();
            while (running) {
                Message message = connector.getWithoutAck(batchSize); // 获取指定数量的数据
                long batchId = message.getId();
                int size = message.getEntries().size();
                if (batchId == -1 || size == 0) {
                    // try {
                    // Thread.sleep(1000);
                    // } catch (InterruptedException e) {
                    // }
                } else {
                    printSummary(message, batchId, size);
                    printEntry(message.getEntries());
                }

                connector.ack(batchId); // 提交确认
                // connector.rollback(batchId); // 处理失败, 回滚数据
            }
        } catch (Exception e) {
            logger.error("process error!", e);
        } finally {
            connector.disconnect();
            MDC.remove("destination");
        }
    }
}

Source File: CanalMessageSerializerUtil.java From canal with Apache License 2.0

5 votes

public static Message deserializer(byte[] data, boolean lazyParseEntry) {
    try {
        if (data == null) {
            return null;
        } else {
            CanalPacket.Packet p = CanalPacket.Packet.parseFrom(data);
            switch (p.getType()) {
                case MESSAGES: {
                    if (!p.getCompression().equals(CanalPacket.Compression.NONE)
                        && !p.getCompression().equals(CanalPacket.Compression.COMPRESSIONCOMPATIBLEPROTO2)) {
                        throw new CanalClientException("compression is not supported in this connector");
                    }

                    CanalPacket.Messages messages = CanalPacket.Messages.parseFrom(p.getBody());
                    Message result = new Message(messages.getBatchId());
                    if (lazyParseEntry) {
                        // byteString
                        result.setRawEntries(messages.getMessagesList());
                        result.setRaw(true);
                    } else {
                        for (ByteString byteString : messages.getMessagesList()) {
                            result.addEntry(CanalEntry.Entry.parseFrom(byteString));
                        }
                        result.setRaw(false);
                    }
                    return result;
                }
                case ACK: {
                    CanalPacket.Ack ack = CanalPacket.Ack.parseFrom(p.getBody());
                    throw new CanalClientException("something goes wrong with reason: " + ack.getErrorMessage());
                }
                default: {
                    throw new CanalClientException("unexpected packet type: " + p.getType());
                }
            }
        }
    } catch (Exception e) {
        throw new CanalClientException("deserializer failed", e);
    }
}

Source File: AbstractCanalClientTest.java From canal with Apache License 2.0

5 votes

protected void process() {
    int batchSize = 5 * 1024;
    while (running) {
        try {
            MDC.put("destination", destination);
            connector.connect();
            connector.subscribe();
            while (running) {
                Message message = connector.getWithoutAck(batchSize); // 获取指定数量的数据
                long batchId = message.getId();
                int size = message.getEntries().size();
                if (batchId == -1 || size == 0) {
                    // try {
                    // Thread.sleep(1000);
                    // } catch (InterruptedException e) {
                    // }
                } else {
                    printSummary(message, batchId, size);
                    printEntry(message.getEntries());
                }

                if (batchId != -1) {
                    connector.ack(batchId); // 提交确认
                    // connector.rollback(batchId); // 处理失败, 回滚数据
                }
            }
        } catch (Exception e) {
            logger.error("process error!", e);
            try {
                Thread.sleep(1000L);
            } catch (InterruptedException e1) {
                // ignore
            }
        } finally {
            connector.disconnect();
            MDC.remove("destination");
        }
    }
}

Source File: SimpleCanalConnector.java From canal with Apache License 2.0

5 votes

public Message getWithoutAck(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
    waitClientRunning();
    if (!running) {
        return null;
    }
    try {
        int size = (batchSize <= 0) ? 1000 : batchSize;
        long time = (timeout == null || timeout < 0) ? -1 : timeout; // -1代表不做timeout控制
        if (unit == null) {
            unit = TimeUnit.MILLISECONDS;
        }

        writeWithHeader(Packet.newBuilder()
            .setType(PacketType.GET)
            .setBody(Get.newBuilder()
                .setAutoAck(false)
                .setDestination(clientIdentity.getDestination())
                .setClientId(String.valueOf(clientIdentity.getClientId()))
                .setFetchSize(size)
                .setTimeout(time)
                .setUnit(unit.ordinal())
                .build()
                .toByteString())
            .build()
            .toByteArray());
        return receiveMessages();
    } catch (IOException e) {
        throw new CanalClientException(e);
    }
}

Source File: RabbitMQCanalConnector.java From canal with Apache License 2.0

5 votes

@Override
public List<Message> getList(Long timeout, TimeUnit unit) throws CanalClientException {
    List<Message> messages = getListWithoutAck(timeout, unit);
    if (messages != null && !messages.isEmpty()) {
        ack();
    }
    return messages;
}

Source File: CanalKafkaConsumer.java From canal with Apache License 2.0

5 votes

@Override
public void connect() {
    if (this.flatMessage) {
        kafkaProperties.put("value.deserializer", StringDeserializer.class);
        this.kafkaConsumer = new KafkaConsumer<String, String>(kafkaProperties);
    } else {
        kafkaProperties.put("value.deserializer", KafkaMessageDeserializer.class);
        this.kafkaConsumer = new KafkaConsumer<String, Message>(kafkaProperties);
    }
    kafkaConsumer.subscribe(Collections.singletonList(topic));
}

Source File: BaseCanalServerWithEmbededTest.java From canal-1.1.3 with Apache License 2.0

5 votes

@Test
public void testGetWithoutAck() {
    int maxEmptyCount = 10;
    int emptyCount = 0;
    int totalCount = 0;
    server.subscribe(clientIdentity);
    while (emptyCount < maxEmptyCount) {
        Message message = server.getWithoutAck(clientIdentity, 11);
        if (CollectionUtils.isEmpty(message.getEntries())) {
            emptyCount++;
            try {
                Thread.sleep(emptyCount * 300L);
            } catch (InterruptedException e) {
                Assert.fail();
            }

            System.out.println("empty count : " + emptyCount);
        } else {
            emptyCount = 0;
            totalCount += message.getEntries().size();
            server.ack(clientIdentity, message.getId());
        }
    }

    System.out.println("!!!!!! testGetWithoutAck totalCount : " + totalCount);
    server.unsubscribe(clientIdentity);
}

Source File: ProtocolTest.java From canal-1.1.3 with Apache License 2.0

5 votes

@Test
public void testSimple() throws IOException {
    Header.Builder headerBuilder = Header.newBuilder();
    headerBuilder.setLogfileName("mysql-bin.000001");
    headerBuilder.setLogfileOffset(1024);
    headerBuilder.setExecuteTime(1024);
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(headerBuilder.build());
    entryBuilder.setEntryType(EntryType.ROWDATA);
    Entry entry = entryBuilder.build();
    Message message = new Message(3, true, Arrays.asList(entry.toByteString()));

    byte[] body = buildData(message);
    Packet packet = Packet.parseFrom(body);
    switch (packet.getType()) {
        case MESSAGES: {
            if (!packet.getCompression().equals(Compression.NONE)) {
                throw new CanalClientException("compression is not supported in this connector");
            }

            Messages messages = Messages.parseFrom(packet.getBody());
            Message result = new Message(messages.getBatchId());
            for (ByteString byteString : messages.getMessagesList()) {
                result.addEntry(Entry.parseFrom(byteString));
            }

            System.out.println(result);
            break;
        }
        default: {
            throw new CanalClientException("unexpected packet type: " + packet.getType());
        }
    }
}

Source File: KafkaCanalConnector.java From canal with Apache License 2.0

5 votes

@Override
public List<Message> getList(Long timeout, TimeUnit unit) throws CanalClientException {
    waitClientRunning();
    if (!running) {
        return Lists.newArrayList();
    }

    List<Message> messages = getListWithoutAck(timeout, unit);
    if (messages != null && !messages.isEmpty()) {
        this.ack();
    }
    return messages;
}

Source File: TotoroChannel.java From canal-elasticsearch with Apache License 2.0

5 votes

public void putMessage(Message e) throws InterruptedException {
    if (rollBack.state() == true) {
        selectorMessageQueue.put(e);
    } else {
        logger.info("The rollback happened =============>  discard message , batchId :{}", e.getId());
    }
}

Source File: AbstractBasicMessageTransponder.java From spring-boot-starter-canal with MIT License

5 votes

@Override
protected void distributeEvent(Message message) {
    List<CanalEntry.Entry> entries = message.getEntries();
    for (CanalEntry.Entry entry : entries) {
        //ignore the transaction operations
        List<CanalEntry.EntryType> ignoreEntryTypes = getIgnoreEntryTypes();
        if (ignoreEntryTypes != null
                && ignoreEntryTypes.stream().anyMatch(t -> entry.getEntryType() == t)) {
            continue;
        }
        CanalEntry.RowChange rowChange;
        try {
            rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
        } catch (Exception e) {
            throw new CanalClientException("ERROR ## parser of event has an error , data:" + entry.toString(),
                    e);
        }
        //ignore the ddl operation
        if (rowChange.hasIsDdl() && rowChange.getIsDdl()) {
            processDdl(rowChange);
            continue;
        }
        for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
            //distribute to listener interfaces
            distributeByImpl(rowChange.getEventType(), rowData);
            //distribute to annotation listener interfaces
            distributeByAnnotation(destination,
                    entry.getHeader().getSchemaName(),
                    entry.getHeader().getTableName(),
                    rowChange.getEventType(),
                    rowData);
        }
    }
}

Source File: ProtocolTest.java From canal with Apache License 2.0

5 votes

@Test(expected = CanalClientException.class)
public void testSimple() throws IOException {
    Header.Builder headerBuilder = Header.newBuilder();
    headerBuilder.setLogfileName("mysql-bin.000001");
    headerBuilder.setLogfileOffset(1024);
    headerBuilder.setExecuteTime(1024);
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(headerBuilder.build());
    entryBuilder.setEntryType(EntryType.ROWDATA);
    Entry entry = entryBuilder.build();
    Message message = new Message(3, true, Arrays.asList(entry.toByteString()));

    byte[] body = buildData(message);
    Packet packet = Packet.parseFrom(body);
    switch (packet.getType()) {
        case MESSAGES: {
            if (!packet.getCompression().equals(Compression.NONE)) {
                throw new CanalClientException("compression is not supported in this connector");
            }

            Messages messages = Messages.parseFrom(packet.getBody());
            Message result = new Message(messages.getBatchId());
            for (ByteString byteString : messages.getMessagesList()) {
                result.addEntry(Entry.parseFrom(byteString));
            }

            System.out.println(result);
            break;
        }
        default: {
            throw new CanalClientException("unexpected packet type: " + packet.getType());
        }
    }
}

com.alibaba.otter.canal.protocol.Message Java Examples