com.alibaba.otter.canal.protocol.exception.CanalClientException Java Examples

public void unsubscribe() throws CanalClientException {
    waitClientRunning();
    if (!running) {
        return;
    }
    try {
        writeWithHeader(Packet.newBuilder()
            .setType(PacketType.UNSUBSCRIPTION)
            .setBody(Unsub.newBuilder()
                .setDestination(clientIdentity.getDestination())
                .setClientId(String.valueOf(clientIdentity.getClientId()))
                .build()
                .toByteString())
            .build()
            .toByteArray());
        //
        Packet p = Packet.parseFrom(readNextPacket());
        Ack ack = Ack.parseFrom(p.getBody());
        if (ack.getErrorCode() > 0) {
            throw new CanalClientException("failed to unSubscribe with reason: " + ack.getErrorMessage());
        }
    } catch (IOException e) {
        throw new CanalClientException(e);
    }
}
 

public Message get(int batchSize) throws CanalClientException {
    int times = 0;
    while (times < retryTimes) {
        try {
            Message msg = currentConnector.get(batchSize);
            return msg;
        } catch (Throwable t) {
            logger.warn(String.format("something goes wrong when getting data from server:%s",
                currentConnector != null ? currentConnector.getAddress() : "null"), t);
            times++;
            restart();
            logger.info("restart the connector for next round retry.");
        }
    }
    throw new CanalClientException("failed to fetch the data after " + times + " times retry");
}
 

public Message get(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
    int times = 0;
    while (times < retryTimes) {
        try {
            Message msg = currentConnector.get(batchSize, timeout, unit);
            return msg;
        } catch (Throwable t) {
            logger.warn(String.format("something goes wrong when getting data from server:%s",
                currentConnector != null ? currentConnector.getAddress() : "null"), t);
            times++;
            restart();
            logger.info("restart the connector for next round retry.");
        }
    }
    throw new CanalClientException("failed to fetch the data after " + times + " times retry");
}
 

@Override
public List<Message> getListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException {
    try {
        if (this.lastGetBatchMessage != null) {
            throw new CanalClientException("mq get/ack not support concurrent & async ack");
        }

        ConsumerBatchMessage batchMessage = messageBlockingQueue.poll(timeout, unit);
        if (batchMessage != null) {
            this.lastGetBatchMessage = batchMessage;
            return batchMessage.getData();
        }
    } catch (InterruptedException ex) {
        logger.warn("Get message timeout", ex);
        throw new CanalClientException("Failed to fetch the data after: " + timeout);
    }
    return Lists.newArrayList();
}
 

public Message getWithoutAck(int batchSize) throws CanalClientException {
    int times = 0;
    while (times < retryTimes) {
        try {
            Message msg = currentConnector.getWithoutAck(batchSize);
            return msg;
        } catch (Throwable t) {
            logger.warn(String.format("something goes wrong when getWithoutAck data from server:%s",
                currentConnector != null ? currentConnector.getAddress() : "null"), t);
            times++;
            restart();
            logger.info("restart the connector for next round retry.");
        }
    }
    throw new CanalClientException("failed to fetch the data after " + times + " times retry");
}
 

public Message getWithoutAck(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
    int times = 0;
    while (times < retryTimes) {
        try {
            Message msg = currentConnector.getWithoutAck(batchSize, timeout, unit);
            return msg;
        } catch (Throwable t) {
            logger.warn(String.format("something goes wrong when getWithoutAck data from server:%s",
                currentConnector != null ? currentConnector.getAddress() : "null"), t);
            times++;
            restart();
            logger.info("restart the connector for next round retry.");
        }
    }
    throw new CanalClientException("failed to fetch the data after " + times + " times retry");
}
 

public void connect() throws CanalClientException {

        RPCHook rpcHook = null;
        if (null != accessKey && accessKey.length() > 0 && null != secretKey && secretKey.length() > 0) {
            SessionCredentials sessionCredentials = new SessionCredentials();
            sessionCredentials.setAccessKey(accessKey);
            sessionCredentials.setSecretKey(secretKey);
            rpcHook = new ClientRPCHook(sessionCredentials);
        }
        rocketMQConsumer = new DefaultMQPushConsumer(groupName, rpcHook, new AllocateMessageQueueAveragely());
        rocketMQConsumer.setVipChannelEnabled(false);
        if (!StringUtils.isBlank(nameServer)) {
            rocketMQConsumer.setNamesrvAddr(nameServer);
        }
        if (batchSize != -1) {
            rocketMQConsumer.setConsumeMessageBatchMaxSize(batchSize);
        }
    }
 

public void rollback(long batchId) throws CanalClientException {
    int times = 0;
    while (times < retryTimes) {
        try {
            currentConnector.rollback(batchId);
            return;
        } catch (Throwable t) {
            logger.warn(String.format("something goes wrong when rollbacking data from server:%s",
                currentConnector != null ? currentConnector.getAddress() : "null"), t);
            times++;
            restart();
            logger.info("restart the connector for next round retry.");
        }
    }
    throw new CanalClientException("failed to rollback after " + times + " times retry");
}
 

public void rollback() throws CanalClientException {
    int times = 0;
    while (times < retryTimes) {
        try {
            currentConnector.rollback();
            return;
        } catch (Throwable t) {
            logger.warn(String.format("something goes wrong when rollbacking data from server:%s",
                currentConnector != null ? currentConnector.getAddress() : "null"), t);
            times++;
            restart();
            logger.info("restart the connector for next round retry.");
        }
    }

    throw new CanalClientException("failed to rollback after " + times + " times retry");
}
 

public void ack(long batchId) throws CanalClientException {
    int times = 0;
    while (times < retryTimes) {
        try {
            currentConnector.ack(batchId);
            return;
        } catch (Throwable t) {
            logger.warn(String.format("something goes wrong when acking data from server:%s",
                currentConnector != null ? currentConnector.getAddress() : "null"), t);
            times++;
            restart();
            logger.info("restart the connector for next round retry.");
        }
    }

    throw new CanalClientException("failed to ack after " + times + " times retry");
}
 

@Override
public List<FlatMessage> getFlatListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException {
    try {
        if (this.lastGetBatchMessage != null) {
            throw new CanalClientException("mq get/ack not support concurrent & async ack");
        }

        ConsumerBatchMessage batchMessage = messageBlockingQueue.poll(timeout, unit);
        if (batchMessage != null) {
            this.lastGetBatchMessage = batchMessage;
            return batchMessage.getData();
        }
    } catch (InterruptedException ex) {
        logger.warn("Get message timeout", ex);
        throw new CanalClientException("Failed to fetch the data after: " + timeout);
    }
    return Lists.newArrayList();
}
 

@Override
public List<Message> getListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException {
    waitClientRunning();
    if (!running) {
        return Lists.newArrayList();
    }

    ConsumerRecords<String, Message> records = kafkaConsumer.poll(unit.toMillis(timeout));

    currentOffsets.clear();
    for (TopicPartition topicPartition : records.partitions()) {
        currentOffsets.put(topicPartition.partition(), kafkaConsumer.position(topicPartition));
    }

    if (!records.isEmpty()) {
        List<Message> messages = new ArrayList<>();
        for (ConsumerRecord<String, Message> record : records) {
            messages.add(record.value());
        }
        return messages;
    }
    return Lists.newArrayList();
}
 

@Override
public List<FlatMessage> getFlatListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException {
    waitClientRunning();
    if (!running) {
        return Lists.newArrayList();
    }

    ConsumerRecords<String, String> records = kafkaConsumer2.poll(unit.toMillis(timeout));

    currentOffsets.clear();
    for (TopicPartition topicPartition : records.partitions()) {
        currentOffsets.put(topicPartition.partition(), kafkaConsumer2.position(topicPartition));
    }

    if (!records.isEmpty()) {
        List<FlatMessage> flatMessages = new ArrayList<>();
        for (ConsumerRecord<String, String> record : records) {
            String flatMessageJson = record.value();
            FlatMessage flatMessage = JSON.parseObject(flatMessageJson, FlatMessage.class);
            flatMessages.add(flatMessage);
        }

        return flatMessages;
    }
    return Lists.newArrayList();
}
 

/**
 * 获取Kafka消息，不确认
 *
 * @param timeout
 * @param unit
 * @param offset  消息偏移地址（-1为不偏移）
 * @return
 * @throws CanalClientException
 */
public List<KafkaFlatMessage> getFlatListWithoutAck(Long timeout, TimeUnit unit, long offset) throws CanalClientException {
    waitClientRunning();
    if (!running) {
        return Lists.newArrayList();
    }

    if (offset > -1) {
        TopicPartition tp = new TopicPartition(topic, partition == null ? 0 : partition);
        kafkaConsumer2.seek(tp, offset);
    }

    ConsumerRecords<String, String> records = kafkaConsumer2.poll(unit.toMillis(timeout));
    if (!records.isEmpty()) {
        List<KafkaFlatMessage> flatMessages = new ArrayList<>();
        for (ConsumerRecord<String, String> record : records) {
            String flatMessageJson = record.value();
            FlatMessage flatMessage = JSON.parseObject(flatMessageJson, FlatMessage.class);
            KafkaFlatMessage message = new KafkaFlatMessage(flatMessage, record.offset());
            flatMessages.add(message);
        }

        return flatMessages;
    }
    return Lists.newArrayList();
}
 

/**
 * 获取Kafka消息，不确认
 *
 * @param timeout
 * @param unit
 * @param offset  消息偏移地址（-1为不偏移）
 * @return
 * @throws CanalClientException
 */
public List<KafkaMessage> getListWithoutAck(Long timeout, TimeUnit unit, long offset) throws CanalClientException {
    waitClientRunning();
    if (!running) {
        return Lists.newArrayList();
    }

    if (offset > -1) {
        TopicPartition tp = new TopicPartition(topic, partition == null ? 0 : partition);
        kafkaConsumer.seek(tp, offset);
    }

    ConsumerRecords<String, Message> records = kafkaConsumer.poll(unit.toMillis(timeout));

    if (!records.isEmpty()) {
        List<KafkaMessage> messages = new ArrayList<>();
        for (ConsumerRecord<String, Message> record : records) {
            KafkaMessage message = new KafkaMessage(record.value(), record.offset());
            messages.add(message);
        }
        return messages;
    }
    return Lists.newArrayList();
}
 

public void ack(long batchId) throws CanalClientException {
    waitClientRunning();
    if (!running) {
        return;
    }
    ClientAck ca = ClientAck.newBuilder()
        .setDestination(clientIdentity.getDestination())
        .setClientId(String.valueOf(clientIdentity.getClientId()))
        .setBatchId(batchId)
        .build();
    try {
        writeWithHeader(Packet.newBuilder()
            .setType(PacketType.CLIENTACK)
            .setBody(ca.toByteString())
            .build()
            .toByteArray());
    } catch (IOException e) {
        throw new CanalClientException(e);
    }
}
 

public void rollback(long batchId) throws CanalClientException {
    waitClientRunning();
    ClientRollback ca = ClientRollback.newBuilder()
        .setDestination(clientIdentity.getDestination())
        .setClientId(String.valueOf(clientIdentity.getClientId()))
        .setBatchId(batchId)
        .build();
    try {
        writeWithHeader(Packet.newBuilder()
            .setType(PacketType.CLIENTROLLBACK)
            .setBody(ca.toByteString())
            .build()
            .toByteArray());
    } catch (IOException e) {
        throw new CanalClientException(e);
    }
}
 

public void connect() throws CanalClientException {
        if (connected) {
            return;
        }

//        client监听器启动
        if (runningMonitor != null) {
            if (!runningMonitor.isStart()) {
                runningMonitor.start();
            }
        } else {
            waitClientRunning();
            if (!running) {
                return;
            }
            doConnect();
            if (filter != null) { // 如果存在条件，说明是自动切换，基于上一次的条件订阅一次
                subscribe(filter);
            }
            if (rollbackOnConnect) {
                rollback();
            }
        }

        connected = true;
    }
 

private void waitClientRunning() {
    try {
        if (zkClientx != null) {
            if (!connected) {// 未调用connect
                throw new CanalClientException("should connect first");
            }

            running = true;
            mutex.get();// 阻塞等待
        } else {
            // 单机模式直接设置为running
            running = true;
        }
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        throw new CanalClientException(e);
    }
}
 

public void subscribe(String filter) throws CanalClientException {
    int times = 0;
    while (times < retryTimes) {
        try {
            currentConnector.subscribe(filter);
            this.filter = filter;
            return;
        } catch (Throwable t) {
            if (retryTimes == -1 && t.getCause() instanceof InterruptedException) {
                logger.info("block waiting interrupted by other thread.");
                return;
            } else {
                logger.warn(String.format("something goes wrong when subscribing from server: %s",
                    currentConnector != null ? currentConnector.getAddress() : "null"), t);
                times++;
                restart();
                logger.info("restart the connector for next round retry.");
            }

        }
    }

    throw new CanalClientException("failed to subscribe after " + times + " times retry.");
}
 

public void unsubscribe() throws CanalClientException {
    int times = 0;
    while (times < retryTimes) {
        try {
            currentConnector.unsubscribe();
            return;
        } catch (Throwable t) {
            logger.warn(String.format("something goes wrong when unsubscribing from server:%s",
                currentConnector != null ? currentConnector.getAddress() : "null"), t);
            times++;
            restart();
            logger.info("restart the connector for next round retry.");
        }
    }
    throw new CanalClientException("failed to unsubscribe after " + times + " times retry.");
}
 

@Override
public List<FlatMessage> getFlatList(Long timeout, TimeUnit unit) throws CanalClientException {
    List<FlatMessage> messages = getFlatListWithoutAck(timeout, unit);
    if (messages != null && !messages.isEmpty()) {
        ack();
    }
    return messages;
}
 

public Message getWithoutAck(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
    waitClientRunning();
    if (!running) {
        return null;
    }
    try {
        int size = (batchSize <= 0) ? 1000 : batchSize;
        long time = (timeout == null || timeout < 0) ? -1 : timeout; // -1代表不做timeout控制
        if (unit == null) {
            unit = TimeUnit.MILLISECONDS;
        }

        writeWithHeader(Packet.newBuilder()
            .setType(PacketType.GET)
            .setBody(Get.newBuilder()
                .setAutoAck(false)
                .setDestination(clientIdentity.getDestination())
                .setClientId(String.valueOf(clientIdentity.getClientId()))
                .setFetchSize(size)
                .setTimeout(time)
                .setUnit(unit.ordinal())
                .build()
                .toByteString())
            .build()
            .toByteArray());
        return receiveMessages();
    } catch (IOException e) {
        throw new CanalClientException(e);
    }
}
 

@Override
public void ack() throws CanalClientException {
    try {
        if (this.lastGetBatchMessage != null) {
            this.lastGetBatchMessage.ack();
        }
    } catch (Throwable e) {
        if (this.lastGetBatchMessage != null) {
            this.lastGetBatchMessage.fail();
        }
    } finally {
        this.lastGetBatchMessage = null;
    }
}
 

@Override
public void rollback() throws CanalClientException {
    try {
        if (this.lastGetBatchMessage != null) {
            this.lastGetBatchMessage.fail();
        }
    } finally {
        this.lastGetBatchMessage = null;
    }
}
 

@Test
public void testSimple() throws IOException {
    Header.Builder headerBuilder = Header.newBuilder();
    headerBuilder.setLogfileName("mysql-bin.000001");
    headerBuilder.setLogfileOffset(1024);
    headerBuilder.setExecuteTime(1024);
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(headerBuilder.build());
    entryBuilder.setEntryType(EntryType.ROWDATA);
    Entry entry = entryBuilder.build();
    Message message = new Message(3, true, Arrays.asList(entry.toByteString()));

    byte[] body = buildData(message);
    Packet packet = Packet.parseFrom(body);
    switch (packet.getType()) {
        case MESSAGES: {
            if (!packet.getCompression().equals(Compression.NONE)) {
                throw new CanalClientException("compression is not supported in this connector");
            }

            Messages messages = Messages.parseFrom(packet.getBody());
            Message result = new Message(messages.getBatchId());
            for (ByteString byteString : messages.getMessagesList()) {
                result.addEntry(Entry.parseFrom(byteString));
            }

            System.out.println(result);
            break;
        }
        default: {
            throw new CanalClientException("unexpected packet type: " + packet.getType());
        }
    }
}
 

@Override
public List<FlatMessage> getFlatList(Long timeout, TimeUnit unit) throws CanalClientException {
    waitClientRunning();
    if (!running) {
        return Lists.newArrayList();
    }

    List<FlatMessage> messages = getFlatListWithoutAck(timeout, unit);
    if (messages != null && !messages.isEmpty()) {
        this.ack();
    }
    return messages;
}
 

@Override
public List<Message> getList(Long timeout, TimeUnit unit) throws CanalClientException {
    waitClientRunning();
    if (!running) {
        return Lists.newArrayList();
    }

    List<Message> messages = getListWithoutAck(timeout, unit);
    if (messages != null && !messages.isEmpty()) {
        this.ack();
    }
    return messages;
}
 

public synchronized void subscribe(String filter) throws CanalClientException {
    if (connected) {
        return;
    }
    try {
        if (rocketMQConsumer == null) {
            this.connect();
        }
        rocketMQConsumer.subscribe(this.topic, "*");
        rocketMQConsumer.registerMessageListener(new MessageListenerOrderly() {

            @Override
            public ConsumeOrderlyStatus consumeMessage(List<MessageExt> messageExts, ConsumeOrderlyContext context) {
                context.setAutoCommit(true);
                boolean isSuccess = process(messageExts);
                if (isSuccess) {
                    return ConsumeOrderlyStatus.SUCCESS;
                } else {
                    return ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT;
                }
            }
        });
        rocketMQConsumer.start();
    } catch (MQClientException ex) {
        connected = false;
        logger.error("Start RocketMQ consumer error", ex);
    }
    connected = true;
}
 

@Override
public List<Message> getList(Long timeout, TimeUnit unit) throws CanalClientException {
    List<Message> messages = getListWithoutAck(timeout, unit);
    if (messages != null && !messages.isEmpty()) {
        ack();
    }
    return messages;
}