org.apache.rocketmq.common.protocol.heartbeat.MessageModel Java Examples

@Test
public void testMessageQueueChanged_CountThreshold() {
    RebalancePushImpl rebalancePush = new RebalancePushImpl(consumerGroup, MessageModel.CLUSTERING,
        new AllocateMessageQueueAveragely(), mqClientInstance, defaultMQPushConsumer);
    init(rebalancePush);

    // Just set pullThresholdForQueue
    defaultMQPushConsumer.getDefaultMQPushConsumer().setPullThresholdForQueue(1024);
    Set<MessageQueue> allocateResultSet = new HashSet<MessageQueue>();
    allocateResultSet.add(new MessageQueue(topic, "BrokerA", 0));
    allocateResultSet.add(new MessageQueue(topic, "BrokerA", 1));
    doRebalanceForcibly(rebalancePush, allocateResultSet);
    assertThat(defaultMQPushConsumer.getDefaultMQPushConsumer().getPullThresholdForQueue()).isEqualTo(1024);

    // Set pullThresholdForTopic
    defaultMQPushConsumer.getDefaultMQPushConsumer().setPullThresholdForTopic(1024);
    doRebalanceForcibly(rebalancePush, allocateResultSet);
    assertThat(defaultMQPushConsumer.getDefaultMQPushConsumer().getPullThresholdForQueue()).isEqualTo(512);

    // Change message queue allocate result
    allocateResultSet.add(new MessageQueue(topic, "BrokerA", 2));
    doRebalanceForcibly(rebalancePush, allocateResultSet);
    assertThat(defaultMQPushConsumer.getDefaultMQPushConsumer().getPullThresholdForQueue()).isEqualTo(341);
}
 

public static void main(String[] args) throws InterruptedException, MQClientException {
    DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("please_rename_unique_group_name_1");

    consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);

    consumer.setMessageModel(MessageModel.BROADCASTING);

    consumer.subscribe("TopicTest", "TagA || TagC || TagD");

    consumer.registerMessageListener(new MessageListenerConcurrently() {

        @Override
        public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,
            ConsumeConcurrentlyContext context) {
            System.out.printf("%s Receive New Messages: %s %n", Thread.currentThread().getName(), msgs);
            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
        }
    });

    consumer.start();
    System.out.printf("Broadcast Consumer Started.%n");
}
 

@Test
public void testMessageQueueChanged_SizeThreshold() {
    RebalancePushImpl rebalancePush = new RebalancePushImpl(consumerGroup, MessageModel.CLUSTERING,
        new AllocateMessageQueueAveragely(), mqClientInstance, defaultMQPushConsumer);
    init(rebalancePush);

    // Just set pullThresholdSizeForQueue
    defaultMQPushConsumer.getDefaultMQPushConsumer().setPullThresholdSizeForQueue(1024);
    Set<MessageQueue> allocateResultSet = new HashSet<MessageQueue>();
    allocateResultSet.add(new MessageQueue(topic, "BrokerA", 0));
    allocateResultSet.add(new MessageQueue(topic, "BrokerA", 1));
    doRebalanceForcibly(rebalancePush, allocateResultSet);
    assertThat(defaultMQPushConsumer.getDefaultMQPushConsumer().getPullThresholdSizeForQueue()).isEqualTo(1024);

    // Set pullThresholdSizeForTopic
    defaultMQPushConsumer.getDefaultMQPushConsumer().setPullThresholdSizeForTopic(1024);
    doRebalanceForcibly(rebalancePush, allocateResultSet);
    assertThat(defaultMQPushConsumer.getDefaultMQPushConsumer().getPullThresholdSizeForQueue()).isEqualTo(512);

    // Change message queue allocate result
    allocateResultSet.add(new MessageQueue(topic, "BrokerA", 2));
    doRebalanceForcibly(rebalancePush, allocateResultSet);
    assertThat(defaultMQPushConsumer.getDefaultMQPushConsumer().getPullThresholdSizeForQueue()).isEqualTo(341);
}
 

@Test
public void testMessageQueueChanged_SizeThreshold() {
    RebalancePushImpl rebalancePush = new RebalancePushImpl(consumerGroup, MessageModel.CLUSTERING,
        new AllocateMessageQueueAveragely(), mqClientInstance, defaultMQPushConsumer);
    init(rebalancePush);

    // Just set pullThresholdSizeForQueue
    defaultMQPushConsumer.getDefaultMQPushConsumer().setPullThresholdSizeForQueue(1024);
    Set<MessageQueue> allocateResultSet = new HashSet<MessageQueue>();
    allocateResultSet.add(new MessageQueue(topic, "BrokerA", 0));
    allocateResultSet.add(new MessageQueue(topic, "BrokerA", 1));
    doRebalanceForcibly(rebalancePush, allocateResultSet);
    assertThat(defaultMQPushConsumer.getDefaultMQPushConsumer().getPullThresholdSizeForQueue()).isEqualTo(1024);

    // Set pullThresholdSizeForTopic
    defaultMQPushConsumer.getDefaultMQPushConsumer().setPullThresholdSizeForTopic(1024);
    doRebalanceForcibly(rebalancePush, allocateResultSet);
    assertThat(defaultMQPushConsumer.getDefaultMQPushConsumer().getPullThresholdSizeForQueue()).isEqualTo(512);

    // Change message queue allocate result
    allocateResultSet.add(new MessageQueue(topic, "BrokerA", 2));
    doRebalanceForcibly(rebalancePush, allocateResultSet);
    assertThat(defaultMQPushConsumer.getDefaultMQPushConsumer().getPullThresholdSizeForQueue()).isEqualTo(341);
}
 

public boolean registerConsumer(final String group, final ClientChannelInfo clientChannelInfo,
    ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere,
    final Set<SubscriptionData> subList, boolean isNotifyConsumerIdsChangedEnable) {

    ConsumerGroupInfo consumerGroupInfo = this.consumerTable.get(group);
    if (null == consumerGroupInfo) {
        ConsumerGroupInfo tmp = new ConsumerGroupInfo(group, consumeType, messageModel, consumeFromWhere);
        ConsumerGroupInfo prev = this.consumerTable.putIfAbsent(group, tmp);
        consumerGroupInfo = prev != null ? prev : tmp;
    }

    boolean r1 =
        consumerGroupInfo.updateChannel(clientChannelInfo, consumeType, messageModel,
            consumeFromWhere);
    boolean r2 = consumerGroupInfo.updateSubscription(subList);

    if (r1 || r2) {
        if (isNotifyConsumerIdsChangedEnable) {
            this.consumerIdsChangeListener.handle(ConsumerGroupEvent.CHANGE, group, consumerGroupInfo.getAllChannel());
        }
    }

    this.consumerIdsChangeListener.handle(ConsumerGroupEvent.REGISTER, group, subList);

    return r1 || r2;
}
 

private int adjustQueueSizeByMaxConsumerCount(String topic) {
    int queueSize = this.deFiBrokerController.getDeFiBusBrokerConfig().getMinQueueNum();
    Set<String> maxCidList = null;
    Set<String> topicConsumeByWho = this.deFiBrokerController.getConsumerManager().queryTopicConsumeByWho(topic);
    for (String group : topicConsumeByWho) {
        DeFiConsumerGroupInfo consumerGroupInfo = (DeFiConsumerGroupInfo) this.deFiBrokerController.getConsumerManager().getConsumerGroupInfo(group);
        if (consumerGroupInfo != null && consumerGroupInfo.getMessageModel() == MessageModel.CLUSTERING) {
            Set<String> cidList = consumerGroupInfo.getClientIdBySubscription(topic);
            if (cidList != null) {
                int scaleSize = this.scaleQueueSize(cidList);
                if (scaleSize >= queueSize) {
                    queueSize = scaleSize;
                    maxCidList = cidList;
                }
            }
        }
    }
    log.info("calculate queue size by max consumer count, result: {} cidList: {}", queueSize, maxCidList);
    return queueSize;
}
 

/**
 * 注册一个消费者
 * @param group group
 * @param clientChannelInfo clientChannelInfo
 * @param consumeType consumeType
 * @param messageModel 消息模式
 * @param consumeFromWhere consumerFromWhere
 * @param subList  SubscriptionData
 * @param isNotifyConsumerIdsChangedEnable isNotifyConsumerIdsChangedEnable
 * @return ;
 */
public boolean registerConsumer(final String group, final ClientChannelInfo clientChannelInfo,
    ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere,
    final Set<SubscriptionData> subList, boolean isNotifyConsumerIdsChangedEnable) {

    ConsumerGroupInfo consumerGroupInfo = this.consumerTable.get(group);
    if (null == consumerGroupInfo) {
        ConsumerGroupInfo tmp = new ConsumerGroupInfo(group, consumeType, messageModel, consumeFromWhere);
        ConsumerGroupInfo prev = this.consumerTable.putIfAbsent(group, tmp);
        consumerGroupInfo = prev != null ? prev : tmp;
    }

    boolean r1 = consumerGroupInfo.updateChannel(clientChannelInfo, consumeType, messageModel, consumeFromWhere);
    boolean r2 = consumerGroupInfo.updateSubscription(subList);

    if (r1 || r2) {
        if (isNotifyConsumerIdsChangedEnable) {
            this.consumerIdsChangeListener.handle(ConsumerGroupEvent.CHANGE, group, consumerGroupInfo.getAllChannel());
        }
    }

    this.consumerIdsChangeListener.handle(ConsumerGroupEvent.REGISTER, group, subList);

    return r1 || r2;
}
 

public static void main(String[] args) {
    DeFiBusClientConfig deFiBusClientConfig = new DeFiBusClientConfig();
    deFiBusClientConfig.setNamesrvAddr("");
    DeFiBusProducer deFiBusProducer = new DeFiBusProducer(deFiBusClientConfig);
    DeFiBusPushConsumer deFiBusPushConsumer = new DeFiBusPushConsumer(deFiBusClientConfig);
    deFiBusPushConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET);
    deFiBusPushConsumer.getDefaultMQPushConsumer().setMessageModel(MessageModel.CLUSTERING);
    deFiBusPushConsumer.registerMessageListener(new DeFiBusMessageListenerConcurrentlyWithReply(deFiBusProducer) {
        @Override
        public String handleMessage(MessageExt msg, ConsumeConcurrentlyContext context) {
            //1. biz handle logic

            //2. create reply content
            String replyContent = "A reply message content";

            return replyContent;
        }
    });
    try {
        deFiBusProducer.start();
        deFiBusPushConsumer.subscribe("REQUEST_REPLY_TOPIC");
        deFiBusPushConsumer.start();
    } catch (MQClientException e) {
        e.printStackTrace();
    }
}
 

public static void main(String[] args) throws InterruptedException, MQClientException {
    DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("please_rename_unique_group_name_1");

    consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);

    consumer.setMessageModel(MessageModel.BROADCASTING);

    consumer.subscribe("TopicTest", "TagA || TagC || TagD");

    consumer.registerMessageListener(new MessageListenerConcurrently() {

        @Override
        public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,
            ConsumeConcurrentlyContext context) {
            System.out.printf("%s Receive New Messages: %s %n", Thread.currentThread().getName(), msgs);
            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
        }
    });

    consumer.start();
    System.out.printf("Broadcast Consumer Started.%n");
}
 

@Override
public void messageQueueChanged(String topic, Set<MessageQueue> mqAll, Set<MessageQueue> mqDivided) {
    MessageModel messageModel =
        MQPullConsumerScheduleService.this.defaultMQPullConsumer.getMessageModel();
    switch (messageModel) {
        //广播消费
        case BROADCASTING:
            MQPullConsumerScheduleService.this.putTask(topic, mqAll);
            break;
            //集群消费
        case CLUSTERING:
            MQPullConsumerScheduleService.this.putTask(topic, mqDivided);
            break;
        default:
            break;
    }
}
 

@BeforeClass
public static void init() throws NoSuchFieldException, IllegalAccessException, InterruptedException, RemotingTimeoutException, MQClientException, RemotingSendRequestException, RemotingConnectException, MQBrokerException {
    mQClientAPIImpl = mock(MQClientAPIImpl.class);
    defaultMQAdminExt = new DefaultMQAdminExt();
    defaultMQAdminExtImpl = new DefaultMQAdminExtImpl(defaultMQAdminExt, 1000);

    Field field = DefaultMQAdminExtImpl.class.getDeclaredField("mqClientInstance");
    field.setAccessible(true);
    field.set(defaultMQAdminExtImpl, mqClientInstance);
    field = MQClientInstance.class.getDeclaredField("mQClientAPIImpl");
    field.setAccessible(true);
    field.set(mqClientInstance, mQClientAPIImpl);
    field = DefaultMQAdminExt.class.getDeclaredField("defaultMQAdminExtImpl");
    field.setAccessible(true);
    field.set(defaultMQAdminExt, defaultMQAdminExtImpl);

    ConsumerConnection consumerConnection = new ConsumerConnection();
    consumerConnection.setConsumeType(ConsumeType.CONSUME_PASSIVELY);
    consumerConnection.setMessageModel(MessageModel.CLUSTERING);
    HashSet<Connection> connections = new HashSet<>();
    connections.add(new Connection());
    consumerConnection.setConnectionSet(connections);
    consumerConnection.setSubscriptionTable(new ConcurrentHashMap<String, SubscriptionData>());
    consumerConnection.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);
    when(mQClientAPIImpl.getConsumerConnectionList(anyString(), anyString(), anyLong())).thenReturn(consumerConnection);
}
 

@BeforeClass
public static void init() throws NoSuchFieldException, IllegalAccessException, InterruptedException, RemotingTimeoutException, MQClientException, RemotingSendRequestException, RemotingConnectException, MQBrokerException {
    mQClientAPIImpl = mock(MQClientAPIImpl.class);
    defaultMQAdminExt = new DefaultMQAdminExt();
    defaultMQAdminExtImpl = new DefaultMQAdminExtImpl(defaultMQAdminExt, 1000);

    Field field = DefaultMQAdminExtImpl.class.getDeclaredField("mqClientInstance");
    field.setAccessible(true);
    field.set(defaultMQAdminExtImpl, mqClientInstance);
    field = MQClientInstance.class.getDeclaredField("mQClientAPIImpl");
    field.setAccessible(true);
    field.set(mqClientInstance, mQClientAPIImpl);
    field = DefaultMQAdminExt.class.getDeclaredField("defaultMQAdminExtImpl");
    field.setAccessible(true);
    field.set(defaultMQAdminExt, defaultMQAdminExtImpl);

    ConsumerConnection consumerConnection = new ConsumerConnection();
    consumerConnection.setConsumeType(ConsumeType.CONSUME_PASSIVELY);
    consumerConnection.setMessageModel(MessageModel.CLUSTERING);
    HashSet<Connection> connections = new HashSet<>();
    connections.add(new Connection());
    consumerConnection.setConnectionSet(connections);
    consumerConnection.setSubscriptionTable(new ConcurrentHashMap<String, SubscriptionData>());
    consumerConnection.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);
    when(mQClientAPIImpl.getConsumerConnectionList(anyString(), anyString(), anyLong())).thenReturn(consumerConnection);
}
 

public static void main(String[] args) throws InterruptedException, MQClientException {
        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("please_rename_unique_group_name_1");

        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);

        consumer.setMessageModel(MessageModel.BROADCASTING);

//        consumer.subscribe("TopicTest", "TagA || TagC || TagD");
        consumer.subscribe("TopicTest", "*");

        consumer.setNamesrvAddr("127.0.0.1:9876"); // TODO add by yunai

        consumer.registerMessageListener(new MessageListenerConcurrently() {

            @Override
            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,
                ConsumeConcurrentlyContext context) {
                System.out.printf(Thread.currentThread().getName() + " Receive New Messages: " + msgs + "%n");
                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
            }
        });

        consumer.start();
        System.out.printf("Broadcast Consumer Started.%n");
    }
 

public static void main(String[] args) throws Exception {
    DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("example_group_name");

    consumer.setNamesrvAddr("127.0.0.1:9876");
    consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);

    //set to broadcast mode
    consumer.setMessageModel(MessageModel.BROADCASTING);

    consumer.subscribe("TopicTest", "TagA || TagC || TagD");

    consumer.registerMessageListener(new MessageListenerConcurrently() {

        @Override
        public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,
                                                        ConsumeConcurrentlyContext context) {
            System.out.printf(Thread.currentThread().getName() + " Receive New Messages: " + msgs + "%n");
            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
        }
    });

    consumer.start();
    System.out.printf("Broadcast Consumer Started.%n");
}
 

/**
 * 启动
 */
@Override
public void start() {
    /*
     * 如果是集群消费。周期性调度：lockMQPeriodically
     */
    if (MessageModel.CLUSTERING.equals(ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl.messageModel())) {
        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
            @Override
            public void run() {
                ConsumeMessageOrderlyService.this.lockMQPeriodically();
            }
        }, 1000 * 1, ProcessQueue.REBALANCE_LOCK_INTERVAL, TimeUnit.MILLISECONDS);
    }
}
 

public boolean updateChannel(final ClientChannelInfo infoNew, ConsumeType consumeType,
        MessageModel messageModel, ConsumeFromWhere consumeFromWhere) {
        boolean updated = false;
        this.consumeType = consumeType;
        this.messageModel = messageModel;
        this.consumeFromWhere = consumeFromWhere;

//        获取channel信息
        ClientChannelInfo infoOld = this.channelInfoTable.get(infoNew.getChannel());
        if (null == infoOld) {
            ClientChannelInfo prev = this.channelInfoTable.put(infoNew.getChannel(), infoNew);
            if (null == prev) {
                log.info("new consumer connected, group: {} {} {} channel: {}", this.groupName, consumeType,
                    messageModel, infoNew.toString());
                updated = true;
            }

            infoOld = infoNew;
        } else {
            if (!infoOld.getClientId().equals(infoNew.getClientId())) {
                log.error("[BUG] consumer channel exist in broker, but clientId not equal. GROUP: {} OLD: {} NEW: {} ",
                    this.groupName,
                    infoOld.toString(),
                    infoNew.toString());
                this.channelInfoTable.put(infoNew.getChannel(), infoNew);
            }
        }

        this.lastUpdateTimestamp = System.currentTimeMillis();
        infoOld.setLastUpdateTimestamp(this.lastUpdateTimestamp);

        return updated;
    }
 

public void shutdown() {
    this.stopped = true;
    this.scheduledExecutorService.shutdown();
    this.consumeExecutor.shutdown();
    if (MessageModel.CLUSTERING.equals(this.defaultMQPushConsumerImpl.messageModel())) {
        this.unlockAllMQ();
    }
}
 

static ConsumerData createConsumerData(String group, String topic) {
    ConsumerData consumerData = new ConsumerData();
    consumerData.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);
    consumerData.setConsumeType(ConsumeType.CONSUME_PASSIVELY);
    consumerData.setGroupName(group);
    consumerData.setMessageModel(MessageModel.CLUSTERING);
    Set<SubscriptionData> subscriptionDataSet = new HashSet<>();
    SubscriptionData subscriptionData = new SubscriptionData();
    subscriptionData.setTopic(topic);
    subscriptionData.setSubString("*");
    subscriptionData.setSubVersion(100L);
    subscriptionDataSet.add(subscriptionData);
    consumerData.setSubscriptionDataSet(subscriptionDataSet);
    return consumerData;
}
 

public RebalanceImpl(String consumerGroup, MessageModel messageModel,
    AllocateMessageQueueStrategy allocateMessageQueueStrategy,
    MQClientInstance mQClientFactory) {
    this.consumerGroup = consumerGroup;
    this.messageModel = messageModel;
    this.allocateMessageQueueStrategy = allocateMessageQueueStrategy;
    this.mQClientFactory = mQClientFactory;
}
 

static ConsumerData createConsumerData(String group, String topic) {
    ConsumerData consumerData = new ConsumerData();
    consumerData.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);
    consumerData.setConsumeType(ConsumeType.CONSUME_PASSIVELY);
    consumerData.setGroupName(group);
    consumerData.setMessageModel(MessageModel.CLUSTERING);
    Set<SubscriptionData> subscriptionDataSet = new HashSet<>();
    SubscriptionData subscriptionData = new SubscriptionData();
    subscriptionData.setTopic(topic);
    subscriptionData.setSubString("*");
    subscriptionData.setSubVersion(100L);
    subscriptionDataSet.add(subscriptionData);
    consumerData.setSubscriptionDataSet(subscriptionDataSet);
    return consumerData;
}
 

public void start() {
        if (MessageModel.CLUSTERING.equals(ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl.messageModel())) {
            this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
                @Override
                public void run() {
//                    锁定消费队列=》
                    ConsumeMessageOrderlyService.this.lockMQPeriodically();
                }
            }, 1000 * 1, ProcessQueue.REBALANCE_LOCK_INTERVAL, TimeUnit.MILLISECONDS);
        }
    }
 

/**
 * Reset consumer topic offset according to time 根据时间重置消费者主题偏移量
 *
 * @param messageModel which model
 * @param instanceName which instance
 * @param consumerGroup consumer group
 * @param topic topic
 * @param timestamp time
 */
public static void resetOffsetByTimestamp(
    final MessageModel messageModel,
    final String instanceName,
    final String consumerGroup,
    final String topic,
    final long timestamp) throws Exception {
    final InternalLogger log = ClientLogger.getLog();

    DefaultMQPullConsumer consumer = new DefaultMQPullConsumer(consumerGroup);
    consumer.setInstanceName(instanceName);
    consumer.setMessageModel(messageModel);
    consumer.start();

    Set<MessageQueue> mqs = null;
    try {
        mqs = consumer.fetchSubscribeMessageQueues(topic);
        if (mqs != null && !mqs.isEmpty()) {
            TreeSet<MessageQueue> mqsNew = new TreeSet<MessageQueue>(mqs);
            for (MessageQueue mq : mqsNew) {
                long offset = consumer.searchOffset(mq, timestamp);
                if (offset >= 0) {
                    consumer.updateConsumeOffset(mq, offset);
                    log.info("resetOffsetByTimestamp updateConsumeOffset success, {} {} {}",
                        consumerGroup, offset, mq);
                }
            }
        }
    } catch (Exception e) {
        log.warn("resetOffsetByTimestamp Exception", e);
        throw e;
    } finally {
        if (mqs != null) {
            consumer.getDefaultMQPullConsumerImpl().getOffsetStore().persistAll(mqs);
        }
        consumer.shutdown();
    }
}
 

@PostConstruct
public void init() {
    defaultMQPushConsumer =
            new DefaultMQPushConsumer(
                MessageProtocolConst.SECKILL_CHARGE_ORDER_TOPIC.getConsumerGroup(),
                    new AclClientRPCHook(new SessionCredentials(aclAccessKey, aclAccessSecret)),
                    // 平均分配队列算法，hash
                    new AllocateMessageQueueAveragely());
    defaultMQPushConsumer.setNamesrvAddr(namesrvConfig.nameSrvAddr());
    // 从头开始消费
    defaultMQPushConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);
    // 消费模式:集群模式
    // 集群：同一条消息 只会被一个消费者节点消费到
    // 广播：同一条消息 每个消费者都会消费到
    defaultMQPushConsumer.setMessageModel(MessageModel.CLUSTERING);
    // 注册监听器
    defaultMQPushConsumer.registerMessageListener(messageListener);
    // 设置每次拉取的消息量，默认为1
    defaultMQPushConsumer.setConsumeMessageBatchMaxSize(1);
    // 订阅所有消息
    try {
        defaultMQPushConsumer.subscribe(MessageProtocolConst.SECKILL_CHARGE_ORDER_TOPIC.getTopic(), "*");
        // 启动消费者
        defaultMQPushConsumer.start();
    } catch (MQClientException e) {
        LOGGER.error("[秒杀下单消费者]--SecKillChargeOrderConsumer加载异常!e={}", LogExceptionWapper.getStackTrace(e));
        throw new RuntimeException("[秒杀下单消费者]--SecKillChargeOrderConsumer加载异常!", e);
    }
    LOGGER.info("[秒杀下单消费者]--SecKillChargeOrderConsumer加载完成!");
}
 

/**
 * 构建 Consumer 配置
 *
 * @param props    Properties
 * @param consumer DefaultMQPushConsumer
 */
public static void buildConsumerConfigs(Properties props, DefaultMQPullConsumer consumer) {
    buildCommonConfigs(props, consumer);

    consumer.setMessageModel(MessageModel.CLUSTERING);

    consumer.setPersistConsumerOffsetInterval(getInteger(props,
            CONSUMER_OFFSET_PERSIST_INTERVAL, DEFAULT_CONSUMER_OFFSET_PERSIST_INTERVAL));
}
 

/**
 * 移除不需要的队列相关的信息
 * 1. 持久化消费进度，并移除之
 * 2. 顺序消费&集群模式，解锁对该队列的锁定
 *
 * @param mq 消息队列
 * @param pq 消息处理队列
 * @return 是否移除成功
 */
@Override
public boolean removeUnnecessaryMessageQueue(MessageQueue mq, ProcessQueue pq) {
    // 同步队列的消费进度，并移除之。
    this.defaultMQPushConsumerImpl.getOffsetStore().persist(mq);
    this.defaultMQPushConsumerImpl.getOffsetStore().removeOffset(mq);
    // 集群模式下，顺序消费移除时，解锁对队列的锁定
    if (this.defaultMQPushConsumerImpl.isConsumeOrderly() && MessageModel.CLUSTERING.equals(this.defaultMQPushConsumerImpl.messageModel())) {
        try {
            // 有序消费模式下,ConsumeMessageOrderlyService在执行 MessageListenerOrderly#consumeMessage() 时会锁定ProcessQueue,
            // 执行此方法完后才会释放ProcessQueue的Lock,见ConsumeMessageOrderlyService#464 行
            // 锁定不成功说明在长达1S的时间内一直都在执行 MessageListenerOrderly#consumeMessage 方法
            // 也就是说 consumeMessage() 方法执行时间超过1S,每条消息的消费速度以及方法参数List<MessageExt>的大小可能存在问题
            // 此时只能等待相应MessageQueue在Broker的锁定时间过期(1m)后再被其他消费者锁定
            if (pq.getLockConsume().tryLock(1000, TimeUnit.MILLISECONDS)) {
                try {
                    return this.unlockDelay(mq, pq);
                } finally {
                    pq.getLockConsume().unlock();
                }
            } else {
                log.warn("[WRONG]mq is consuming, so can not unlock it, {}. maybe hanged for a while, {}", mq, pq.getTryUnlockTimes());
                pq.incTryUnlockTimes();
            }
        } catch (Exception e) {
            log.error("removeUnnecessaryMessageQueue Exception", e);
        }

        return false;
    }
    return true;
}
 

/**
 * Reset consumer topic offset according to time
 *
 * @param messageModel which model
 * @param instanceName which instance
 * @param consumerGroup consumer group
 * @param topic topic
 * @param timestamp time
 */
public static void resetOffsetByTimestamp(
    final MessageModel messageModel,
    final String instanceName,
    final String consumerGroup,
    final String topic,
    final long timestamp) throws Exception {
    final Logger log = ClientLogger.getLog();

    DefaultMQPullConsumer consumer = new DefaultMQPullConsumer(consumerGroup);
    consumer.setInstanceName(instanceName);
    consumer.setMessageModel(messageModel);
    consumer.start();

    Set<MessageQueue> mqs = null;
    try {
        mqs = consumer.fetchSubscribeMessageQueues(topic);
        if (mqs != null && !mqs.isEmpty()) {
            TreeSet<MessageQueue> mqsNew = new TreeSet<MessageQueue>(mqs);
            for (MessageQueue mq : mqsNew) {
                long offset = consumer.searchOffset(mq, timestamp);
                if (offset >= 0) {
                    consumer.updateConsumeOffset(mq, offset);
                    log.info("resetOffsetByTimestamp updateConsumeOffset success, {} {} {}",
                        consumerGroup, offset, mq);
                }
            }
        }
    } catch (Exception e) {
        log.warn("resetOffsetByTimestamp Exception", e);
        throw e;
    } finally {
        if (mqs != null) {
            consumer.getDefaultMQPullConsumerImpl().getOffsetStore().persistAll(mqs);
        }
        consumer.shutdown();
    }
}
 

public void shutdown() {
    this.stopped = true;
    if (scheduledFuture != null) {
        scheduledFuture.cancel(false);
    }
    if (this.defaultMQPushConsumer.getSharedThreadPoolExecutor() == null) {
        this.consumeExecutor.shutdown();
    }
    if (this.defaultMQPushConsumer.getSharedScheduledExecutorService() == null) {
        this.scheduledExecutorService.shutdown();
    }
    if (MessageModel.CLUSTERING.equals(this.defaultMQPushConsumerImpl.messageModel())) {
        this.unlockAllMQ();
    }
}
 

public boolean updateChannel(final ClientChannelInfo infoNew, ConsumeType consumeType,
    MessageModel messageModel, ConsumeFromWhere consumeFromWhere) {
    boolean updated = false;
    this.consumeType = consumeType;
    this.messageModel = messageModel;
    this.consumeFromWhere = consumeFromWhere;

    ClientChannelInfo infoOld = this.channelInfoTable.get(infoNew.getChannel());
    if (null == infoOld) {
        ClientChannelInfo prev = this.channelInfoTable.put(infoNew.getChannel(), infoNew);
        if (null == prev) {
            log.info("new consumer connected, group: {} {} {} channel: {}", this.groupName, consumeType,
                messageModel, infoNew.toString());
            updated = true;
        }

        infoOld = infoNew;
    } else {
        if (!infoOld.getClientId().equals(infoNew.getClientId())) {
            log.error("[BUG] consumer channel exist in broker, but clientId not equal. GROUP: {} OLD: {} NEW: {} ",
                this.groupName,
                infoOld.toString(),
                infoNew.toString());
            this.channelInfoTable.put(infoNew.getChannel(), infoNew);
        }
    }

    this.lastUpdateTimestamp = System.currentTimeMillis();
    infoOld.setLastUpdateTimestamp(this.lastUpdateTimestamp);

    return updated;
}
 

public ConsumerGroupInfo(String groupName, ConsumeType consumeType, MessageModel messageModel,
    ConsumeFromWhere consumeFromWhere) {
    this.groupName = groupName;
    this.consumeType = consumeType;
    this.messageModel = messageModel;
    this.consumeFromWhere = consumeFromWhere;
}
 

/**
 * 构造函数
 * @param consumerGroup ;
 * @param messageModel ;
 * @param allocateMessageQueueStrategy ;
 * @param mQClientFactory ;
 */
public RebalanceImpl(String consumerGroup, MessageModel messageModel,
    AllocateMessageQueueStrategy allocateMessageQueueStrategy,
    MQClientInstance mQClientFactory) {

    this.consumerGroup = consumerGroup;
    this.messageModel = messageModel;
    this.allocateMessageQueueStrategy = allocateMessageQueueStrategy;
    this.mQClientFactory = mQClientFactory;
}