com.alibaba.rocketmq.common.message.MessageQueue Java Examples

public MessageQueue selectOneMessageQueue(final String lastBrokerName) {
    if (lastBrokerName == null) {
        return selectOneMessageQueue();
    } else {
        int index = this.sendWhichQueue.getAndIncrement();
        for (int i = 0; i < this.messageQueueList.size(); i++) {
            int pos = Math.abs(index++) % this.messageQueueList.size();
            if (pos < 0)
                pos = 0;
            MessageQueue mq = this.messageQueueList.get(pos);
            if (!mq.getBrokerName().equals(lastBrokerName)) {
                return mq;
            }
        }
        return selectOneMessageQueue();
    }
}
 

@Override
public void persistAll(Set<MessageQueue> mqs) {
    if (null == mqs || mqs.isEmpty())
        return;

    OffsetSerializeWrapper offsetSerializeWrapper = new OffsetSerializeWrapper();
    for (MessageQueue mq : this.offsetTable.keySet()) {
        if (mqs.contains(mq)) {
            AtomicLong offset = this.offsetTable.get(mq);
            offsetSerializeWrapper.getOffsetTable().put(mq, offset);
        }
    }

    String jsonString = offsetSerializeWrapper.toJson(true);
    if (jsonString != null) {
        try {
            MixAll.string2File(jsonString, this.storePath);
        }
        catch (IOException e) {
            log.error("persistAll consumer offset Exception, " + this.storePath, e);
        }
    }
}
 

@Test
public void testAllocate() {
    AllocateMessageQueueAveragely allocateMessageQueueAveragely = new AllocateMessageQueueAveragely();
    String topic = "topic_test";
    String currentCID = "CID";
    int queueSize = 19;
    int consumerSize = 10;
    List<MessageQueue> mqAll = new ArrayList<MessageQueue>();
    for (int i = 0; i < queueSize; i++) {
        MessageQueue mq = new MessageQueue(topic, "brokerName", i);
        mqAll.add(mq);
    }

    List<String> cidAll = new ArrayList<String>();
    for (int j = 0; j < consumerSize; j++) {
        cidAll.add("CID" + j);
    }
    System.out.println(mqAll.toString());
    System.out.println(cidAll.toString());
    for (int i = 0; i < consumerSize; i++) {
        List<MessageQueue> rs = allocateMessageQueueAveragely.allocate("", currentCID + i, mqAll, cidAll);
        System.out.println("rs[" + currentCID + i + "]:" + rs.toString());
    }
}
 

private long fetchConsumeOffsetFromBroker(MessageQueue mq)
        throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
    FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(mq.getBrokerName());
    if (null == findBrokerResult) {
        // TODO Here may be heavily overhead for Name Server,need tuning
        this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic());
        findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(mq.getBrokerName());
    }

    if (findBrokerResult != null) {
        QueryConsumerOffsetRequestHeader requestHeader = new QueryConsumerOffsetRequestHeader();
        requestHeader.setTopic(mq.getTopic());
        requestHeader.setConsumerGroup(this.groupName);
        requestHeader.setQueueId(mq.getQueueId());

        return this.mQClientFactory.getMQClientAPIImpl()
            .queryConsumerOffset(findBrokerResult.getBrokerAddr(), requestHeader, 1000 * 5);
    }
    else {
        throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null);
    }
}
 

/**
 * Update the Consumer Offset, once the Master is off, updated to Slave,
 * here need to be optimized.
 */
private void updateConsumeOffsetToBroker(MessageQueue mq, long offset) throws RemotingException,
        MQBrokerException, InterruptedException, MQClientException {
    FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(mq.getBrokerName());
    if (null == findBrokerResult) {
        // TODO Here may be heavily overhead for Name Server,need tuning
        this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic());
        findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(mq.getBrokerName());
    }

    if (findBrokerResult != null) {
        UpdateConsumerOffsetRequestHeader requestHeader = new UpdateConsumerOffsetRequestHeader();
        requestHeader.setTopic(mq.getTopic());
        requestHeader.setConsumerGroup(this.groupName);
        requestHeader.setQueueId(mq.getQueueId());
        requestHeader.setCommitOffset(offset);

        this.mQClientFactory.getMQClientAPIImpl().updateConsumerOffsetOneway(
                findBrokerResult.getBrokerAddr(), requestHeader, 1000 * 5);
    } else {
        throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null);
    }
}
 

private boolean isLocked(final String group, final MessageQueue mq, final String clientId) {
    ConcurrentHashMap<MessageQueue, LockEntry> groupValue = this.mqLockTable.get(group);
    if (groupValue != null) {
        LockEntry lockEntry = groupValue.get(mq);
        if (lockEntry != null) {
            boolean locked = lockEntry.isLocked(clientId);
            if (locked) {
                lockEntry.setLastUpdateTimestamp(System.currentTimeMillis());
            }

            return locked;
        }
    }

    return false;
}
 

@Override
public void updateOffset(MessageQueue mq, long offset, boolean increaseOnly) {
    if (mq != null) {
        AtomicLong offsetOld = this.offsetTable.get(mq);
        if (null == offsetOld) {
            offsetOld = this.offsetTable.putIfAbsent(mq, new AtomicLong(offset));
        }

        if (null != offsetOld) {
            if (increaseOnly) {
                MixAll.compareAndIncreaseOnly(offsetOld, offset);
            } else {
                offsetOld.set(offset);
            }
        }
    }
}
 

/**
 * Update the Consumer Offset, once the Master is off, updated to Slave,
 * here need to be optimized.
 */
private void updateConsumeOffsetToBroker(MessageQueue mq, long offset)
        throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
    FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(mq.getBrokerName());
    if (null == findBrokerResult) {
        // TODO Here may be heavily overhead for Name Server,need tuning
        this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic());
        findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(mq.getBrokerName());
    }

    if (findBrokerResult != null) {
        UpdateConsumerOffsetRequestHeader requestHeader = new UpdateConsumerOffsetRequestHeader();
        requestHeader.setTopic(mq.getTopic());
        requestHeader.setConsumerGroup(this.groupName);
        requestHeader.setQueueId(mq.getQueueId());
        requestHeader.setCommitOffset(offset);

        this.mQClientFactory.getMQClientAPIImpl()
            .updateConsumerOffsetOneway(findBrokerResult.getBrokerAddr(), requestHeader, 1000 * 5);
    }
    else {
        throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null);
    }
}
 

public synchronized boolean lockOneMQ(final MessageQueue mq) {
    if (!this.stopped) {
        return this.defaultMQPushConsumerImpl.getRebalanceImpl().lock(mq);
    }

    return false;
}
 

public static void main(String[] args) throws MQClientException {
    final MQPullConsumerScheduleService scheduleService = new MQPullConsumerScheduleService("GroupName1");

    scheduleService.setMessageModel(MessageModel.CLUSTERING);
    scheduleService.registerPullTaskCallback("TopicTest1", new PullTaskCallback() {

        @Override
        public void doPullTask(MessageQueue mq, PullTaskContext context) {
            MQPullConsumer consumer = context.getPullConsumer();
            try {
                long offset = consumer.fetchConsumeOffset(mq, false);
                if (offset < 0)
                    offset = 0;

                PullResult pullResult = consumer.pull(mq, "*", offset, 32);
                System.out.println(offset + "\t" + mq + "\t" + pullResult);
                switch (pullResult.getPullStatus()) {
                case FOUND:
                    break;
                case NO_MATCHED_MSG:
                    break;
                case NO_NEW_MSG:
                case OFFSET_ILLEGAL:
                    break;
                default:
                    break;
                }

                consumer.updateConsumeOffset(mq, pullResult.getNextBeginOffset());

                context.setPullNextDelayTimeMillis(100);
            }
            catch (Exception e) {
                e.printStackTrace();
            }
        }
    });

    scheduleService.start();
}
 

@Override
public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) {
    String srcGroup = commandLine.getOptionValue("s").trim();
    String destGroup = commandLine.getOptionValue("d").trim();
    String topic = commandLine.getOptionValue("t").trim();

    DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook);
    defaultMQAdminExt.setInstanceName("admin-" + Long.toString(System.currentTimeMillis()));

    try {
        defaultMQAdminExt.start();
        ConsumeStats consumeStats = defaultMQAdminExt.examineConsumeStats(srcGroup);
        Set<MessageQueue> mqs = consumeStats.getOffsetTable().keySet();
        if (mqs != null && !mqs.isEmpty()) {
            TopicRouteData topicRoute = defaultMQAdminExt.examineTopicRouteInfo(topic);
            for (MessageQueue mq : mqs) {
                String addr = null;
                for (BrokerData brokerData : topicRoute.getBrokerDatas()) {
                    if (brokerData.getBrokerName().equals(mq.getBrokerName())) {
                        addr = brokerData.selectBrokerAddr();
                        break;
                    }
                }
                long offset = consumeStats.getOffsetTable().get(mq).getBrokerOffset();
                if (offset >= 0) {
                    defaultMQAdminExt.updateConsumeOffset(addr, destGroup, mq, offset);
                }
            }
        }
        System.out.printf("clone group offset success. srcGroup[%s], destGroup=[%s], topic[%s]",
                srcGroup, destGroup, topic);
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        defaultMQAdminExt.shutdown();
    }
}
 

@Test
public void testConsumer3CurrentCID2() {
    currentCID = "2";
    createConsumerIdList(3);
    createMessageQueueList(5);
    List<MessageQueue> result =
            allocateMessageQueueAveragely.allocate("", currentCID, messageQueueList, consumerIdList);
    printMessageQueue(result, "testConsumer3CurrentCID2");
    Assert.assertEquals(result.size(), 3);
    Assert.assertEquals(result.containsAll(getMessageQueueList().subList(2, 5)), true);
}
 

public void printMessageQueue(List<MessageQueue> messageQueueList, String name) {
    if (messageQueueList == null || messageQueueList.size() < 1)
        return;
    System.out.println(name + ".......................................start");
    for (MessageQueue messageQueue : messageQueueList) {
        System.out.println(messageQueue);
    }
    System.out.println(name + ".......................................end");
}
 

private static long getMessageQueueOffset(MessageQueue mq) {
    Long offset = offseTable.get(mq);
    if (offset != null)
        return offset;

    return 0;
}
 

public static Set<MessageQueue> topicRouteData2TopicSubscribeInfo(final String topic, final TopicRouteData route) {
    Set<MessageQueue> mqList = new HashSet<MessageQueue>();
    List<QueueData> qds = route.getQueueDatas();
    for (QueueData qd : qds) {
        if (PermName.isReadable(qd.getPerm())) {
            for (int i = 0; i < qd.getReadQueueNums(); i++) {
                MessageQueue mq = new MessageQueue(topic, qd.getBrokerName(), i);
                mqList.add(mq);
            }
        }
    }

    return mqList;
}
 

@Override
public boolean removeUnnecessaryMessageQueue(MessageQueue mq, ProcessQueue pq) {
    this.defaultMQPushConsumerImpl.getOffsetStore().persist(mq);
    this.defaultMQPushConsumerImpl.getOffsetStore().removeOffset(mq);
    if (this.defaultMQPushConsumerImpl.isConsumeOrderly()
            && MessageModel.CLUSTERING.equals(this.defaultMQPushConsumerImpl.messageModel())) {
        try {
            if (pq.getLockConsume().tryLock(1000, TimeUnit.MILLISECONDS)) {
                try {
                    this.unlock(mq, true);
                    return true;
                }
                finally {
                    pq.getLockConsume().unlock();
                }
            }
            else {
                log.warn("[WRONG]mq is consuming, so can not unlock it, {}. maybe hanged for a while, {}", //
                    mq, //
                    pq.getTryUnlockTimes());

                pq.incTryUnlockTimes();
            }
        }
        catch (Exception e) {
            log.error("removeUnnecessaryMessageQueue Exception", e);
        }

        return false;
    }
    return true;
}
 

public long commitableOffset(final MessageQueue mq) {
    CachedQueue cachedQueue = this.mqCachedTable.get(mq);
    if (null != cachedQueue) {
        return cachedQueue.getMsgCachedTable().firstKey();
    }

    return -1;
}
 

@Test
public void testConsumer3CurrentCID2() {
    currentCID = "2";
    createConsumerIdList(3);
    createMessageQueueList(5);
    List<MessageQueue> result =
            allocateMessageQueueAveragely.allocate("", currentCID, messageQueueList, consumerIdList);
    printMessageQueue(result, "testConsumer3CurrentCID2");
    Assert.assertEquals(result.size(), 3);
    Assert.assertEquals(result.containsAll(getMessageQueueList().subList(2, 5)), true);
}
 

public static Set<MessageQueue> topicRouteData2TopicSubscribeInfo(final String topic,
                                                                  final TopicRouteData route) {
    Set<MessageQueue> mqList = new HashSet<MessageQueue>();
    List<QueueData> qds = route.getQueueDatas();
    for (QueueData qd : qds) {
        if (PermName.isReadable(qd.getPerm())) {
            for (int i = 0; i < qd.getReadQueueNums(); i++) {
                MessageQueue mq = new MessageQueue(topic, qd.getBrokerName(), i);
                mqList.add(mq);
            }
        }
    }

    return mqList;
}
 

@Override
public void submitConsumeRequest(//
        final List<MessageExt> msgs, //
        final ProcessQueue processQueue, //
        final MessageQueue messageQueue, //
        final boolean dispathToConsume) {
    if (dispathToConsume) {
        ConsumeRequest consumeRequest = new ConsumeRequest(processQueue, messageQueue);
        this.consumeExecutor.submit(consumeRequest);
    }
}
 

@Override
public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) {
    DefaultMQAdminExt adminExt = new DefaultMQAdminExt(rpcHook);
    adminExt.setInstanceName(Long.toString(System.currentTimeMillis()));
    try {
        adminExt.start();

        String topic = commandLine.getOptionValue('t').trim();
        String ips = commandLine.getOptionValue('i').trim();
        final String[] split = ips.split(",");
        final List<String> ipList = new LinkedList<String>();
        for (String ip : split) {
            ipList.add(ip);
        }

        final TopicRouteData topicRouteData = adminExt.examineTopicRouteInfo(topic);
        final Set<MessageQueue> mqs = MQClientInstance.topicRouteData2TopicSubscribeInfo(topic, topicRouteData);

        final AllocateMessageQueueAveragely averagely = new AllocateMessageQueueAveragely();


        RebalanceResult rr = new RebalanceResult();

        for (String i : ipList) {
            final List<MessageQueue> mqResult = averagely.allocate("aa", i, new ArrayList<MessageQueue>(mqs), ipList);
            rr.getResult().put(i, mqResult);
        }

        final String json = RemotingSerializable.toJson(rr, false);
        System.out.println(json);
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        adminExt.shutdown();
    }
}
 

public MessageQueue selectOneMessageQueue() {
    int index = this.sendWhichQueue.getAndIncrement();
    int pos = Math.abs(index) % this.messageQueueList.size();
    if (pos < 0)
        pos = 0;
    return this.messageQueueList.get(pos);
}
 

public Object fetchLockObject(final MessageQueue mq) {
    Object objLock = this.mqLockTable.get(mq);
    if (null == objLock) {
        objLock = new Object();
        Object prevLock = this.mqLockTable.putIfAbsent(mq, objLock);
        if (prevLock != null) {
            objLock = prevLock;
        }
    }

    return objLock;
}
 

/**
 * 获取topic状态相关信息
 * 
 * @param addr
 * @param topic
 * @param timeoutMillis
 * @return
 * @throws InterruptedException
 * @throws RemotingTimeoutException
 * @throws RemotingSendRequestException
 * @throws RemotingConnectException
 * @throws MQBrokerException
 */
public TopicStatsTable getTopicStatsInfo(final String addr, final String topic, final long timeoutMillis)
        throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException,
        RemotingConnectException, MQBrokerException {
    String topicWithProjectGroup = topic;
    if (!UtilAll.isBlank(projectGroupPrefix)) {
        topicWithProjectGroup = VirtualEnvUtil.buildWithProjectGroup(topic, projectGroupPrefix);
    }

    GetTopicStatsInfoRequestHeader requestHeader = new GetTopicStatsInfoRequestHeader();
    requestHeader.setTopic(topicWithProjectGroup);

    RemotingCommand request =
            RemotingCommand.createRequestCommand(RequestCode.GET_TOPIC_STATS_INFO, requestHeader);

    RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis);
    switch (response.getCode()) {
    case ResponseCode.SUCCESS: {
        TopicStatsTable topicStatsTable =
                TopicStatsTable.decode(response.getBody(), TopicStatsTable.class);
        if (!UtilAll.isBlank(projectGroupPrefix)) {
            HashMap<MessageQueue, TopicOffset> newTopicOffsetMap =
                    new HashMap<MessageQueue, TopicOffset>();
            for (Map.Entry<MessageQueue, TopicOffset> messageQueue : topicStatsTable.getOffsetTable()
                .entrySet()) {
                MessageQueue key = messageQueue.getKey();
                key.setTopic(VirtualEnvUtil.clearProjectGroup(key.getTopic(), projectGroupPrefix));
                newTopicOffsetMap.put(key, messageQueue.getValue());
            }
            topicStatsTable.setOffsetTable(newTopicOffsetMap);
        }
        return topicStatsTable;
    }
    default:
        break;
    }

    throw new MQBrokerException(response.getCode(), response.getRemark());
}
 

public Map<MessageQueue, Long> getOffsetTable() {
    return offsetTable;
}
 

@Override
public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) {
    DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook);
    defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis()));
    try {
        String group = commandLine.getOptionValue("g").trim();
        String topic = commandLine.getOptionValue("t").trim();
        String originClientId = "";
        if (commandLine.hasOption("i")) {
            originClientId = commandLine.getOptionValue("i").trim();
        }
        defaultMQAdminExt.start();

        Map<String, Map<MessageQueue, Long>> consumerStatusTable =
                defaultMQAdminExt.getConsumeStatus(topic, group, originClientId);
        System.out.printf("get consumer status from client. group=%s, topic=%s, originClientId=%s%n",
                group, topic, originClientId);

        System.out.printf("%-50s  %-15s  %-15s  %-20s%n",//
                "#clientId",//
                "#brokerName", //
                "#queueId",//
                "#offset");

        for(Map.Entry<String, Map<MessageQueue, Long>> entry: consumerStatusTable.entrySet()){
            String clientId = entry.getKey();
            Map<MessageQueue, Long> mqTable = entry.getValue();
            for(Map.Entry<MessageQueue,Long> entry1: mqTable.entrySet()){
                MessageQueue mq = entry1.getKey();
                System.out.printf("%-50s  %-15s  %-15d  %-20d%n",//
                        UtilAll.frontStringAtLeast(clientId, 50),//
                        mq.getBrokerName(),//
                        mq.getQueueId(),//
                        mqTable.get(mq));
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        defaultMQAdminExt.shutdown();
    }
}
 

public void setMessageQueue(MessageQueue messageQueue) {
    this.messageQueue = messageQueue;
}
 

@Override
public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException {
    return this.defaultMQPushConsumerImpl.earliestMsgStoreTime(mq);
}
 

public List<MessageQueue> getMessageQueueList() {
    return messageQueueList;
}
 

public MessageQueue getMq() {
    return mq;
}