org.apache.rocketmq.store.schedule.ScheduleMessageService Java Examples

The following examples show how to use org.apache.rocketmq.store.schedule.ScheduleMessageService. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DefaultMessageStore.java    From DDMQ with Apache License 2.0 6 votes vote down vote up 
@Override
public int cleanUnusedTopic(Set<String> topics) {
    Iterator<Entry<String, ConcurrentMap<Integer, ConsumeQueue>>> it = this.consumeQueueTable.entrySet().iterator();
    while (it.hasNext()) {
        Entry<String, ConcurrentMap<Integer, ConsumeQueue>> next = it.next();
        String topic = next.getKey();

        if (!topics.contains(topic) && !topic.equals(ScheduleMessageService.SCHEDULE_TOPIC)) {
            ConcurrentMap<Integer, ConsumeQueue> queueTable = next.getValue();
            for (ConsumeQueue cq : queueTable.values()) {
                cq.destroy();
                log.info("cleanUnusedTopic: {} {} ConsumeQueue cleaned",
                    cq.getTopic(),
                    cq.getQueueId()
                );

                this.commitLog.removeQueueFromTopicQueueTable(cq.getTopic(), cq.getQueueId());
            }
            it.remove();

            log.info("cleanUnusedTopic: {},topic destroyed", topic);
        }
    }

    return 0;
}
Example #2
Source File: DefaultMessageStore.java    From DDMQ with Apache License 2.0 6 votes vote down vote up 
@Override
public int cleanUnusedTopic(Set<String> topics) {
    Iterator<Entry<String, ConcurrentMap<Integer, ConsumeQueue>>> it = this.consumeQueueTable.entrySet().iterator();
    while (it.hasNext()) {
        Entry<String, ConcurrentMap<Integer, ConsumeQueue>> next = it.next();
        String topic = next.getKey();

        if (!topics.contains(topic) && !topic.equals(ScheduleMessageService.SCHEDULE_TOPIC)) {
            ConcurrentMap<Integer, ConsumeQueue> queueTable = next.getValue();
            for (ConsumeQueue cq : queueTable.values()) {
                cq.destroy();
                log.info("cleanUnusedTopic: {} {} ConsumeQueue cleaned",
                    cq.getTopic(),
                    cq.getQueueId()
                );

                this.commitLog.removeQueueFromTopicQueueTable(cq.getTopic(), cq.getQueueId());
            }
            it.remove();

            log.info("cleanUnusedTopic: {},topic destroyed", topic);
        }
    }

    return 0;
}
Example #3
Source File: DefaultMessageStore.java    From rocketmq-all-4.1.0-incubating with Apache License 2.0 6 votes vote down vote up 
@Override
public int cleanUnusedTopic(Set<String> topics) {
    Iterator<Entry<String, ConcurrentMap<Integer, ConsumeQueue>>> it = this.consumeQueueTable.entrySet().iterator();
    while (it.hasNext()) {
        Entry<String, ConcurrentMap<Integer, ConsumeQueue>> next = it.next();
        String topic = next.getKey();

        if (!topics.contains(topic) && !topic.equals(ScheduleMessageService.SCHEDULE_TOPIC)) {
            ConcurrentMap<Integer, ConsumeQueue> queueTable = next.getValue();
            for (ConsumeQueue cq : queueTable.values()) {
                cq.destroy();
                log.info("cleanUnusedTopic: {} {} ConsumeQueue cleaned", //
                    cq.getTopic(), //
                    cq.getQueueId() //
                );

                this.commitLog.removeQueueFromTopicQueueTable(cq.getTopic(), cq.getQueueId());
            }
            it.remove();

            log.info("cleanUnusedTopic: {},topic destroyed", topic);
        }
    }

    return 0;
}
Example #4
Source File: DefaultMessageStore.java    From rocketmq with Apache License 2.0 6 votes vote down vote up 
@Override
public int cleanUnusedTopic(Set<String> topics) {
    Iterator<Entry<String, ConcurrentHashMap<Integer, ConsumeQueue>>> it = this.consumeQueueTable.entrySet().iterator();
    while (it.hasNext()) {
        Entry<String, ConcurrentHashMap<Integer, ConsumeQueue>> next = it.next();
        String topic = next.getKey();

        if (!topics.contains(topic) && !topic.equals(ScheduleMessageService.SCHEDULE_TOPIC)) {
            ConcurrentHashMap<Integer, ConsumeQueue> queueTable = next.getValue();
            for (ConsumeQueue cq : queueTable.values()) {
                cq.destroy();
                log.info("cleanUnusedTopic: {} {} ConsumeQueue cleaned", //
                    cq.getTopic(), //
                    cq.getQueueId() //
                );

                this.commitLog.removeQueueFromTopicQueueTable(cq.getTopic(), cq.getQueueId());
            }
            it.remove();

            log.info("cleanUnusedTopic: {},topic destroyed", topic);
        }
    }

    return 0;
}
Example #5
Source File: DefaultMessageStore.java    From rocketmq-read with Apache License 2.0 6 votes vote down vote up 
@Override
public int cleanUnusedTopic(Set<String> topics) {
    Iterator<Entry<String, ConcurrentMap<Integer, ConsumeQueue>>> it = this.consumeQueueTable.entrySet().iterator();
    while (it.hasNext()) {
        Entry<String, ConcurrentMap<Integer, ConsumeQueue>> next = it.next();
        String topic = next.getKey();

        if (!topics.contains(topic) && !topic.equals(ScheduleMessageService.SCHEDULE_TOPIC)) {
            ConcurrentMap<Integer, ConsumeQueue> queueTable = next.getValue();
            for (ConsumeQueue cq : queueTable.values()) {
                cq.destroy();
                log.info("cleanUnusedTopic: {} {} ConsumeQueue cleaned",
                    cq.getTopic(),
                    cq.getQueueId()
                );

                this.commitLog.removeQueueFromTopicQueueTable(cq.getTopic(), cq.getQueueId());
            }
            it.remove();

            log.info("cleanUnusedTopic: {},topic destroyed", topic);
        }
    }

    return 0;
}
Example #6
Source File: DefaultMessageStore.java    From rocketmq_trans_message with Apache License 2.0 6 votes vote down vote up 
@Override
public int cleanUnusedTopic(Set<String> topics) {
    Iterator<Entry<String, ConcurrentHashMap<Integer, ConsumeQueue>>> it = this.consumeQueueTable.entrySet().iterator();
    while (it.hasNext()) {
        Entry<String, ConcurrentHashMap<Integer, ConsumeQueue>> next = it.next();
        String topic = next.getKey();

        if (!topics.contains(topic) && !topic.equals(ScheduleMessageService.SCHEDULE_TOPIC)) {
            ConcurrentHashMap<Integer, ConsumeQueue> queueTable = next.getValue();
            for (ConsumeQueue cq : queueTable.values()) {
                cq.destroy();
                log.info("cleanUnusedTopic: {} {} ConsumeQueue cleaned", //
                        cq.getTopic(), //
                        cq.getQueueId() //
                );

                this.commitLog.removeQueueFromTopicQueueTable(cq.getTopic(), cq.getQueueId());
            }
            it.remove();

            log.info("cleanUnusedTopic: {},topic destroyed", topic);
        }
    }

    return 0;
}
Example #7
Source File: DefaultMessageStore.java    From rocketmq-4.3.0 with Apache License 2.0 5 votes vote down vote up 
@Override
    public int cleanUnusedTopic(Set<String> topics) {
//        遍历缓存的消息队列
        Iterator<Entry<String, ConcurrentMap<Integer, ConsumeQueue>>> it = this.consumeQueueTable.entrySet().iterator();
        while (it.hasNext()) {
            Entry<String, ConcurrentMap<Integer, ConsumeQueue>> next = it.next();
            String topic = next.getKey();

            if (!topics.contains(topic) && !topic.equals(ScheduleMessageService.SCHEDULE_TOPIC)) {
                ConcurrentMap<Integer, ConsumeQueue> queueTable = next.getValue();
                for (ConsumeQueue cq : queueTable.values()) {
//                    消费队列销毁=》
                    cq.destroy();
                    log.info("cleanUnusedTopic: {} {} ConsumeQueue cleaned",
                        cq.getTopic(),
                        cq.getQueueId()
                    );

//                    删除消息队列的offset=》
                    this.commitLog.removeQueueFromTopicQueueTable(cq.getTopic(), cq.getQueueId());
                }
//                topic所在的消息队列删除完毕后消费队列所在的集合元素删除
                it.remove();

                log.info("cleanUnusedTopic: {},topic destroyed", topic);
            }
        }

        return 0;
    }
Example #8
Source File: DefaultMessageStore.java    From rocketmq-4.3.0 with Apache License 2.0 5 votes vote down vote up 
public DefaultMessageStore(final MessageStoreConfig messageStoreConfig, final BrokerStatsManager brokerStatsManager,
    final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig) throws IOException {
    this.messageArrivingListener = messageArrivingListener;
    this.brokerConfig = brokerConfig;
    this.messageStoreConfig = messageStoreConfig;
    this.brokerStatsManager = brokerStatsManager;
    this.allocateMappedFileService = new AllocateMappedFileService(this);
    this.commitLog = new CommitLog(this);
    this.consumeQueueTable = new ConcurrentHashMap<>(32);

    this.flushConsumeQueueService = new FlushConsumeQueueService();
    this.cleanCommitLogService = new CleanCommitLogService();
    this.cleanConsumeQueueService = new CleanConsumeQueueService();
    this.storeStatsService = new StoreStatsService();
    this.indexService = new IndexService(this);
    this.haService = new HAService(this);

    this.reputMessageService = new ReputMessageService();

    this.scheduleMessageService = new ScheduleMessageService(this);

    this.transientStorePool = new TransientStorePool(messageStoreConfig);

    if (messageStoreConfig.isTransientStorePoolEnable()) {
        this.transientStorePool.init();
    }

    this.allocateMappedFileService.start();

    this.indexService.start();

    this.dispatcherList = new LinkedList<>();
    this.dispatcherList.addLast(new CommitLogDispatcherBuildConsumeQueue());
    this.dispatcherList.addLast(new CommitLogDispatcherBuildIndex());

    File file = new File(StorePathConfigHelper.getLockFile(messageStoreConfig.getStorePathRootDir()));
    MappedFile.ensureDirOK(file.getParent());
    lockFile = new RandomAccessFile(file, "rw");
}
Example #9
Source File: DefaultMessageStore.java    From rocketmq_trans_message with Apache License 2.0 5 votes vote down vote up 
public DefaultMessageStore(final MessageStoreConfig messageStoreConfig, final BrokerStatsManager brokerStatsManager,
                           final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig, final TransactionCheckExecuter transactionCheckExecuter) throws IOException {
    this.messageArrivingListener = messageArrivingListener;
    this.brokerConfig = brokerConfig;
    this.messageStoreConfig = messageStoreConfig;
    this.brokerStatsManager = brokerStatsManager;
    this.allocateMappedFileService = new AllocateMappedFileService(this);
    this.commitLog = new CommitLog(this);
    this.consumeQueueTable = new ConcurrentHashMap<>(32);
    this.transactionCheckExecuter = transactionCheckExecuter;

    this.flushConsumeQueueService = new FlushConsumeQueueService();
    this.cleanCommitLogService = new CleanCommitLogService();
    this.cleanConsumeQueueService = new CleanConsumeQueueService();
    this.storeStatsService = new StoreStatsService();
    this.indexService = new IndexService(this);
    this.haService = new HAService(this);


    this.reputMessageService = new ReputMessageService();

    this.scheduleMessageService = new ScheduleMessageService(this);
    //只有持久化DB的才启动
    if (DefaultMessageStore.this.getMessageStoreConfig().isTransactionLogInDB()) {
        this.transactionStateService = new TransactionStateDBService(this);
    } else {
        this.transactionStateService = new TransactionStateFileService(this);
    }

    this.transientStorePool = new TransientStorePool(messageStoreConfig);

    if (messageStoreConfig.isTransientStorePoolEnable()) {
        this.transientStorePool.init();
    }

    this.allocateMappedFileService.start();

    this.indexService.start();
}
Example #10
Source File: DefaultMessageStore.java    From rocketmq with Apache License 2.0 5 votes vote down vote up 
public DefaultMessageStore(final MessageStoreConfig messageStoreConfig, final BrokerStatsManager brokerStatsManager,
                           final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig) throws IOException {
    this.messageArrivingListener = messageArrivingListener;
    this.brokerConfig = brokerConfig;
    this.messageStoreConfig = messageStoreConfig;
    this.brokerStatsManager = brokerStatsManager;
    this.allocateMappedFileService = new AllocateMappedFileService(this);
    this.commitLog = new CommitLog(this);
    this.consumeQueueTable = new ConcurrentHashMap<>(32);

    this.flushConsumeQueueService = new FlushConsumeQueueService();
    this.cleanCommitLogService = new CleanCommitLogService();
    this.cleanConsumeQueueService = new CleanConsumeQueueService();
    this.storeStatsService = new StoreStatsService();
    this.indexService = new IndexService(this);
    this.haService = new HAService(this);

    this.reputMessageService = new ReputMessageService();

    this.scheduleMessageService = new ScheduleMessageService(this);

    this.transientStorePool = new TransientStorePool(messageStoreConfig);

    if (messageStoreConfig.isTransientStorePoolEnable()) {
        this.transientStorePool.init();
    }

    this.allocateMappedFileService.start();

    this.indexService.start();
}
Example #11
Source File: DefaultMessageStore.java    From DDMQ with Apache License 2.0 5 votes vote down vote up 
public DefaultMessageStore(final MessageStoreConfig messageStoreConfig, final BrokerStatsManager brokerStatsManager,
    final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig) throws IOException {
    this.messageArrivingListener = messageArrivingListener;
    this.brokerConfig = brokerConfig;
    this.messageStoreConfig = messageStoreConfig;
    this.brokerStatsManager = brokerStatsManager;
    this.allocateMappedFileService = new AllocateMappedFileService(this);
    this.commitLog = new CommitLog(this);
    this.consumeQueueTable = new ConcurrentHashMap<>(32);

    this.flushConsumeQueueService = new FlushConsumeQueueService();
    this.cleanCommitLogService = new CleanCommitLogService();
    this.cleanConsumeQueueService = new CleanConsumeQueueService();
    this.storeStatsService = new StoreStatsService();
    this.indexService = new IndexService(this);
    this.haService = new HAService(this);

    this.reputMessageService = new ReputMessageService();

    this.scheduleMessageService = new ScheduleMessageService(this);

    this.transientStorePool = new TransientStorePool(messageStoreConfig);

    if (messageStoreConfig.isTransientStorePoolEnable()) {
        this.transientStorePool.init();
    }

    this.allocateMappedFileService.start();

    this.indexService.start();

    this.dispatcherList = new LinkedList<>();
    this.dispatcherList.addLast(new CommitLogDispatcherBuildConsumeQueue());
    this.dispatcherList.addLast(new CommitLogDispatcherBuildIndex());

    File file = new File(StorePathConfigHelper.getLockFile(messageStoreConfig.getStorePathRootDir()));
    MappedFile.ensureDirOK(file.getParent());
    lockFile = new RandomAccessFile(file, "rw");
}
Example #12
Source File: DefaultMessageStore.java    From DDMQ with Apache License 2.0 5 votes vote down vote up 
public DefaultMessageStore(final MessageStoreConfig messageStoreConfig, final BrokerStatsManager brokerStatsManager,
    final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig) throws IOException {
    this.messageArrivingListener = messageArrivingListener;
    this.brokerConfig = brokerConfig;
    this.messageStoreConfig = messageStoreConfig;
    this.brokerStatsManager = brokerStatsManager;
    this.allocateMappedFileService = new AllocateMappedFileService(this);
    this.commitLog = new CommitLog(this);
    this.consumeQueueTable = new ConcurrentHashMap<>(32);

    this.flushConsumeQueueService = new FlushConsumeQueueService();
    this.cleanCommitLogService = new CleanCommitLogService();
    this.cleanConsumeQueueService = new CleanConsumeQueueService();
    this.storeStatsService = new StoreStatsService();
    this.indexService = new IndexService(this);
    this.haService = new HAService(this);

    this.reputMessageService = new ReputMessageService();

    this.scheduleMessageService = new ScheduleMessageService(this);

    this.transientStorePool = new TransientStorePool(messageStoreConfig);

    if (messageStoreConfig.isTransientStorePoolEnable()) {
        this.transientStorePool.init();
    }

    this.allocateMappedFileService.start();

    this.indexService.start();

    this.dispatcherList = new LinkedList<>();
    this.dispatcherList.addLast(new CommitLogDispatcherBuildConsumeQueue());
    this.dispatcherList.addLast(new CommitLogDispatcherBuildIndex());

    File file = new File(StorePathConfigHelper.getLockFile(messageStoreConfig.getStorePathRootDir()));
    MappedFile.ensureDirOK(file.getParent());
    lockFile = new RandomAccessFile(file, "rw");
}
Example #13
Source File: DefaultMessageStore.java    From rocketmq-all-4.1.0-incubating with Apache License 2.0 5 votes vote down vote up 
public DefaultMessageStore(final MessageStoreConfig messageStoreConfig, final BrokerStatsManager brokerStatsManager,
    final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig) throws IOException {
    this.messageArrivingListener = messageArrivingListener;
    this.brokerConfig = brokerConfig;
    this.messageStoreConfig = messageStoreConfig;
    this.brokerStatsManager = brokerStatsManager;
    this.allocateMappedFileService = new AllocateMappedFileService(this);
    this.commitLog = new CommitLog(this);
    this.consumeQueueTable = new ConcurrentHashMap<>(32);

    this.flushConsumeQueueService = new FlushConsumeQueueService();
    this.cleanCommitLogService = new CleanCommitLogService();
    this.cleanConsumeQueueService = new CleanConsumeQueueService();
    this.storeStatsService = new StoreStatsService();
    this.indexService = new IndexService(this);
    this.haService = new HAService(this);

    this.reputMessageService = new ReputMessageService();

    this.scheduleMessageService = new ScheduleMessageService(this);

    this.transientStorePool = new TransientStorePool(messageStoreConfig);

    if (messageStoreConfig.isTransientStorePoolEnable()) {
        this.transientStorePool.init();
    }

    // load过程依赖此服务，所以提前启动
    this.allocateMappedFileService.start(); //MapedFile对象服务,执行线程run方法

    // 因为下面的recover会分发请求到索引服务，如果不启动，分发过程会被流控
    this.indexService.start(); //消息索引服务 空方法其实啥没做

    this.dispatcherList = new LinkedList<>();
    this.dispatcherList.addLast(new CommitLogDispatcherBuildConsumeQueue());
    this.dispatcherList.addLast(new CommitLogDispatcherBuildIndex());
}
Example #14
Source File: DefaultMessageStore.java    From rocketmq_trans_message with Apache License 2.0 4 votes vote down vote up 
public ScheduleMessageService getScheduleMessageService() {
    return scheduleMessageService;
}
Example #15
Source File: DefaultMessageStore.java    From rocketmq_trans_message with Apache License 2.0 4 votes vote down vote up 
public void cleanExpiredConsumerQueue() {
    long minCommitLogOffset = this.commitLog.getMinOffset();

    Iterator<Entry<String, ConcurrentHashMap<Integer, ConsumeQueue>>> it = this.consumeQueueTable.entrySet().iterator();
    while (it.hasNext()) {
        Entry<String, ConcurrentHashMap<Integer, ConsumeQueue>> next = it.next();
        String topic = next.getKey();
        if (!topic.equals(ScheduleMessageService.SCHEDULE_TOPIC)) {
            ConcurrentHashMap<Integer, ConsumeQueue> queueTable = next.getValue();
            Iterator<Entry<Integer, ConsumeQueue>> itQT = queueTable.entrySet().iterator();
            while (itQT.hasNext()) {
                Entry<Integer, ConsumeQueue> nextQT = itQT.next();
                long maxCLOffsetInConsumeQueue = nextQT.getValue().getLastOffset();

                if (maxCLOffsetInConsumeQueue == -1) {
                    log.warn("maybe ConsumeQueue was created just now. topic={} queueId={} maxPhysicOffset={} minLogicOffset={}.", //
                            nextQT.getValue().getTopic(), //
                            nextQT.getValue().getQueueId(), //
                            nextQT.getValue().getMaxPhysicOffset(), //
                            nextQT.getValue().getMinLogicOffset());
                } else if (maxCLOffsetInConsumeQueue < minCommitLogOffset) {
                    log.info(
                            "cleanExpiredConsumerQueue: {} {} consumer queue destroyed, minCommitLogOffset: {} maxCLOffsetInConsumeQueue: {}", //
                            topic, //
                            nextQT.getKey(), //
                            minCommitLogOffset, //
                            maxCLOffsetInConsumeQueue);

                    DefaultMessageStore.this.commitLog.removeQueueFromTopicQueueTable(nextQT.getValue().getTopic(),
                            nextQT.getValue().getQueueId());

                    nextQT.getValue().destroy();
                    itQT.remove();
                }
            }

            if (queueTable.isEmpty()) {
                log.info("cleanExpiredConsumerQueue: {},topic destroyed", topic);
                it.remove();
            }
        }
    }
}
Example #16
Source File: DefaultMessageStore.java    From rocketmq-all-4.1.0-incubating with Apache License 2.0 4 votes vote down vote up 
public ScheduleMessageService getScheduleMessageService() {
    return scheduleMessageService;
}
Example #17
Source File: DefaultMessageStore.java    From rocketmq with Apache License 2.0 4 votes vote down vote up 
public DefaultMessageStore(final MessageStoreConfig messageStoreConfig, final BrokerStatsManager brokerStatsManager,
    final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig) throws IOException {
    this.messageArrivingListener = messageArrivingListener;
    this.brokerConfig = brokerConfig;
    this.messageStoreConfig = messageStoreConfig;
    this.brokerStatsManager = brokerStatsManager;
    this.allocateMappedFileService = new AllocateMappedFileService(this);
    if (messageStoreConfig.isEnableDLegerCommitLog()) {
        this.commitLog = new DLedgerCommitLog(this);
    } else {
        this.commitLog = new CommitLog(this);
    }
    this.consumeQueueTable = new ConcurrentHashMap<>(32);

    this.flushConsumeQueueService = new FlushConsumeQueueService();
    this.cleanCommitLogService = new CleanCommitLogService();
    this.cleanConsumeQueueService = new CleanConsumeQueueService();
    this.storeStatsService = new StoreStatsService();
    this.indexService = new IndexService(this);
    if (!messageStoreConfig.isEnableDLegerCommitLog()) {
        this.haService = new HAService(this);
    } else {
        this.haService = null;
    }
    this.reputMessageService = new ReputMessageService();

    this.scheduleMessageService = new ScheduleMessageService(this);

    this.transientStorePool = new TransientStorePool(messageStoreConfig);

    if (messageStoreConfig.isTransientStorePoolEnable()) {
        this.transientStorePool.init();
    }

    this.allocateMappedFileService.start();

    this.indexService.start();

    this.dispatcherList = new LinkedList<>();
    this.dispatcherList.addLast(new CommitLogDispatcherBuildConsumeQueue());
    this.dispatcherList.addLast(new CommitLogDispatcherBuildIndex());

    File file = new File(StorePathConfigHelper.getLockFile(messageStoreConfig.getStorePathRootDir()));
    MappedFile.ensureDirOK(file.getParent());
    lockFile = new RandomAccessFile(file, "rw");
}
Example #18
Source File: DefaultMessageStore.java    From rocketmq-all-4.1.0-incubating with Apache License 2.0 4 votes vote down vote up 
public void cleanExpiredConsumerQueue() {
    // CommitLog的最小Offset
    long minCommitLogOffset = this.commitLog.getMinOffset();

    Iterator<Entry<String, ConcurrentMap<Integer, ConsumeQueue>>> it = this.consumeQueueTable.entrySet().iterator();
    while (it.hasNext()) {
        Entry<String, ConcurrentMap<Integer, ConsumeQueue>> next = it.next();
        String topic = next.getKey();
        if (!topic.equals(ScheduleMessageService.SCHEDULE_TOPIC)) {
            ConcurrentMap<Integer, ConsumeQueue> queueTable = next.getValue();
            Iterator<Entry<Integer, ConsumeQueue>> itQT = queueTable.entrySet().iterator();
            while (itQT.hasNext()) {
                Entry<Integer, ConsumeQueue> nextQT = itQT.next();
                long maxCLOffsetInConsumeQueue = nextQT.getValue().getLastOffset();

                // maxCLOffsetInConsumeQueue==-1有可能正好是索引文件刚好创建的那一时刻,此时不清除数据
                if (maxCLOffsetInConsumeQueue == -1) {
                    log.warn("maybe ConsumeQueue was created just now. topic={} queueId={} maxPhysicOffset={} minLogicOffset={}.", //
                        nextQT.getValue().getTopic(), //
                        nextQT.getValue().getQueueId(), //
                        nextQT.getValue().getMaxPhysicOffset(), //
                        nextQT.getValue().getMinLogicOffset());
                } else if (maxCLOffsetInConsumeQueue < minCommitLogOffset) {
                    log.info(
                        "cleanExpiredConsumerQueue: {} {} consumer queue destroyed, minCommitLogOffset: {} maxCLOffsetInConsumeQueue: {}", //
                        topic, //
                        nextQT.getKey(), //
                        minCommitLogOffset, //
                        maxCLOffsetInConsumeQueue);

                    DefaultMessageStore.this.commitLog.removeQueueFromTopicQueueTable(nextQT.getValue().getTopic(),
                        nextQT.getValue().getQueueId());

                    nextQT.getValue().destroy();
                    itQT.remove();
                }
            }

            if (queueTable.isEmpty()) {
                log.info("cleanExpiredConsumerQueue: {},topic destroyed", topic);
                it.remove();
            }
        }
    }
}
Example #19
Source File: DefaultMessageStore.java    From rocketmq with Apache License 2.0 4 votes vote down vote up 
public ScheduleMessageService getScheduleMessageService() {
    return scheduleMessageService;
}
Example #20
Source File: DefaultMessageStore.java    From DDMQ with Apache License 2.0 4 votes vote down vote up 
public ScheduleMessageService getScheduleMessageService() {
    return scheduleMessageService;
}
Example #21
Source File: DefaultMessageStore.java    From DDMQ with Apache License 2.0 4 votes vote down vote up 
public void cleanExpiredConsumerQueue() {
    long minCommitLogOffset = this.commitLog.getMinOffset();

    Iterator<Entry<String, ConcurrentMap<Integer, ConsumeQueue>>> it = this.consumeQueueTable.entrySet().iterator();
    while (it.hasNext()) {
        Entry<String, ConcurrentMap<Integer, ConsumeQueue>> next = it.next();
        String topic = next.getKey();
        if (!topic.equals(ScheduleMessageService.SCHEDULE_TOPIC)) {
            ConcurrentMap<Integer, ConsumeQueue> queueTable = next.getValue();
            Iterator<Entry<Integer, ConsumeQueue>> itQT = queueTable.entrySet().iterator();
            while (itQT.hasNext()) {
                Entry<Integer, ConsumeQueue> nextQT = itQT.next();
                long maxCLOffsetInConsumeQueue = nextQT.getValue().getLastOffset();

                if (maxCLOffsetInConsumeQueue == -1) {
                    log.warn("maybe ConsumeQueue was created just now. topic={} queueId={} maxPhysicOffset={} minLogicOffset={}.",
                        nextQT.getValue().getTopic(),
                        nextQT.getValue().getQueueId(),
                        nextQT.getValue().getMaxPhysicOffset(),
                        nextQT.getValue().getMinLogicOffset());
                } else if (maxCLOffsetInConsumeQueue < minCommitLogOffset) {
                    log.info(
                        "cleanExpiredConsumerQueue: {} {} consumer queue destroyed, minCommitLogOffset: {} maxCLOffsetInConsumeQueue: {}",
                        topic,
                        nextQT.getKey(),
                        minCommitLogOffset,
                        maxCLOffsetInConsumeQueue);

                    DefaultMessageStore.this.commitLog.removeQueueFromTopicQueueTable(nextQT.getValue().getTopic(),
                        nextQT.getValue().getQueueId());

                    nextQT.getValue().destroy();
                    itQT.remove();
                }
            }

            if (queueTable.isEmpty()) {
                log.info("cleanExpiredConsumerQueue: {},topic destroyed", topic);
                it.remove();
            }
        }
    }
}
Example #22
Source File: DefaultMessageStore.java    From rocketmq with Apache License 2.0 4 votes vote down vote up 
public ScheduleMessageService getScheduleMessageService() {
    return scheduleMessageService;
}
Example #23
Source File: DefaultMessageStore.java    From rocketmq with Apache License 2.0 4 votes vote down vote up 
public void cleanExpiredConsumerQueue() {
    long minCommitLogOffset = this.commitLog.getMinOffset();

    Iterator<Entry<String, ConcurrentHashMap<Integer, ConsumeQueue>>> it = this.consumeQueueTable.entrySet().iterator();
    while (it.hasNext()) {
        Entry<String, ConcurrentHashMap<Integer, ConsumeQueue>> next = it.next();
        String topic = next.getKey();
        if (!topic.equals(ScheduleMessageService.SCHEDULE_TOPIC)) {
            ConcurrentHashMap<Integer, ConsumeQueue> queueTable = next.getValue();
            Iterator<Entry<Integer, ConsumeQueue>> itQT = queueTable.entrySet().iterator();
            while (itQT.hasNext()) {
                Entry<Integer, ConsumeQueue> nextQT = itQT.next();
                long maxCLOffsetInConsumeQueue = nextQT.getValue().getLastOffset();

                if (maxCLOffsetInConsumeQueue == -1) {
                    log.warn("maybe ConsumeQueue was created just now. topic={} queueId={} maxPhysicOffset={} minLogicOffset={}.", //
                        nextQT.getValue().getTopic(), //
                        nextQT.getValue().getQueueId(), //
                        nextQT.getValue().getMaxPhysicOffset(), //
                        nextQT.getValue().getMinLogicOffset());
                } else if (maxCLOffsetInConsumeQueue < minCommitLogOffset) {
                    log.info(
                        "cleanExpiredConsumerQueue: {} {} consumer queue destroyed, minCommitLogOffset: {} maxCLOffsetInConsumeQueue: {}", //
                        topic, //
                        nextQT.getKey(), //
                        minCommitLogOffset, //
                        maxCLOffsetInConsumeQueue);

                    DefaultMessageStore.this.commitLog.removeQueueFromTopicQueueTable(nextQT.getValue().getTopic(),
                        nextQT.getValue().getQueueId());

                    nextQT.getValue().destroy();
                    itQT.remove();
                }
            }

            if (queueTable.isEmpty()) {
                log.info("cleanExpiredConsumerQueue: {},topic destroyed", topic);
                it.remove();
            }
        }
    }
}
Example #24
Source File: DefaultMessageStore.java    From rocketmq-read with Apache License 2.0 4 votes vote down vote up 
public ScheduleMessageService getScheduleMessageService() {
    return scheduleMessageService;
}
Example #25
Source File: DefaultMessageStore.java    From rocketmq-read with Apache License 2.0 4 votes vote down vote up 
@Override
public void cleanExpiredConsumerQueue() {
    long minCommitLogOffset = this.commitLog.getMinOffset();

    Iterator<Entry<String, ConcurrentMap<Integer, ConsumeQueue>>> it = this.consumeQueueTable.entrySet().iterator();
    while (it.hasNext()) {
        Entry<String, ConcurrentMap<Integer, ConsumeQueue>> next = it.next();
        String topic = next.getKey();
        if (!topic.equals(ScheduleMessageService.SCHEDULE_TOPIC)) {
            ConcurrentMap<Integer, ConsumeQueue> queueTable = next.getValue();
            Iterator<Entry<Integer, ConsumeQueue>> itQT = queueTable.entrySet().iterator();
            while (itQT.hasNext()) {
                Entry<Integer, ConsumeQueue> nextQT = itQT.next();
                long maxCLOffsetInConsumeQueue = nextQT.getValue().getLastOffset();

                if (maxCLOffsetInConsumeQueue == -1) {
                    log.warn("maybe ConsumeQueue was created just now. topic={} queueId={} maxPhysicOffset={} minLogicOffset={}.",
                        nextQT.getValue().getTopic(),
                        nextQT.getValue().getQueueId(),
                        nextQT.getValue().getMaxPhysicOffset(),
                        nextQT.getValue().getMinLogicOffset());
                } else if (maxCLOffsetInConsumeQueue < minCommitLogOffset) {
                    log.info(
                        "cleanExpiredConsumerQueue: {} {} consumer queue destroyed, minCommitLogOffset: {} maxCLOffsetInConsumeQueue: {}",
                        topic,
                        nextQT.getKey(),
                        minCommitLogOffset,
                        maxCLOffsetInConsumeQueue);

                    DefaultMessageStore.this.commitLog.removeQueueFromTopicQueueTable(nextQT.getValue().getTopic(),
                        nextQT.getValue().getQueueId());

                    nextQT.getValue().destroy();
                    itQT.remove();
                }
            }

            if (queueTable.isEmpty()) {
                log.info("cleanExpiredConsumerQueue: {},topic destroyed", topic);
                it.remove();
            }
        }
    }
}
Example #26
Source File: DefaultMessageStore.java    From rocketmq-read with Apache License 2.0 4 votes vote down vote up 
/**
 * 构造函数
 * @param messageStoreConfig ;
 * @param brokerStatsManager ;
 * @param messageArrivingListener ;
 * @param brokerConfig ;
 * @throws IOException ;
 */
public DefaultMessageStore(final MessageStoreConfig messageStoreConfig, final BrokerStatsManager brokerStatsManager,
    final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig) throws IOException {
    this.messageArrivingListener = messageArrivingListener;
    this.brokerConfig = brokerConfig;
    this.messageStoreConfig = messageStoreConfig;
    this.brokerStatsManager = brokerStatsManager;

    this.allocateMappedFileService = new AllocateMappedFileService(this);
    this.commitLog = new CommitLog(this);
    this.consumeQueueTable = new ConcurrentHashMap<>(32);

    this.flushConsumeQueueService = new FlushConsumeQueueService();
    this.cleanCommitLogService = new CleanCommitLogService();
    this.cleanConsumeQueueService = new CleanConsumeQueueService();
    this.storeStatsService = new StoreStatsService();

    this.indexService = new IndexService(this);
    this.haService = new HAService(this);

    this.reputMessageService = new ReputMessageService();
    this.scheduleMessageService = new ScheduleMessageService(this);

    this.transientStorePool = new TransientStorePool(messageStoreConfig);

    if (messageStoreConfig.isTransientStorePoolEnable()) {
        this.transientStorePool.init();
    }

    this.allocateMappedFileService.start();

    this.indexService.start();

    this.dispatcherList = new LinkedList<>();
    this.dispatcherList.addLast(new CommitLogDispatcherBuildConsumeQueue());
    this.dispatcherList.addLast(new CommitLogDispatcherBuildIndex());

    //获取lockFile的路径
    File file = new File(StorePathConfigHelper.getLockFile(messageStoreConfig.getStorePathRootDir()));
    MappedFile.ensureDirOK(file.getParent());
    lockFile = new RandomAccessFile(file, "rw");
}
Example #27
Source File: DefaultMessageStore.java    From rocketmq-4.3.0 with Apache License 2.0 4 votes vote down vote up 
public ScheduleMessageService getScheduleMessageService() {
    return scheduleMessageService;
}
Example #28
Source File: DefaultMessageStore.java    From rocketmq-4.3.0 with Apache License 2.0 4 votes vote down vote up 
public void cleanExpiredConsumerQueue() {
//        从commitLog中获取最小的offset=》
        long minCommitLogOffset = this.commitLog.getMinOffset();

        Iterator<Entry<String, ConcurrentMap<Integer, ConsumeQueue>>> it = this.consumeQueueTable.entrySet().iterator();
        while (it.hasNext()) {
            Entry<String, ConcurrentMap<Integer, ConsumeQueue>> next = it.next();
            String topic = next.getKey();
            if (!topic.equals(ScheduleMessageService.SCHEDULE_TOPIC)) {
                ConcurrentMap<Integer, ConsumeQueue> queueTable = next.getValue();
                Iterator<Entry<Integer, ConsumeQueue>> itQT = queueTable.entrySet().iterator();
                while (itQT.hasNext()) {
                    Entry<Integer, ConsumeQueue> nextQT = itQT.next();
//                    获取消息队列的lastOffset=》
                    long maxCLOffsetInConsumeQueue = nextQT.getValue().getLastOffset();

                    if (maxCLOffsetInConsumeQueue == -1) {
                        log.warn("maybe ConsumeQueue was created just now. topic={} queueId={} maxPhysicOffset={} minLogicOffset={}.",
                            nextQT.getValue().getTopic(),
                            nextQT.getValue().getQueueId(),
                            nextQT.getValue().getMaxPhysicOffset(),
                            nextQT.getValue().getMinLogicOffset());
//                        正常情况下应该等于，如果是小于说明有的消息队列已过期
                    } else if (maxCLOffsetInConsumeQueue < minCommitLogOffset) {
                        log.info(
                            "cleanExpiredConsumerQueue: {} {} consumer queue destroyed, minCommitLogOffset: {} maxCLOffsetInConsumeQueue: {}",
                            topic,
                            nextQT.getKey(),
                            minCommitLogOffset,
                            maxCLOffsetInConsumeQueue);

//                        按topic和queueId删除topic和queue的offset=》
                        DefaultMessageStore.this.commitLog.removeQueueFromTopicQueueTable(nextQT.getValue().getTopic(),
                            nextQT.getValue().getQueueId());

                        nextQT.getValue().destroy();
//                      消息队列销毁了，消息队列所在元素删除
                        itQT.remove();
                    }
                }

                if (queueTable.isEmpty()) {
                    log.info("cleanExpiredConsumerQueue: {},topic destroyed", topic);
//                  topic销毁了，消息队列所在的元素删除
                    it.remove();
                }
            }
        }
    }
Example #29
Source File: DefaultMessageStore.java    From DDMQ with Apache License 2.0 4 votes vote down vote up 
public ScheduleMessageService getScheduleMessageService() {
    return scheduleMessageService;
}
Example #30
Source File: DefaultMessageStore.java    From DDMQ with Apache License 2.0 4 votes vote down vote up 
public void cleanExpiredConsumerQueue() {
    long minCommitLogOffset = this.commitLog.getMinOffset();

    Iterator<Entry<String, ConcurrentMap<Integer, ConsumeQueue>>> it = this.consumeQueueTable.entrySet().iterator();
    while (it.hasNext()) {
        Entry<String, ConcurrentMap<Integer, ConsumeQueue>> next = it.next();
        String topic = next.getKey();
        if (!topic.equals(ScheduleMessageService.SCHEDULE_TOPIC)) {
            ConcurrentMap<Integer, ConsumeQueue> queueTable = next.getValue();
            Iterator<Entry<Integer, ConsumeQueue>> itQT = queueTable.entrySet().iterator();
            while (itQT.hasNext()) {
                Entry<Integer, ConsumeQueue> nextQT = itQT.next();
                long maxCLOffsetInConsumeQueue = nextQT.getValue().getLastOffset();

                if (maxCLOffsetInConsumeQueue == -1) {
                    log.warn("maybe ConsumeQueue was created just now. topic={} queueId={} maxPhysicOffset={} minLogicOffset={}.",
                        nextQT.getValue().getTopic(),
                        nextQT.getValue().getQueueId(),
                        nextQT.getValue().getMaxPhysicOffset(),
                        nextQT.getValue().getMinLogicOffset());
                } else if (maxCLOffsetInConsumeQueue < minCommitLogOffset) {
                    log.info(
                        "cleanExpiredConsumerQueue: {} {} consumer queue destroyed, minCommitLogOffset: {} maxCLOffsetInConsumeQueue: {}",
                        topic,
                        nextQT.getKey(),
                        minCommitLogOffset,
                        maxCLOffsetInConsumeQueue);

                    DefaultMessageStore.this.commitLog.removeQueueFromTopicQueueTable(nextQT.getValue().getTopic(),
                        nextQT.getValue().getQueueId());

                    nextQT.getValue().destroy();
                    itQT.remove();
                }
            }

            if (queueTable.isEmpty()) {
                log.info("cleanExpiredConsumerQueue: {},topic destroyed", topic);
                it.remove();
            }
        }
    }
}