org.apache.rocketmq.store.config.FlushDiskType Java Examples

The following examples show how to use org.apache.rocketmq.store.config.FlushDiskType. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: CommitLog.java    From DDMQ with Apache License 2.0 6 votes vote down vote up 
public CommitLog(final DefaultMessageStore defaultMessageStore) {
    this.mappedFileQueue = new MappedFileQueue(defaultMessageStore.getMessageStoreConfig().getStorePathCommitLog(),
        defaultMessageStore.getMessageStoreConfig().getMapedFileSizeCommitLog(), defaultMessageStore.getAllocateMappedFileService());
    this.defaultMessageStore = defaultMessageStore;

    if (FlushDiskType.SYNC_FLUSH == defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) {
        this.flushCommitLogService = new GroupCommitService();
    } else {
        this.flushCommitLogService = new FlushRealTimeService();
    }

    this.commitLogService = new CommitRealTimeService();

    this.appendMessageCallback = new DefaultAppendMessageCallback(defaultMessageStore.getMessageStoreConfig().getMaxMessageSize());
    batchEncoderThreadLocal = new ThreadLocal<MessageExtBatchEncoder>() {
        @Override
        protected MessageExtBatchEncoder initialValue() {
            return new MessageExtBatchEncoder(defaultMessageStore.getMessageStoreConfig().getMaxMessageSize());
        }
    };
    this.putMessageLock = defaultMessageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage() ? new PutMessageReentrantLock() : new PutMessageSpinLock();

}
Example #2
Source File: CommitLog.java    From DDMQ with Apache License 2.0 6 votes vote down vote up 
public CommitLog(final DefaultMessageStore defaultMessageStore) {
    this.mappedFileQueue = new MappedFileQueue(defaultMessageStore.getMessageStoreConfig().getStorePathCommitLog(),
        defaultMessageStore.getMessageStoreConfig().getMapedFileSizeCommitLog(), defaultMessageStore.getAllocateMappedFileService());
    this.defaultMessageStore = defaultMessageStore;

    if (FlushDiskType.SYNC_FLUSH == defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) {
        this.flushCommitLogService = new GroupCommitService();
    } else {
        this.flushCommitLogService = new FlushRealTimeService();
    }

    this.commitLogService = new CommitRealTimeService();

    this.appendMessageCallback = new DefaultAppendMessageCallback(defaultMessageStore.getMessageStoreConfig().getMaxMessageSize());
    batchEncoderThreadLocal = new ThreadLocal<MessageExtBatchEncoder>() {
        @Override
        protected MessageExtBatchEncoder initialValue() {
            return new MessageExtBatchEncoder(defaultMessageStore.getMessageStoreConfig().getMaxMessageSize());
        }
    };
    this.putMessageLock = defaultMessageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage() ? new PutMessageReentrantLock() : new PutMessageSpinLock();

}
Example #3
Source File: CommitLog.java    From rocketmq-all-4.1.0-incubating with Apache License 2.0 6 votes vote down vote up 
public CommitLog(final DefaultMessageStore defaultMessageStore) {
    this.mappedFileQueue = new MappedFileQueue(defaultMessageStore.getMessageStoreConfig().getStorePathCommitLog(),
        defaultMessageStore.getMessageStoreConfig().getMapedFileSizeCommitLog(), defaultMessageStore.getAllocateMappedFileService());
    this.defaultMessageStore = defaultMessageStore;

    if (FlushDiskType.SYNC_FLUSH == defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) {
        this.flushCommitLogService = new GroupCommitService();
    } else {
        this.flushCommitLogService = new FlushRealTimeService();
    }

    this.commitLogService = new CommitRealTimeService();

    this.appendMessageCallback = new DefaultAppendMessageCallback(defaultMessageStore.getMessageStoreConfig().getMaxMessageSize());
    batchEncoderThreadLocal = new ThreadLocal<MessageExtBatchEncoder>() {
        @Override protected MessageExtBatchEncoder initialValue() {
            return new MessageExtBatchEncoder(defaultMessageStore.getMessageStoreConfig().getMaxMessageSize());
        }
    };
    this.putMessageLock =  defaultMessageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage() ? new PutMessageReentrantLock() : new PutMessageSpinLock();

}
Example #4
Source File: CommitLog.java    From rocketmq with Apache License 2.0 6 votes vote down vote up 
public CommitLog(final DefaultMessageStore defaultMessageStore) {
    this.mappedFileQueue = new MappedFileQueue(defaultMessageStore.getMessageStoreConfig().getStorePathCommitLog(),
        defaultMessageStore.getMessageStoreConfig().getMappedFileSizeCommitLog(), defaultMessageStore.getAllocateMappedFileService());
    this.defaultMessageStore = defaultMessageStore;

    if (FlushDiskType.SYNC_FLUSH == defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) {
        this.flushCommitLogService = new GroupCommitService();
    } else {
        this.flushCommitLogService = new FlushRealTimeService();
    }

    this.commitLogService = new CommitRealTimeService();

    this.appendMessageCallback = new DefaultAppendMessageCallback(defaultMessageStore.getMessageStoreConfig().getMaxMessageSize());
    batchEncoderThreadLocal = new ThreadLocal<MessageExtBatchEncoder>() {
        @Override
        protected MessageExtBatchEncoder initialValue() {
            return new MessageExtBatchEncoder(defaultMessageStore.getMessageStoreConfig().getMaxMessageSize());
        }
    };
    this.putMessageLock = defaultMessageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage() ? new PutMessageReentrantLock() : new PutMessageSpinLock();

}
Example #5
Source File: CommitLog.java    From rocketmq with Apache License 2.0 6 votes vote down vote up 
public CompletableFuture<PutMessageStatus> submitFlushRequest(AppendMessageResult result, PutMessageResult putMessageResult,
                                                              MessageExt messageExt) {
    // Synchronization flush
    if (FlushDiskType.SYNC_FLUSH == this.defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) {
        final GroupCommitService service = (GroupCommitService) this.flushCommitLogService;
        if (messageExt.isWaitStoreMsgOK()) {
            GroupCommitRequest request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes(),
                    this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout());
            service.putRequest(request);
            return request.future();
        } else {
            service.wakeup();
            return CompletableFuture.completedFuture(PutMessageStatus.PUT_OK);
        }
    }
    // Asynchronous flush
    else {
        if (!this.defaultMessageStore.getMessageStoreConfig().isTransientStorePoolEnable()) {
            flushCommitLogService.wakeup();
        } else  {
            commitLogService.wakeup();
        }
        return CompletableFuture.completedFuture(PutMessageStatus.PUT_OK);
    }
}
Example #6
Source File: CommitLog.java    From rocketmq-4.3.0 with Apache License 2.0 6 votes vote down vote up 
public CommitLog(final DefaultMessageStore defaultMessageStore) {
    this.mappedFileQueue = new MappedFileQueue(defaultMessageStore.getMessageStoreConfig().getStorePathCommitLog(),
        defaultMessageStore.getMessageStoreConfig().getMapedFileSizeCommitLog(), defaultMessageStore.getAllocateMappedFileService());
    this.defaultMessageStore = defaultMessageStore;

    if (FlushDiskType.SYNC_FLUSH == defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) {
        this.flushCommitLogService = new GroupCommitService();
    } else {
        this.flushCommitLogService = new FlushRealTimeService();
    }

    this.commitLogService = new CommitRealTimeService();

    this.appendMessageCallback = new DefaultAppendMessageCallback(defaultMessageStore.getMessageStoreConfig().getMaxMessageSize());
    batchEncoderThreadLocal = new ThreadLocal<MessageExtBatchEncoder>() {
        @Override
        protected MessageExtBatchEncoder initialValue() {
            return new MessageExtBatchEncoder(defaultMessageStore.getMessageStoreConfig().getMaxMessageSize());
        }
    };
    this.putMessageLock = defaultMessageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage() ? new PutMessageReentrantLock() : new PutMessageSpinLock();

}
Example #7
Source File: MessageStoreTestBase.java    From rocketmq with Apache License 2.0 6 votes vote down vote up 
protected DefaultMessageStore createMessageStore(String base, boolean createAbort) throws Exception {
    baseDirs.add(base);
    MessageStoreConfig storeConfig = new MessageStoreConfig();
    storeConfig.setMappedFileSizeCommitLog(1024 * 100);
    storeConfig.setMappedFileSizeConsumeQueue(1024);
    storeConfig.setMaxHashSlotNum(100);
    storeConfig.setMaxIndexNum(100 * 10);
    storeConfig.setStorePathRootDir(base);
    storeConfig.setStorePathCommitLog(base + File.separator + "commitlog");
    storeConfig.setFlushDiskType(FlushDiskType.ASYNC_FLUSH);
    DefaultMessageStore defaultMessageStore = new DefaultMessageStore(storeConfig,  new BrokerStatsManager("CommitlogTest"), (topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties) -> {

    }, new BrokerConfig());

    if (createAbort) {
        String fileName = StorePathConfigHelper.getAbortFile(storeConfig.getStorePathRootDir());
        makeSureFileExists(fileName);
    }
    Assert.assertTrue(defaultMessageStore.load());
    defaultMessageStore.start();
    return defaultMessageStore;
}
Example #8
Source File: MappedFile.java    From rocketmq_trans_message with Apache License 2.0 5 votes vote down vote up 
public void warmMappedFile(FlushDiskType type, int pages) {
    long beginTime = System.currentTimeMillis();
    ByteBuffer byteBuffer = this.mappedByteBuffer.slice();
    int flush = 0;
    long time = System.currentTimeMillis();
    for (int i = 0, j = 0; i < this.fileSize; i += MappedFile.OS_PAGE_SIZE, j++) {
        byteBuffer.put(i, (byte) 0);
        // force flush when flush disk type is sync
        if (type == FlushDiskType.SYNC_FLUSH) {
            if ((i / OS_PAGE_SIZE) - (flush / OS_PAGE_SIZE) >= pages) {
                flush = i;
                mappedByteBuffer.force();
            }
        }

        // prevent gc
        if (j % 1000 == 0) {
            log.info("j={}, costTime={}", j, System.currentTimeMillis() - time);
            time = System.currentTimeMillis();
            try {
                Thread.sleep(0);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }

    // force flush when prepare load finished
    if (type == FlushDiskType.SYNC_FLUSH) {
        log.info("mapped file warm-up done, force to disk, mappedFile={}, costTime={}",
            this.getFileName(), System.currentTimeMillis() - beginTime);
        mappedByteBuffer.force();
    }
    log.info("mapped file warm-up done. mappedFile={}, costTime={}", this.getFileName(),
        System.currentTimeMillis() - beginTime);

    this.mlock();
}
Example #9
Source File: HATest.java    From rocketmq with Apache License 2.0 5 votes vote down vote up 
private void buildMessageStoreConfig(MessageStoreConfig messageStoreConfig){
    messageStoreConfig.setMappedFileSizeCommitLog(1024 * 1024 * 10);
    messageStoreConfig.setMappedFileSizeConsumeQueue(1024 * 1024 * 10);
    messageStoreConfig.setMaxHashSlotNum(10000);
    messageStoreConfig.setMaxIndexNum(100 * 100);
    messageStoreConfig.setFlushDiskType(FlushDiskType.SYNC_FLUSH);
    messageStoreConfig.setFlushIntervalConsumeQueue(1);
}
Example #10
Source File: CommitLog.java    From DDMQ with Apache License 2.0 5 votes vote down vote up 
public void handleDiskFlush(AppendMessageResult result, PutMessageResult putMessageResult, MessageExt messageExt) {
    // Synchronization flush
    if (FlushDiskType.SYNC_FLUSH == this.defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) {
        final GroupCommitService service = (GroupCommitService) this.flushCommitLogService;
        if (messageExt.isWaitStoreMsgOK()) {
            GroupCommitRequest request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes(),
                this.defaultMessageStore.getSystemClock().now() + this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout());
            service.putRequest(request);
            boolean flushOK = request.waitForFlush(this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout());
            if (!flushOK) {
                log.error("do groupcommit, wait for flush failed, topic: " + messageExt.getTopic() + " tags: " + messageExt.getTags()
                    + " client address: " + messageExt.getBornHostString());
                putMessageResult.setPutMessageStatus(PutMessageStatus.FLUSH_DISK_TIMEOUT);
            }
        } else {
            service.wakeup();
        }
    }
    // Asynchronous flush
    else {
        if (!this.defaultMessageStore.getMessageStoreConfig().isTransientStorePoolEnable()) {
            flushCommitLogService.wakeup();
        } else {
            commitLogService.wakeup();
        }
    }
}
Example #11
Source File: DefaultMessageStoreTest.java    From DDMQ with Apache License 2.0 5 votes vote down vote up 
public MessageStore buildMessageStore() throws Exception {
    MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
    messageStoreConfig.setMapedFileSizeCommitLog(1024 * 1024 * 10);
    messageStoreConfig.setMapedFileSizeConsumeQueue(1024 * 1024 * 10);
    messageStoreConfig.setMaxHashSlotNum(10000);
    messageStoreConfig.setMaxIndexNum(100 * 100);
    messageStoreConfig.setFlushDiskType(FlushDiskType.SYNC_FLUSH);
    return new DefaultMessageStore(messageStoreConfig, new BrokerStatsManager("simpleTest"), new MyMessageArrivingListener(), new BrokerConfig());
}
Example #12
Source File: DefaultMessageStoreShuwDownTest.java    From DDMQ with Apache License 2.0 5 votes vote down vote up 
public DefaultMessageStore buildMessageStore() throws Exception {
    MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
    messageStoreConfig.setMapedFileSizeCommitLog(1024 * 1024 * 10);
    messageStoreConfig.setMapedFileSizeConsumeQueue(1024 * 1024 * 10);
    messageStoreConfig.setMaxHashSlotNum(10000);
    messageStoreConfig.setMaxIndexNum(100 * 100);
    messageStoreConfig.setFlushDiskType(FlushDiskType.SYNC_FLUSH);
    return new DefaultMessageStore(messageStoreConfig, new BrokerStatsManager("simpleTest"), null, new BrokerConfig());
}
Example #13
Source File: MappedFile.java    From rocketmq-all-4.1.0-incubating with Apache License 2.0 5 votes vote down vote up 
public void warmMappedFile(FlushDiskType type, int pages) {
    long beginTime = System.currentTimeMillis();
    ByteBuffer byteBuffer = this.mappedByteBuffer.slice();
    int flush = 0;
    long time = System.currentTimeMillis();
    for (int i = 0, j = 0; i < this.fileSize; i += MappedFile.OS_PAGE_SIZE, j++) {
        byteBuffer.put(i, (byte) 0);
        // force flush when flush disk type is sync
        if (type == FlushDiskType.SYNC_FLUSH) {
            if ((i / OS_PAGE_SIZE) - (flush / OS_PAGE_SIZE) >= pages) {
                flush = i;
                mappedByteBuffer.force();
            }
        }

        // prevent gc
        if (j % 1000 == 0) {
            log.info("j={}, costTime={}", j, System.currentTimeMillis() - time);
            time = System.currentTimeMillis();
            try {
                Thread.sleep(0);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }

    // force flush when prepare load finished
    if (type == FlushDiskType.SYNC_FLUSH) {
        log.info("mapped file warm-up done, force to disk, mappedFile={}, costTime={}",
            this.getFileName(), System.currentTimeMillis() - beginTime);
        mappedByteBuffer.force();
    }
    log.info("mapped file warm-up done. mappedFile={}, costTime={}", this.getFileName(),
        System.currentTimeMillis() - beginTime);

    this.mlock();
}
Example #14
Source File: MappedFile.java    From DDMQ with Apache License 2.0 5 votes vote down vote up 
public void warmMappedFile(FlushDiskType type, int pages) {
    long beginTime = System.currentTimeMillis();
    ByteBuffer byteBuffer = this.mappedByteBuffer.slice();
    int flush = 0;
    long time = System.currentTimeMillis();
    for (int i = 0, j = 0; i < this.fileSize; i += MappedFile.OS_PAGE_SIZE, j++) {
        byteBuffer.put(i, (byte) 0);
        // force flush when flush disk type is sync
        if (type == FlushDiskType.SYNC_FLUSH) {
            if ((i / OS_PAGE_SIZE) - (flush / OS_PAGE_SIZE) >= pages) {
                flush = i;
                mappedByteBuffer.force();
            }
        }

        // prevent gc
        if (j % 1000 == 0) {
            log.info("j={}, costTime={}", j, System.currentTimeMillis() - time);
            time = System.currentTimeMillis();
            try {
                Thread.sleep(0);
            } catch (InterruptedException e) {
                log.error("Interrupted", e);
            }
        }
    }

    // force flush when prepare load finished
    if (type == FlushDiskType.SYNC_FLUSH) {
        log.info("mapped file warm-up done, force to disk, mappedFile={}, costTime={}",
            this.getFileName(), System.currentTimeMillis() - beginTime);
        mappedByteBuffer.force();
    }
    log.info("mapped file warm-up done. mappedFile={}, costTime={}", this.getFileName(),
        System.currentTimeMillis() - beginTime);

    this.mlock();
}
Example #15
Source File: CommitLog.java    From rocketmq-all-4.1.0-incubating with Apache License 2.0 5 votes vote down vote up 
/**
 *刷盘逻辑
 */
public void handleDiskFlush(AppendMessageResult result, PutMessageResult putMessageResult, MessageExt messageExt) {
    // Synchronization flush  同步刷盘
    if (FlushDiskType.SYNC_FLUSH == this.defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) {
        final GroupCommitService service = (GroupCommitService) this.flushCommitLogService;
        if (messageExt.isWaitStoreMsgOK()) {
            GroupCommitRequest request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes());
            service.putRequest(request);
            boolean flushOK = request.waitForFlush(this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout());
            if (!flushOK) {
                log.error("do groupcommit, wait for flush failed, topic: " + messageExt.getTopic() + " tags: " + messageExt.getTags()
                    + " client address: " + messageExt.getBornHostString());
                putMessageResult.setPutMessageStatus(PutMessageStatus.FLUSH_DISK_TIMEOUT);
            }
        } else {
            service.wakeup();
        }
    }
    // Asynchronous flush   异步刷盘
    else {
        if (!this.defaultMessageStore.getMessageStoreConfig().isTransientStorePoolEnable()) {
            flushCommitLogService.wakeup();
        } else {
            commitLogService.wakeup();
        }
    }
}
Example #16
Source File: DefaultMessageStoreTest.java    From rocketmq-all-4.1.0-incubating with Apache License 2.0 5 votes vote down vote up 
@Test
public void testGroupCommit() throws Exception {
    long totalMsgs = 100;
    QUEUE_TOTAL = 1;
    MessageBody = StoreMessage.getBytes();
    MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
    messageStoreConfig.setMapedFileSizeCommitLog(1024 * 8);
    messageStoreConfig.setFlushDiskType(FlushDiskType.SYNC_FLUSH);
    MessageStore master = new DefaultMessageStore(messageStoreConfig, null, new MyMessageArrivingListener(), new BrokerConfig());
    boolean load = master.load();
    assertTrue(load);

    master.start();
    try {
        for (long i = 0; i < totalMsgs; i++) {
            master.putMessage(buildMessage());
        }

        for (long i = 0; i < totalMsgs; i++) {
            GetMessageResult result = master.getMessage("GROUP_A", "TOPIC_A", 0, i, 1024 * 1024, null);
            assertThat(result).isNotNull();
            result.release();

        }
    } finally {
        master.shutdown();
        master.destroy();
    }
}
Example #17
Source File: DefaultMessageStoreShutDownTest.java    From rocketmq with Apache License 2.0 5 votes vote down vote up 
public DefaultMessageStore buildMessageStore() throws Exception {
    MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
    messageStoreConfig.setMappedFileSizeCommitLog(1024 * 1024 * 10);
    messageStoreConfig.setMappedFileSizeConsumeQueue(1024 * 1024 * 10);
    messageStoreConfig.setMaxHashSlotNum(10000);
    messageStoreConfig.setMaxIndexNum(100 * 100);
    messageStoreConfig.setFlushDiskType(FlushDiskType.SYNC_FLUSH);
    return new DefaultMessageStore(messageStoreConfig, new BrokerStatsManager("simpleTest"), null, new BrokerConfig());
}
Example #18
Source File: CommitLog.java    From rocketmq_trans_message with Apache License 2.0 5 votes vote down vote up 
public CommitLog(final DefaultMessageStore defaultMessageStore) {
    this.mappedFileQueue = new MappedFileQueue(defaultMessageStore.getMessageStoreConfig().getStorePathCommitLog(),
            defaultMessageStore.getMessageStoreConfig().getMapedFileSizeCommitLog(), defaultMessageStore.getAllocateMappedFileService());
    this.defaultMessageStore = defaultMessageStore;

    if (FlushDiskType.SYNC_FLUSH == defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) {
        this.flushCommitLogService = new GroupCommitService();
    } else {
        this.flushCommitLogService = new FlushRealTimeService();
    }

    this.commitLogService = new CommitRealTimeService();

    this.appendMessageCallback = new DefaultAppendMessageCallback(defaultMessageStore.getMessageStoreConfig().getMaxMessageSize());
}
Example #19
Source File: DefaultMessageStoreTest.java    From rocketmq_trans_message with Apache License 2.0 5 votes vote down vote up 
@Test
public void testGroupCommit() throws Exception {
    long totalMsgs = 100;
    QUEUE_TOTAL = 1;
    MessageBody = StoreMessage.getBytes();
    MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
    messageStoreConfig.setMapedFileSizeCommitLog(1024 * 8);
    messageStoreConfig.setFlushDiskType(FlushDiskType.SYNC_FLUSH);
    MessageStore master = new DefaultMessageStore(messageStoreConfig, null, new MyMessageArrivingListener(), new BrokerConfig(), null);
    boolean load = master.load();
    assertTrue(load);

    master.start();
    try {
        for (long i = 0; i < totalMsgs; i++) {
            master.putMessage(buildMessage());
        }

        for (long i = 0; i < totalMsgs; i++) {
            GetMessageResult result = master.getMessage("GROUP_A", "TOPIC_A", 0, i, 1024 * 1024, null);
            assertThat(result).isNotNull();
            result.release();

        }
    } finally {
        master.shutdown();
        master.destroy();
    }
}
Example #20
Source File: MappedFile.java    From rocketmq with Apache License 2.0 5 votes vote down vote up 
public void warmMappedFile(FlushDiskType type, int pages) {
    long beginTime = System.currentTimeMillis();
    ByteBuffer byteBuffer = this.mappedByteBuffer.slice();
    int flush = 0;
    long time = System.currentTimeMillis();
    for (int i = 0, j = 0; i < this.fileSize; i += MappedFile.OS_PAGE_SIZE, j++) {
        byteBuffer.put(i, (byte) 0);
        // force flush when flush disk type is sync
        if (type == FlushDiskType.SYNC_FLUSH) {
            if ((i / OS_PAGE_SIZE) - (flush / OS_PAGE_SIZE) >= pages) {
                flush = i;
                mappedByteBuffer.force();
            }
        }

        // prevent gc
        if (j % 1000 == 0) {
            log.info("j={}, costTime={}", j, System.currentTimeMillis() - time);
            time = System.currentTimeMillis();
            try {
                Thread.sleep(0);
            } catch (InterruptedException e) {
                log.error("Interrupted", e);
            }
        }
    }

    // force flush when prepare load finished
    if (type == FlushDiskType.SYNC_FLUSH) {
        log.info("mapped file warm-up done, force to disk, mappedFile={}, costTime={}",
            this.getFileName(), System.currentTimeMillis() - beginTime);
        mappedByteBuffer.force();
    }
    log.info("mapped file warm-up done. mappedFile={}, costTime={}", this.getFileName(),
        System.currentTimeMillis() - beginTime);

    this.mlock();
}
Example #21
Source File: DefaultMessageStoreCleanFilesTest.java    From rocketmq with Apache License 2.0 5 votes vote down vote up 
private void initMessageStore(String deleteWhen, int diskMaxUsedSpaceRatio, double diskSpaceCleanForciblyRatio) throws Exception {
    MessageStoreConfig messageStoreConfig = new MessageStoreConfigForTest();
    messageStoreConfig.setMappedFileSizeCommitLog(mappedFileSize);
    messageStoreConfig.setMappedFileSizeConsumeQueue(mappedFileSize);
    messageStoreConfig.setMaxHashSlotNum(100);
    messageStoreConfig.setMaxIndexNum(8);
    messageStoreConfig.setFlushDiskType(FlushDiskType.SYNC_FLUSH);
    messageStoreConfig.setFlushIntervalConsumeQueue(1);

    // Invalidate DefaultMessageStore`s scheduled task of cleaning expired files.
    // work with the code 'Thread.sleep(1000 * 60 + 100)' behind.
    messageStoreConfig.setCleanResourceInterval(Integer.MAX_VALUE);

    messageStoreConfig.setFileReservedTime(fileReservedTime);
    messageStoreConfig.setDeleteWhen(deleteWhen);
    messageStoreConfig.setDiskMaxUsedSpaceRatio(diskMaxUsedSpaceRatio);

    String storePathRootDir = System.getProperty("user.home") + File.separator
            + "DefaultMessageStoreCleanFilesTest-" + UUID.randomUUID();
    String storePathCommitLog = storePathRootDir + File.separator + "commitlog";
    messageStoreConfig.setStorePathRootDir(storePathRootDir);
    messageStoreConfig.setStorePathCommitLog(storePathCommitLog);

    messageStore = new DefaultMessageStore(messageStoreConfig,
            new BrokerStatsManager("test"), new MyMessageArrivingListener(), new BrokerConfig());

    cleanCommitLogService = getCleanCommitLogService(diskSpaceCleanForciblyRatio);
    cleanConsumeQueueService = getCleanConsumeQueueService();

    assertTrue(messageStore.load());
    messageStore.start();
}
Example #22
Source File: CommitLog.java    From rocketmq with Apache License 2.0 5 votes vote down vote up 
public void handleDiskFlush(AppendMessageResult result, PutMessageResult putMessageResult, MessageExt messageExt) {
    // Synchronization flush
    if (FlushDiskType.SYNC_FLUSH == this.defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) {
        final GroupCommitService service = (GroupCommitService) this.flushCommitLogService;
        if (messageExt.isWaitStoreMsgOK()) {
            GroupCommitRequest request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes());
            service.putRequest(request);
            CompletableFuture<PutMessageStatus> flushOkFuture = request.future();
            PutMessageStatus flushStatus = null;
            try {
                flushStatus = flushOkFuture.get(this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout(),
                        TimeUnit.MILLISECONDS);
            } catch (InterruptedException | ExecutionException | TimeoutException e) {
                //flushOK=false;
            }
            if (flushStatus != PutMessageStatus.PUT_OK) {
                log.error("do groupcommit, wait for flush failed, topic: " + messageExt.getTopic() + " tags: " + messageExt.getTags()
                    + " client address: " + messageExt.getBornHostString());
                putMessageResult.setPutMessageStatus(PutMessageStatus.FLUSH_DISK_TIMEOUT);
            }
        } else {
            service.wakeup();
        }
    }
    // Asynchronous flush
    else {
        if (!this.defaultMessageStore.getMessageStoreConfig().isTransientStorePoolEnable()) {
            flushCommitLogService.wakeup();
        } else {
            commitLogService.wakeup();
        }
    }
}
Example #23
Source File: BatchPutMessageTest.java    From rocketmq with Apache License 2.0 5 votes vote down vote up 
private MessageStore buildMessageStore() throws Exception {
    MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
    messageStoreConfig.setMappedFileSizeCommitLog(1024 * 8);
    messageStoreConfig.setMappedFileSizeConsumeQueue(1024 * 4);
    messageStoreConfig.setMaxHashSlotNum(100);
    messageStoreConfig.setMaxIndexNum(100 * 10);
    messageStoreConfig.setFlushDiskType(FlushDiskType.SYNC_FLUSH);
    messageStoreConfig.setFlushIntervalConsumeQueue(1);
    messageStoreConfig.setStorePathRootDir(System.getProperty("user.home") + File.separator + "putmessagesteststore");
    messageStoreConfig.setStorePathCommitLog(System.getProperty("user.home") + File.separator + "putmessagesteststore" + File.separator + "commitlog");
    return new DefaultMessageStore(messageStoreConfig, new BrokerStatsManager("simpleTest"), new MyMessageArrivingListener(), new BrokerConfig());
}
Example #24
Source File: DefaultMessageStoreTest.java    From rocketmq with Apache License 2.0 5 votes vote down vote up 
private MessageStore buildMessageStore() throws Exception {
    MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
    messageStoreConfig.setMappedFileSizeCommitLog(1024 * 1024 * 10);
    messageStoreConfig.setMappedFileSizeConsumeQueue(1024 * 1024 * 10);
    messageStoreConfig.setMaxHashSlotNum(10000);
    messageStoreConfig.setMaxIndexNum(100 * 100);
    messageStoreConfig.setFlushDiskType(FlushDiskType.SYNC_FLUSH);
    messageStoreConfig.setFlushIntervalConsumeQueue(1);
    return new DefaultMessageStore(messageStoreConfig, new BrokerStatsManager("simpleTest"), new MyMessageArrivingListener(), new BrokerConfig());
}
Example #25
Source File: CommitLog.java    From rocketmq-read with Apache License 2.0 5 votes vote down vote up 
/**
 * 构造函数
 * @param defaultMessageStore ;
 */
public CommitLog(final DefaultMessageStore defaultMessageStore) {
    this.mappedFileQueue = new MappedFileQueue(defaultMessageStore.getMessageStoreConfig().getStorePathCommitLog(),
        defaultMessageStore.getMessageStoreConfig().getMapedFileSizeCommitLog(), defaultMessageStore.getAllocateMappedFileService());
    this.defaultMessageStore = defaultMessageStore;

    if (FlushDiskType.SYNC_FLUSH == defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) {
        //同步刷盘，使用GroupCommitService
        this.flushCommitLogService = new GroupCommitService();
    } else {
        //异步刷盘，使用FlushRealTimeService
        this.flushCommitLogService = new FlushRealTimeService();
    }
    //commitlogService
    this.commitLogService = new CommitRealTimeService();
    //appendMessageCallback
    this.appendMessageCallback = new DefaultAppendMessageCallback(defaultMessageStore.getMessageStoreConfig().getMaxMessageSize());
    //批量消息Encoder
    batchEncoderThreadLocal = new ThreadLocal<MessageExtBatchEncoder>() {
        @Override
        protected MessageExtBatchEncoder initialValue() {
            return new MessageExtBatchEncoder(defaultMessageStore.getMessageStoreConfig().getMaxMessageSize());
        }
    };
    //添加消息的锁
    this.putMessageLock = defaultMessageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage() ? new PutMessageReentrantLock() : new PutMessageSpinLock();

}
Example #26
Source File: CommitLog.java    From DDMQ with Apache License 2.0 5 votes vote down vote up 
public void handleDiskFlush(AppendMessageResult result, PutMessageResult putMessageResult, MessageExt messageExt) {
    // Synchronization flush
    if (FlushDiskType.SYNC_FLUSH == this.defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) {
        final GroupCommitService service = (GroupCommitService) this.flushCommitLogService;
        if (messageExt.isWaitStoreMsgOK()) {
            GroupCommitRequest request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes(),
                this.defaultMessageStore.getSystemClock().now() + this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout());
            service.putRequest(request);
            boolean flushOK = request.waitForFlush(this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout());
            if (!flushOK) {
                log.error("do groupcommit, wait for flush failed, topic: " + messageExt.getTopic() + " tags: " + messageExt.getTags()
                    + " client address: " + messageExt.getBornHostString());
                putMessageResult.setPutMessageStatus(PutMessageStatus.FLUSH_DISK_TIMEOUT);
            }
        } else {
            service.wakeup();
        }
    }
    // Asynchronous flush
    else {
        if (!this.defaultMessageStore.getMessageStoreConfig().isTransientStorePoolEnable()) {
            flushCommitLogService.wakeup();
        } else {
            commitLogService.wakeup();
        }
    }
}
Example #27
Source File: DefaultMessageStoreTest.java    From DDMQ with Apache License 2.0 5 votes vote down vote up 
public MessageStore buildMessageStore() throws Exception {
    MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
    messageStoreConfig.setMapedFileSizeCommitLog(1024 * 1024 * 10);
    messageStoreConfig.setMapedFileSizeConsumeQueue(1024 * 1024 * 10);
    messageStoreConfig.setMaxHashSlotNum(10000);
    messageStoreConfig.setMaxIndexNum(100 * 100);
    messageStoreConfig.setFlushDiskType(FlushDiskType.SYNC_FLUSH);
    return new DefaultMessageStore(messageStoreConfig, new BrokerStatsManager("simpleTest"), new MyMessageArrivingListener(), new BrokerConfig());
}
Example #28
Source File: DefaultMessageStoreShuwDownTest.java    From DDMQ with Apache License 2.0 5 votes vote down vote up 
public DefaultMessageStore buildMessageStore() throws Exception {
    MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
    messageStoreConfig.setMapedFileSizeCommitLog(1024 * 1024 * 10);
    messageStoreConfig.setMapedFileSizeConsumeQueue(1024 * 1024 * 10);
    messageStoreConfig.setMaxHashSlotNum(10000);
    messageStoreConfig.setMaxIndexNum(100 * 100);
    messageStoreConfig.setFlushDiskType(FlushDiskType.SYNC_FLUSH);
    return new DefaultMessageStore(messageStoreConfig, new BrokerStatsManager("simpleTest"), null, new BrokerConfig());
}
Example #29
Source File: MappedFile.java    From rocketmq-4.3.0 with Apache License 2.0 5 votes vote down vote up 
public void warmMappedFile(FlushDiskType type, int pages) {
    long beginTime = System.currentTimeMillis();
    ByteBuffer byteBuffer = this.mappedByteBuffer.slice();
    int flush = 0;
    long time = System.currentTimeMillis();
    for (int i = 0, j = 0; i < this.fileSize; i += MappedFile.OS_PAGE_SIZE, j++) {
        byteBuffer.put(i, (byte) 0);
        // force flush when flush disk type is sync
        if (type == FlushDiskType.SYNC_FLUSH) {
            if ((i / OS_PAGE_SIZE) - (flush / OS_PAGE_SIZE) >= pages) {
                flush = i;
                mappedByteBuffer.force();
            }
        }

        // prevent gc
        if (j % 1000 == 0) {
            log.info("j={}, costTime={}", j, System.currentTimeMillis() - time);
            time = System.currentTimeMillis();
            try {
                Thread.sleep(0);
            } catch (InterruptedException e) {
                log.error("Interrupted", e);
            }
        }
    }

    // force flush when prepare load finished
    if (type == FlushDiskType.SYNC_FLUSH) {
        log.info("mapped file warm-up done, force to disk, mappedFile={}, costTime={}",
            this.getFileName(), System.currentTimeMillis() - beginTime);
        mappedByteBuffer.force();
    }
    log.info("mapped file warm-up done. mappedFile={}, costTime={}", this.getFileName(),
        System.currentTimeMillis() - beginTime);

    this.mlock();
}
Example #30
Source File: CommitLog.java    From rocketmq-4.3.0 with Apache License 2.0 5 votes vote down vote up 
public void handleDiskFlush(AppendMessageResult result, PutMessageResult putMessageResult, MessageExt messageExt) {
        // Synchronization flush 同步刷新
        if (FlushDiskType.SYNC_FLUSH == this.defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) {
            final GroupCommitService service = (GroupCommitService) this.flushCommitLogService;
            if (messageExt.isWaitStoreMsgOK()) {
                GroupCommitRequest request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes());
                service.putRequest(request);
//                countdownLatch.await() 同步等待刷新结果，除非超时
                boolean flushOK = request.waitForFlush(this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout());
                if (!flushOK) {
                    log.error("do groupcommit, wait for flush failed, topic: " + messageExt.getTopic() + " tags: " + messageExt.getTags()
                        + " client address: " + messageExt.getBornHostString());
                    putMessageResult.setPutMessageStatus(PutMessageStatus.FLUSH_DISK_TIMEOUT);
                }
            } else {
//                如果异步直接解除阻塞 countdownLatch.countDown()
                service.wakeup();
            }
        }
        // Asynchronous flush 异步刷新
        else {
            if (!this.defaultMessageStore.getMessageStoreConfig().isTransientStorePoolEnable()) {
                flushCommitLogService.wakeup();
            } else {
                commitLogService.wakeup();
            }
        }
    }