org.apache.rocketmq.store.DispatchRequest Java Examples

private IndexFile putKey(IndexFile indexFile, DispatchRequest msg, String idxKey) {
    for (boolean ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp()); !ok; ) {
        log.warn("Index file [" + indexFile.getFileName() + "] is full, trying to create another one");

        indexFile = retryGetAndCreateIndexFile();
        if (null == indexFile) {
            return null;
        }

        ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp());
    }

    return indexFile;
}
 

private IndexFile putKey(IndexFile indexFile, DispatchRequest msg, String idxKey) {
    for (boolean ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp()); !ok; ) {
        log.warn("Index file [" + indexFile.getFileName() + "] is full, trying to create another one");

        indexFile = retryGetAndCreateIndexFile();
        if (null == indexFile) {
            return null;
        }

        ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp());
    }

    return indexFile;
}
 

@Override
public DispatchRequest checkMessageAndReturnSize(ByteBuffer byteBuffer, final boolean checkCRC,
    final boolean readBody) {
    if (isInrecoveringOldCommitlog) {
        return super.checkMessageAndReturnSize(byteBuffer, checkCRC, readBody);
    }
    try {
        int bodyOffset = DLedgerEntry.BODY_OFFSET;
        int pos = byteBuffer.position();
        int magic = byteBuffer.getInt();
        //In dledger, this field is size, it must be gt 0, so it could prevent collision
        int magicOld = byteBuffer.getInt();
        if (magicOld == CommitLog.BLANK_MAGIC_CODE || magicOld == CommitLog.MESSAGE_MAGIC_CODE) {
            byteBuffer.position(pos);
            return super.checkMessageAndReturnSize(byteBuffer, checkCRC, readBody);
        }
        if (magic == MmapFileList.BLANK_MAGIC_CODE) {
            return new DispatchRequest(0, true);
        }
        byteBuffer.position(pos + bodyOffset);
        DispatchRequest dispatchRequest = super.checkMessageAndReturnSize(byteBuffer, checkCRC, readBody);
        if (dispatchRequest.isSuccess()) {
            dispatchRequest.setBufferSize(dispatchRequest.getMsgSize() + bodyOffset);
        } else if (dispatchRequest.getMsgSize() > 0) {
            dispatchRequest.setBufferSize(dispatchRequest.getMsgSize() + bodyOffset);
        }
        return dispatchRequest;
    } catch (Throwable ignored) {
    }

    return new DispatchRequest(-1, false /* success */);
}
 

protected DefaultMessageStore gen(ConsumerFilterManager filterManager) throws Exception {
    MessageStoreConfig messageStoreConfig = buildStoreConfig(
        commitLogFileSize, cqFileSize, true, cqExtFileSize
    );

    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);
    brokerConfig.setMaxErrorRateOfBloomFilter(20);
    brokerConfig.setExpectConsumerNumUseFilter(64);

    DefaultMessageStore master = new DefaultMessageStore(
        messageStoreConfig,
        new BrokerStatsManager(brokerConfig.getBrokerClusterName()),
        new MessageArrivingListener() {
            @Override
            public void arriving(String topic, int queueId, long logicOffset, long tagsCode,
                                 long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {
            }
        }
        , brokerConfig);

    master.getDispatcherList().addFirst(new CommitLogDispatcher() {
        @Override
        public void dispatch(DispatchRequest request) {
            try {
            } catch (Throwable e) {
                e.printStackTrace();
            }
        }
    });
    master.getDispatcherList().addFirst(new CommitLogDispatcherCalcBitMap(brokerConfig, filterManager));

    assertThat(master.load()).isTrue();

    master.start();

    return master;
}
 

@Test
public void testDispatch_blankFilterData() {
    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);

    ConsumerFilterManager filterManager = new ConsumerFilterManager();

    CommitLogDispatcherCalcBitMap calcBitMap = new CommitLogDispatcherCalcBitMap(brokerConfig,
        filterManager);

    for (int i = 0; i < 10; i++) {
        Map<String, String> properties = new HashMap<String, String>(4);
        properties.put("a", String.valueOf(i * 10 + 5));

        String topic = "topic" + i;

        DispatchRequest dispatchRequest = new DispatchRequest(
            topic,
            0,
            i * 100 + 123,
            100,
            (long) ("tags" + i).hashCode(),
            System.currentTimeMillis(),
            i,
            null,
            UUID.randomUUID().toString(),
            0,
            0,
            properties
        );

        calcBitMap.dispatch(dispatchRequest);

        assertThat(dispatchRequest.getBitMap()).isNull();
    }
}
 

private IndexFile putKey(IndexFile indexFile, DispatchRequest msg, String idxKey) {
    for (boolean ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp()); !ok; ) {
        log.warn("Index file [" + indexFile.getFileName() + "] is full, trying to create another one");

        indexFile = retryGetAndCreateIndexFile();
        if (null == indexFile) {
            return null;
        }

        ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp());
    }

    return indexFile;
}
 

public void appendPreparedTransaction(DispatchRequest dispatchRequest) {
    try {
        flowController.acquire(1);
        while (true) {
            DispatchRequestCollections requests = dispatchRequestBufferQueue.peek();
            if (requests.latch.getAndIncrement() >= 0) {
                requests.requestlist.add(dispatchRequest);
                break;
            }
        }
    } catch (InterruptedException e) {
        log.error("putDispatchRequest interrupted");
    }
}
 

@Override
public boolean updateTransactionState(DispatchRequest request) {
    try {
        transactionRecordFlush2DBService.appendPreparedTransaction(request);
        return true;
    } catch (Throwable e) {
        log.error(e.getMessage(), e);
        return false;
    }
}
 

@Override
public boolean appendPreparedTransaction(DispatchRequest request) {
    try {
        transactionRecordFlush2DBService.appendPreparedTransaction(request);
        return true;
    } catch (Throwable e) {
        log.error(e.getMessage(), e);
        return false;
    }
}
 

private IndexFile putKey(IndexFile indexFile, DispatchRequest msg, String idxKey) {
    for (boolean ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp()); !ok; ) {
        log.warn("Index file [" + indexFile.getFileName() + "] is full, trying to create another one");

        indexFile = retryGetAndCreateIndexFile();//如果indexFile有就获取没有就创建
        if (null == indexFile) {
            return null;
        }

        ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp());//写入索引消息
    }

    return indexFile;
}
 

@Test
public void testDispatch_blankFilterData() {
    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);

    ConsumerFilterManager filterManager = new ConsumerFilterManager();

    CommitLogDispatcherCalcBitMap calcBitMap = new CommitLogDispatcherCalcBitMap(brokerConfig,
        filterManager);

    for (int i = 0; i < 10; i++) {
        Map<String, String> properties = new HashMap<String, String>(4);
        properties.put("a", String.valueOf(i * 10 + 5));

        String topic = "topic" + i;

        DispatchRequest dispatchRequest = new DispatchRequest(
            topic,
            0,
            i * 100 + 123,
            100,
            (long) ("tags" + i).hashCode(),
            System.currentTimeMillis(),
            i,
            null,
            UUID.randomUUID().toString(),
            0,
            0,
            properties
        );

        calcBitMap.dispatch(dispatchRequest);

        assertThat(dispatchRequest.getBitMap()).isNull();
    }
}
 

protected DefaultMessageStore gen(ConsumerFilterManager filterManager) throws Exception {
    MessageStoreConfig messageStoreConfig = buildStoreConfig(
        commitLogFileSize, cqFileSize, true, cqExtFileSize
    );

    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);
    brokerConfig.setMaxErrorRateOfBloomFilter(20);
    brokerConfig.setExpectConsumerNumUseFilter(64);

    DefaultMessageStore master = new DefaultMessageStore(
        messageStoreConfig,
        new BrokerStatsManager(brokerConfig.getBrokerClusterName()),
        new MessageArrivingListener() {
            @Override
            public void arriving(String topic, int queueId, long logicOffset, long tagsCode,
                                 long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {
            }
        }
        , brokerConfig);

    master.getDispatcherList().addFirst(new CommitLogDispatcher() {
        @Override
        public void dispatch(DispatchRequest request) {
            try {
            } catch (Throwable e) {
                e.printStackTrace();
            }
        }
    });
    master.getDispatcherList().addFirst(new CommitLogDispatcherCalcBitMap(brokerConfig, filterManager));

    assertThat(master.load()).isTrue();

    master.start();

    return master;
}
 

@Test
public void testDispatch_blankFilterData() {
    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);

    ConsumerFilterManager filterManager = new ConsumerFilterManager();

    CommitLogDispatcherCalcBitMap calcBitMap = new CommitLogDispatcherCalcBitMap(brokerConfig,
        filterManager);

    for (int i = 0; i < 10; i++) {
        Map<String, String> properties = new HashMap<String, String>(4);
        properties.put("a", String.valueOf(i * 10 + 5));

        String topic = "topic" + i;

        DispatchRequest dispatchRequest = new DispatchRequest(
            topic,
            0,
            i * 100 + 123,
            100,
            (long) ("tags" + i).hashCode(),
            System.currentTimeMillis(),
            i,
            null,
            UUID.randomUUID().toString(),
            0,
            0,
            properties
        );

        calcBitMap.dispatch(dispatchRequest);

        assertThat(dispatchRequest.getBitMap()).isNull();
    }
}
 

/**
 * 只有在消息中指定了keys或者uniqueKey,才会构建索引
 *
 * @param indexFile
 * @param msg
 * @param idxKey    topic#keys;topic#uniqueKey
 * @return
 */
private IndexFile putKey(IndexFile indexFile, DispatchRequest msg, String idxKey) {
    for (boolean ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp()); !ok; ) {
        log.warn("Index file [" + indexFile.getFileName() + "] is full, trying to create another one");

        indexFile = retryGetAndCreateIndexFile();
        if (null == indexFile) {
            return null;
        }

        ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp());
    }

    return indexFile;
}
 

protected DefaultMessageStore gen(ConsumerFilterManager filterManager) throws Exception {
    MessageStoreConfig messageStoreConfig = buildStoreConfig(
        commitLogFileSize, cqFileSize, true, cqExtFileSize
    );

    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);
    brokerConfig.setMaxErrorRateOfBloomFilter(20);
    brokerConfig.setExpectConsumerNumUseFilter(64);

    DefaultMessageStore master = new DefaultMessageStore(
        messageStoreConfig,
        new BrokerStatsManager(brokerConfig.getBrokerClusterName()),
        new MessageArrivingListener() {
            @Override
            public void arriving(String topic, int queueId, long logicOffset, long tagsCode,
                                 long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {
            }
        }
        , brokerConfig);

    master.getDispatcherList().addFirst(new CommitLogDispatcher() {
        @Override
        public void dispatch(DispatchRequest request) {
            try {
            } catch (Throwable e) {
                e.printStackTrace();
            }
        }
    });
    master.getDispatcherList().addFirst(new CommitLogDispatcherCalcBitMap(brokerConfig, filterManager));

    assertThat(master.load()).isTrue();

    master.start();

    return master;
}
 

@Test
public void testDispatch_blankFilterData() {
    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);

    ConsumerFilterManager filterManager = new ConsumerFilterManager();

    CommitLogDispatcherCalcBitMap calcBitMap = new CommitLogDispatcherCalcBitMap(brokerConfig,
        filterManager);

    for (int i = 0; i < 10; i++) {
        Map<String, String> properties = new HashMap<String, String>(4);
        properties.put("a", String.valueOf(i * 10 + 5));

        String topic = "topic" + i;

        DispatchRequest dispatchRequest = new DispatchRequest(
            topic,
            0,
            i * 100 + 123,
            100,
            (long) ("tags" + i).hashCode(),
            System.currentTimeMillis(),
            i,
            null,
            UUID.randomUUID().toString(),
            0,
            0,
            properties
        );

        calcBitMap.dispatch(dispatchRequest);

        assertThat(dispatchRequest.getBitMap()).isNull();
    }
}
 

protected DefaultMessageStore gen(ConsumerFilterManager filterManager) throws Exception {
    MessageStoreConfig messageStoreConfig = buildStoreConfig(
        commitLogFileSize, cqFileSize, true, cqExtFileSize
    );

    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);
    brokerConfig.setMaxErrorRateOfBloomFilter(20);
    brokerConfig.setExpectConsumerNumUseFilter(64);

    DefaultMessageStore master = new DefaultMessageStore(
        messageStoreConfig,
        new BrokerStatsManager(brokerConfig.getBrokerClusterName()),
        new MessageArrivingListener() {
            @Override
            public void arriving(String topic, int queueId, long logicOffset, long tagsCode,
                                 long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {
            }
        }
        , brokerConfig);

    master.getDispatcherList().addFirst(new CommitLogDispatcher() {
        @Override
        public void dispatch(DispatchRequest request) {
            try {
            } catch (Throwable e) {
                e.printStackTrace();
            }
        }
    });
    master.getDispatcherList().addFirst(new CommitLogDispatcherCalcBitMap(brokerConfig, filterManager));

    assertThat(master.load()).isTrue();

    master.start();

    return master;
}
 

private IndexFile putKey(IndexFile indexFile, DispatchRequest msg, String idxKey) {
    for (boolean ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp()); !ok; ) {
        log.warn("Index file [" + indexFile.getFileName() + "] is full, trying to create another one");

        indexFile = retryGetAndCreateIndexFile();
        if (null == indexFile) {
            return null;
        }

        ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp());
    }

    return indexFile;
}
 

@Test
public void testDispatch_blankFilterData() {
    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);

    ConsumerFilterManager filterManager = new ConsumerFilterManager();

    CommitLogDispatcherCalcBitMap calcBitMap = new CommitLogDispatcherCalcBitMap(brokerConfig,
        filterManager);

    for (int i = 0; i < 10; i++) {
        Map<String, String> properties = new HashMap<String, String>(4);
        properties.put("a", String.valueOf(i * 10 + 5));

        String topic = "topic" + i;

        DispatchRequest dispatchRequest = new DispatchRequest(
            topic,
            0,
            i * 100 + 123,
            100,
            (long) ("tags" + i).hashCode(),
            System.currentTimeMillis(),
            i,
            null,
            UUID.randomUUID().toString(),
            0,
            0,
            properties
        );

        calcBitMap.dispatch(dispatchRequest);

        assertThat(dispatchRequest.getBitMap()).isNull();
    }
}
 

/**
 * 添加消息
 * @param indexFile ;
 * @param msg ;
 * @param idxKey ;
 * @return ;
 */
private IndexFile putKey(IndexFile indexFile, DispatchRequest msg, String idxKey) {
    for (boolean ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp()); !ok; ) {
        log.warn("Index file [" + indexFile.getFileName() + "] is full, trying to create another one");

        indexFile = retryGetAndCreateIndexFile();
        if (null == indexFile) {
            return null;
        }

        ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp());
    }

    return indexFile;
}
 

private IndexFile putKey(IndexFile indexFile, DispatchRequest msg, String idxKey) {
    for (boolean ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp()); !ok; ) {
        log.warn("Index file [" + indexFile.getFileName() + "] is full, trying to create another one");

        indexFile = retryGetAndCreateIndexFile();
        if (null == indexFile) {
            return null;
        }

        ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp());
    }

    return indexFile;
}
 

@Test
public void testDispatch_blankFilterData() {
    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);

    ConsumerFilterManager filterManager = new ConsumerFilterManager();

    CommitLogDispatcherCalcBitMap calcBitMap = new CommitLogDispatcherCalcBitMap(brokerConfig,
        filterManager);

    for (int i = 0; i < 10; i++) {
        Map<String, String> properties = new HashMap<String, String>(4);
        properties.put("a", String.valueOf(i * 10 + 5));

        String topic = "topic" + i;

        DispatchRequest dispatchRequest = new DispatchRequest(
            topic,
            0,
            i * 100 + 123,
            100,
            (long) ("tags" + i).hashCode(),
            System.currentTimeMillis(),
            i,
            null,
            UUID.randomUUID().toString(),
            0,
            0,
            properties
        );

        calcBitMap.dispatch(dispatchRequest);

        assertThat(dispatchRequest.getBitMap()).isNull();
    }
}
 

protected DefaultMessageStore gen(ConsumerFilterManager filterManager) throws Exception {
    MessageStoreConfig messageStoreConfig = buildStoreConfig(
        commitLogFileSize, cqFileSize, true, cqExtFileSize
    );

    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);
    brokerConfig.setMaxErrorRateOfBloomFilter(20);
    brokerConfig.setExpectConsumerNumUseFilter(64);

    DefaultMessageStore master = new DefaultMessageStore(
        messageStoreConfig,
        new BrokerStatsManager(brokerConfig.getBrokerClusterName()),
        new MessageArrivingListener() {
            @Override
            public void arriving(String topic, int queueId, long logicOffset, long tagsCode,
                                 long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {
            }
        }
        , brokerConfig);

    master.getDispatcherList().addFirst(new CommitLogDispatcher() {
        @Override
        public void dispatch(DispatchRequest request) {
            try {
            } catch (Throwable e) {
                e.printStackTrace();
            }
        }
    });
    master.getDispatcherList().addFirst(new CommitLogDispatcherCalcBitMap(brokerConfig, filterManager));

    assertThat(master.load()).isTrue();

    master.start();

    return master;
}
 

/**
 * 只有在消息中指定了keys或者uniqueKey,才会构建索引
 * {@linkplain MessageSysFlag#TRANSACTION_ROLLBACK_TYPE}类型消息不会构建索引
 *
 * @param req
 */
public void buildIndex(DispatchRequest req) {
    IndexFile indexFile = retryGetAndCreateIndexFile();
    if (indexFile != null) {
        long endPhyOffset = indexFile.getEndPhyOffset();
        DispatchRequest msg = req;
        String topic = msg.getTopic();
        String keys = msg.getKeys();  //默认值是""
        if (msg.getCommitLogOffset() < endPhyOffset) {  //待构建消息的偏移量应大于索引文件最大物理偏移量
            return;
        }

        final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag());
        switch (tranType) {
            case MessageSysFlag.TRANSACTION_NOT_TYPE:
            case MessageSysFlag.TRANSACTION_PREPARED_TYPE:
            case MessageSysFlag.TRANSACTION_COMMIT_TYPE:
                break;
            // 事务类型为ROLLBACK_TYPE的消息不会进IndexFile
            case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE:
                return;
        }
        //若消息体中指定了uniqueKey属性(Producer创建时Client都会生成)
        if (req.getUniqKey() != null) {
            // key = topic#uniqueKey
            indexFile = putKey(indexFile, msg, buildKey(topic, req.getUniqKey()));
            if (indexFile == null) {
                log.error("putKey error commitlog {} uniqkey {}", req.getCommitLogOffset(), req.getUniqKey());
                return;
            }
        }

        if (keys != null && keys.length() > 0) {   //若消息中未指定keys及uniqueKey,则不会构建索引
            String[] keyset = keys.split(MessageConst.KEY_SEPARATOR);
            for (int i = 0; i < keyset.length; i++) {
                String key = keyset[i];
                if (key.length() > 0) {
                    indexFile = putKey(indexFile, msg, buildKey(topic, key));  //key = topic#key
                    if (indexFile == null) {
                        log.error("putKey error commitlog {} uniqkey {}", req.getCommitLogOffset(), req.getUniqKey());
                        return;
                    }
                }
            }
        }
    } else {
        log.error("build index error, stop building index");
    }
}
 

@Test
public void testDispatch_filterDataIllegal() {
    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);

    ConsumerFilterManager filterManager = new ConsumerFilterManager();

    filterManager.register("topic0", "CID_0", "a is not null and a >= 5",
        ExpressionType.SQL92, System.currentTimeMillis());

    filterManager.register("topic0", "CID_1", "a is not null and a >= 15",
        ExpressionType.SQL92, System.currentTimeMillis());

    ConsumerFilterData nullExpression = filterManager.get("topic0", "CID_0");
    nullExpression.setExpression(null);
    nullExpression.setCompiledExpression(null);
    ConsumerFilterData nullBloomData = filterManager.get("topic0", "CID_1");
    nullBloomData.setBloomFilterData(null);

    CommitLogDispatcherCalcBitMap calcBitMap = new CommitLogDispatcherCalcBitMap(brokerConfig,
        filterManager);

    for (int i = 0; i < 1; i++) {
        Map<String, String> properties = new HashMap<String, String>(4);
        properties.put("a", String.valueOf(i * 10 + 5));

        String topic = "topic" + i;

        DispatchRequest dispatchRequest = new DispatchRequest(
            topic,
            0,
            i * 100 + 123,
            100,
            (long) ("tags" + i).hashCode(),
            System.currentTimeMillis(),
            i,
            null,
            UUID.randomUUID().toString(),
            0,
            0,
            properties
        );

        calcBitMap.dispatch(dispatchRequest);

        assertThat(dispatchRequest.getBitMap()).isNotNull();

        BitsArray bitsArray = BitsArray.create(dispatchRequest.getBitMap(),
            filterManager.getBloomFilter().getM());

        for (int j = 0; j < bitsArray.bitLength(); j++) {
            assertThat(bitsArray.getBit(j)).isFalse();
        }
    }
}
 

public void buildIndex(DispatchRequest req) {
    IndexFile indexFile = retryGetAndCreateIndexFile();
    if (indexFile != null) {
        long endPhyOffset = indexFile.getEndPhyOffset();
        DispatchRequest msg = req;
        String topic = msg.getTopic();
        String keys = msg.getKeys();
        if (msg.getCommitLogOffset() < endPhyOffset) {
            return;
        }

        final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag());
        switch (tranType) {
            case MessageSysFlag.TRANSACTION_NOT_TYPE:
            case MessageSysFlag.TRANSACTION_PREPARED_TYPE:
            case MessageSysFlag.TRANSACTION_COMMIT_TYPE:
                break;
            case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE:
                return;
        }

        if (req.getUniqKey() != null) {
            indexFile = putKey(indexFile, msg, buildKey(topic, req.getUniqKey()));
            if (indexFile == null) {
                log.error("putKey error commitlog {} uniqkey {}", req.getCommitLogOffset(), req.getUniqKey());
                return;
            }
        }

        if (keys != null && keys.length() > 0) {
            String[] keyset = keys.split(MessageConst.KEY_SEPARATOR);
            for (int i = 0; i < keyset.length; i++) {
                String key = keyset[i];
                if (key.length() > 0) {
                    indexFile = putKey(indexFile, msg, buildKey(topic, key));
                    if (indexFile == null) {
                        log.error("putKey error commitlog {} uniqkey {}", req.getCommitLogOffset(), req.getUniqKey());
                        return;
                    }
                }
            }
        }
    } else {
        log.error("build index error, stop building index");
    }
}
 

/**
 * 构造索引。当commitlog添加成功后构造索引
 * 构造keys的每一个key的索引文件和uniquekeys的文件
 * @param req req
 */
public void buildIndex(DispatchRequest req) {

    IndexFile indexFile = retryGetAndCreateIndexFile();
    if (indexFile != null) {

        long endPhyOffset = indexFile.getEndPhyOffset();
        DispatchRequest msg = req;

        String topic = msg.getTopic();
        String keys = msg.getKeys();

        //已经提交过偏移量了，直接return
        if (msg.getCommitLogOffset() < endPhyOffset) {
            return;
        }

        final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag());
        switch (tranType) {
            case MessageSysFlag.TRANSACTION_NOT_TYPE:
            case MessageSysFlag.TRANSACTION_PREPARED_TYPE:
            case MessageSysFlag.TRANSACTION_COMMIT_TYPE:
                break;
            case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE:
                return;
        }

        if (req.getUniqKey() != null) {
            indexFile = putKey(indexFile, msg, buildKey(topic, req.getUniqKey()));
            if (indexFile == null) {
                log.error("putKey error commitlog {} uniqkey {}", req.getCommitLogOffset(), req.getUniqKey());
                return;
            }
        }

        //遍历keys循环添加索引文件
        if (keys != null && keys.length() > 0) {
            String[] keyset = keys.split(MessageConst.KEY_SEPARATOR);
            for (int i = 0; i < keyset.length; i++) {
                String key = keyset[i];
                if (key.length() > 0) {
                    //添加到索引文件
                    indexFile = putKey(indexFile, msg, buildKey(topic, key));
                    if (indexFile == null) {
                        log.error("putKey error commitlog {} uniqkey {}", req.getCommitLogOffset(), req.getUniqKey());
                        return;
                    }
                }
            }
        }
    } else {
        log.error("build index error, stop building index");
    }
}
 

@Override
public DispatchRequest checkMessageAndReturnSize(ByteBuffer byteBuffer, final boolean checkCRC) {
    return this.checkMessageAndReturnSize(byteBuffer, checkCRC, true);
}
 

public void buildIndex(DispatchRequest req) {
    IndexFile indexFile = retryGetAndCreateIndexFile();
    if (indexFile != null) {
        long endPhyOffset = indexFile.getEndPhyOffset();
        DispatchRequest msg = req;
        String topic = msg.getTopic();
        String keys = msg.getKeys();
        Map<String, String> properties = msg.getPropertiesMap();
        if (msg.getCommitLogOffset() < endPhyOffset) {
            return;
        }

        final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag());
        switch (tranType) {
            case MessageSysFlag.TRANSACTION_NOT_TYPE:
            case MessageSysFlag.TRANSACTION_PREPARED_TYPE:
            case MessageSysFlag.TRANSACTION_COMMIT_TYPE:
                break;
            case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE:
                return;
        }

        if (req.getUniqKey() != null) {
            indexFile = putKey(indexFile, msg, buildKey(topic, req.getUniqKey()));
            if (indexFile == null) {
                log.error("putKey error commitlog {} uniqkey {}", req.getCommitLogOffset(), req.getUniqKey());
                return;
            }
        }

        if (keys != null && keys.length() > 0) {
            String[] keyset = keys.split(MessageConst.KEY_SEPARATOR);
            for (int i = 0; i < keyset.length; i++) {
                String key = keyset[i];
                if (key.length() > 0) {
                    indexFile = putKey(indexFile, msg, buildKey(topic, key));
                    if (indexFile == null) {
                        log.error("putKey error commitlog {} key {}", req.getCommitLogOffset(), key);
                        return;
                    }
                }
            }
        }

        if (properties != null & properties.containsKey(MessageConst.PROPERTY_USER_TRACE_ID)) {
            String traceId = properties.get(MessageConst.PROPERTY_USER_TRACE_ID);
            if (traceId != null) {
                indexFile = putKey(indexFile, msg, buildKey(topic, traceId));
                if (indexFile == null) {
                    log.error("putKey error commitlog {} traceId {}", req.getCommitLogOffset(), traceId);
                    return;
                }
            }
        }
    } else {
        log.error("build index error, stop building index");
    }
}
 

@Test
public void testDispatch_filterDataIllegal() {
    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);

    ConsumerFilterManager filterManager = new ConsumerFilterManager();

    filterManager.register("topic0", "CID_0", "a is not null and a >= 5",
        ExpressionType.SQL92, System.currentTimeMillis());

    filterManager.register("topic0", "CID_1", "a is not null and a >= 15",
        ExpressionType.SQL92, System.currentTimeMillis());

    ConsumerFilterData nullExpression = filterManager.get("topic0", "CID_0");
    nullExpression.setExpression(null);
    nullExpression.setCompiledExpression(null);
    ConsumerFilterData nullBloomData = filterManager.get("topic0", "CID_1");
    nullBloomData.setBloomFilterData(null);

    CommitLogDispatcherCalcBitMap calcBitMap = new CommitLogDispatcherCalcBitMap(brokerConfig,
        filterManager);

    for (int i = 0; i < 1; i++) {
        Map<String, String> properties = new HashMap<String, String>(4);
        properties.put("a", String.valueOf(i * 10 + 5));

        String topic = "topic" + i;

        DispatchRequest dispatchRequest = new DispatchRequest(
            topic,
            0,
            i * 100 + 123,
            100,
            (long) ("tags" + i).hashCode(),
            System.currentTimeMillis(),
            i,
            null,
            UUID.randomUUID().toString(),
            0,
            0,
            properties
        );

        calcBitMap.dispatch(dispatchRequest);

        assertThat(dispatchRequest.getBitMap()).isNotNull();

        BitsArray bitsArray = BitsArray.create(dispatchRequest.getBitMap(),
            filterManager.getBloomFilter().getM());

        for (int j = 0; j < bitsArray.bitLength(); j++) {
            assertThat(bitsArray.getBit(j)).isFalse();
        }
    }
}