Java Code Examples for org.apache.rocketmq.filter.util.BitsArray#create()

The following examples show how to use org.apache.rocketmq.filter.util.BitsArray#create() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: BloomFilterTest.java    From rocketmq-all-4.1.0-incubating with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCheckFalseHit() {
    BloomFilter bloomFilter = BloomFilter.createByFn(1, 300);
    BitsArray bits = BitsArray.create(bloomFilter.getM());
    int falseHit = 0;
    for (int i = 0; i < bloomFilter.getN(); i++) {
        String str = randomString((new Random(System.nanoTime())).nextInt(127) + 10);
        int[] bitPos = bloomFilter.calcBitPositions(str);

        if (bloomFilter.checkFalseHit(bitPos, bits)) {
            falseHit++;
        }

        bloomFilter.hashTo(bitPos, bits);
    }

    assertThat(falseHit).isLessThanOrEqualTo(bloomFilter.getF() * bloomFilter.getN() / 100);
}
Example 2
Source File: BloomFilterTest.java    From rocketmq-4.3.0 with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCheckFalseHit() {
    BloomFilter bloomFilter = BloomFilter.createByFn(1, 300);
    BitsArray bits = BitsArray.create(bloomFilter.getM());
    int falseHit = 0;
    for (int i = 0; i < bloomFilter.getN(); i++) {
        String str = randomString((new Random(System.nanoTime())).nextInt(127) + 10);
        int[] bitPos = bloomFilter.calcBitPositions(str);

        if (bloomFilter.checkFalseHit(bitPos, bits)) {
            falseHit++;
        }

        bloomFilter.hashTo(bitPos, bits);
    }

    assertThat(falseHit).isLessThanOrEqualTo(bloomFilter.getF() * bloomFilter.getN() / 100);
}
Example 3
Source File: BitsArrayTest.java    From rocketmq-all-4.1.0-incubating with Apache License 2.0 5 votes vote down vote up 
@Test
public void testOr() {
    BitsArray b1 = BitsArray.create(new byte[]{(byte) 0xff, 0x00});
    BitsArray b2 = BitsArray.create(new byte[]{0x00, (byte) 0xff});

    b1.or(b2);

    for (int i = 0; i < b1.bitLength(); i++) {
        assertThat(b1.getBit(i)).isTrue();
    }
}
Example 4
Source File: BitsArrayTest.java    From DDMQ with Apache License 2.0 5 votes vote down vote up 
@Test
public void testOr() {
    BitsArray b1 = BitsArray.create(new byte[] {(byte) 0xff, 0x00});
    BitsArray b2 = BitsArray.create(new byte[] {0x00, (byte) 0xff});

    b1.or(b2);

    for (int i = 0; i < b1.bitLength(); i++) {
        assertThat(b1.getBit(i)).isTrue();
    }
}
Example 5
Source File: BitsArrayTest.java    From rocketmq-all-4.1.0-incubating with Apache License 2.0 5 votes vote down vote up 
BitsArray gen(int bitCount) {
    BitsArray bitsArray = BitsArray.create(bitCount);

    for (int i = 0; i < bitCount / Byte.SIZE; i++) {
        bitsArray.setByte(i, (byte) (new Random(System.currentTimeMillis())).nextInt(0xff));
        try {
            Thread.sleep(2);
        } catch (InterruptedException e) {
        }
    }

    return bitsArray;
}
Example 6
Source File: BitsArrayTest.java    From rocketmq-4.3.0 with Apache License 2.0 5 votes vote down vote up 
@Test
public void testOr() {
    BitsArray b1 = BitsArray.create(new byte[] {(byte) 0xff, 0x00});
    BitsArray b2 = BitsArray.create(new byte[] {0x00, (byte) 0xff});

    b1.or(b2);

    for (int i = 0; i < b1.bitLength(); i++) {
        assertThat(b1.getBit(i)).isTrue();
    }
}
Example 7
Source File: BitsArrayTest.java    From rocketmq with Apache License 2.0 5 votes vote down vote up 
BitsArray gen(int bitCount) {
    BitsArray bitsArray = BitsArray.create(bitCount);

    for (int i = 0; i < bitCount / Byte.SIZE; i++) {
        bitsArray.setByte(i, (byte) (new Random(System.currentTimeMillis())).nextInt(0xff));
        try {
            Thread.sleep(2);
        } catch (InterruptedException e) {
        }
    }

    return bitsArray;
}
Example 8
Source File: BitsArrayTest.java    From DDMQ with Apache License 2.0 5 votes vote down vote up 
BitsArray gen(int bitCount) {
    BitsArray bitsArray = BitsArray.create(bitCount);

    for (int i = 0; i < bitCount / Byte.SIZE; i++) {
        bitsArray.setByte(i, (byte) (new Random(System.currentTimeMillis())).nextInt(0xff));
        try {
            Thread.sleep(2);
        } catch (InterruptedException e) {
        }
    }

    return bitsArray;
}
Example 9
Source File: BloomFilterTest.java    From rocketmq-4.3.0 with Apache License 2.0 4 votes vote down vote up 
@Test
public void testHashTo() {
    String cid = "CID_abc_efg";

    BloomFilter bloomFilter = BloomFilter.createByFn(10, 20);

    BitsArray bits = BitsArray.create(bloomFilter.getM());

    int[] bitPos = bloomFilter.calcBitPositions(cid);

    bloomFilter.hashTo(cid, bits);

    for (int bit : bitPos) {
        assertThat(bits.getBit(bit)).isTrue();
    }
}
Example 10
Source File: BloomFilterTest.java    From rocketmq with Apache License 2.0 4 votes vote down vote up 
@Test
public void testIsHit() {
    String cid = "CID_abc_efg";
    String cid2 = "CID_abc_123";

    BloomFilter bloomFilter = BloomFilter.createByFn(10, 20);

    BitsArray bits = BitsArray.create(bloomFilter.getM());

    bloomFilter.hashTo(cid, bits);

    assertThat(bloomFilter.isHit(cid, bits)).isTrue();
    assertThat(!bloomFilter.isHit(cid2, bits)).isTrue();

    bloomFilter.hashTo(cid2, bits);

    assertThat(bloomFilter.isHit(cid, bits)).isTrue();
    assertThat(bloomFilter.isHit(cid2, bits)).isTrue();
}
Example 11
Source File: BloomFilterTest.java    From rocketmq-all-4.1.0-incubating with Apache License 2.0 4 votes vote down vote up 
@Test
public void testHashTo() {
    String cid = "CID_abc_efg";

    BloomFilter bloomFilter = BloomFilter.createByFn(10, 20);

    BitsArray bits = BitsArray.create(bloomFilter.getM());

    int[] bitPos = bloomFilter.calcBitPositions(cid);

    bloomFilter.hashTo(cid, bits);

    for (int bit : bitPos) {
        assertThat(bits.getBit(bit)).isTrue();
    }
}
Example 12
Source File: ExpressionMessageFilter.java    From DDMQ with Apache License 2.0 4 votes vote down vote up 
@Override
public boolean isMatchedByConsumeQueue(Long tagsCode, ConsumeQueueExt.CqExtUnit cqExtUnit) {
    if (null == subscriptionData) {
        return true;
    }

    if (subscriptionData.isClassFilterMode()) {
        return true;
    }

    // by tags code.
    if (ExpressionType.isTagType(subscriptionData.getExpressionType())) {

        if (tagsCode == null) {
            return true;
        }

        if (subscriptionData.getSubString().equals(SubscriptionData.SUB_ALL)) {
            return true;
        }

        return subscriptionData.getCodeSet().contains(tagsCode.intValue());
    } else {
        // no expression or no bloom
        if (consumerFilterData == null || consumerFilterData.getExpression() == null
            || consumerFilterData.getCompiledExpression() == null || consumerFilterData.getBloomFilterData() == null) {
            return true;
        }

        // message is before consumer
        if (cqExtUnit == null || !consumerFilterData.isMsgInLive(cqExtUnit.getMsgStoreTime())) {
            log.debug("Pull matched because not in live: {}, {}", consumerFilterData, cqExtUnit);
            return true;
        }

        byte[] filterBitMap = cqExtUnit.getFilterBitMap();
        BloomFilter bloomFilter = this.consumerFilterManager.getBloomFilter();
        if (filterBitMap == null || !this.bloomDataValid
            || filterBitMap.length * Byte.SIZE != consumerFilterData.getBloomFilterData().getBitNum()) {
            return true;
        }

        BitsArray bitsArray = null;
        try {
            bitsArray = BitsArray.create(filterBitMap);
            boolean ret = bloomFilter.isHit(consumerFilterData.getBloomFilterData(), bitsArray);
            log.debug("Pull {} by bit map:{}, {}, {}", ret, consumerFilterData, bitsArray, cqExtUnit);
            return ret;
        } catch (Throwable e) {
            log.error("bloom filter error, sub=" + subscriptionData
                + ", filter=" + consumerFilterData + ", bitMap=" + bitsArray, e);
        }
    }

    return true;
}
Example 13
Source File: ExpressionMessageFilter.java    From rocketmq-4.3.0 with Apache License 2.0 4 votes vote down vote up 
@Override
public boolean isMatchedByConsumeQueue(Long tagsCode, ConsumeQueueExt.CqExtUnit cqExtUnit) {
    if (null == subscriptionData) {
        return true;
    }

    if (subscriptionData.isClassFilterMode()) {
        return true;
    }

    // by tags code.
    if (ExpressionType.isTagType(subscriptionData.getExpressionType())) {

        if (tagsCode == null) {
            return true;
        }

        if (subscriptionData.getSubString().equals(SubscriptionData.SUB_ALL)) {
            return true;
        }

        return subscriptionData.getCodeSet().contains(tagsCode.intValue());
    } else {
        // no expression or no bloom
        if (consumerFilterData == null || consumerFilterData.getExpression() == null
            || consumerFilterData.getCompiledExpression() == null || consumerFilterData.getBloomFilterData() == null) {
            return true;
        }

        // message is before consumer
        if (cqExtUnit == null || !consumerFilterData.isMsgInLive(cqExtUnit.getMsgStoreTime())) {
            log.debug("Pull matched because not in live: {}, {}", consumerFilterData, cqExtUnit);
            return true;
        }

        byte[] filterBitMap = cqExtUnit.getFilterBitMap();
        BloomFilter bloomFilter = this.consumerFilterManager.getBloomFilter();
        if (filterBitMap == null || !this.bloomDataValid
            || filterBitMap.length * Byte.SIZE != consumerFilterData.getBloomFilterData().getBitNum()) {
            return true;
        }

        BitsArray bitsArray = null;
        try {
            bitsArray = BitsArray.create(filterBitMap);
            boolean ret = bloomFilter.isHit(consumerFilterData.getBloomFilterData(), bitsArray);
            log.debug("Pull {} by bit map:{}, {}, {}", ret, consumerFilterData, bitsArray, cqExtUnit);
            return ret;
        } catch (Throwable e) {
            log.error("bloom filter error, sub=" + subscriptionData
                + ", filter=" + consumerFilterData + ", bitMap=" + bitsArray, e);
        }
    }

    return true;
}
Example 14
Source File: ExpressionMessageFilter.java    From DDMQ with Apache License 2.0 4 votes vote down vote up 
@Override
public boolean isMatchedByConsumeQueue(Long tagsCode, ConsumeQueueExt.CqExtUnit cqExtUnit) {
    if (null == subscriptionData) {
        return true;
    }

    if (subscriptionData.isClassFilterMode()) {
        return true;
    }

    // by tags code.
    if (ExpressionType.isTagType(subscriptionData.getExpressionType())) {

        if (tagsCode == null) {
            return true;
        }

        if (subscriptionData.getSubString().equals(SubscriptionData.SUB_ALL)) {
            return true;
        }

        return subscriptionData.getCodeSet().contains(tagsCode.intValue());
    } else {
        // no expression or no bloom
        if (consumerFilterData == null || consumerFilterData.getExpression() == null
            || consumerFilterData.getCompiledExpression() == null || consumerFilterData.getBloomFilterData() == null) {
            return true;
        }

        // message is before consumer
        if (cqExtUnit == null || !consumerFilterData.isMsgInLive(cqExtUnit.getMsgStoreTime())) {
            log.debug("Pull matched because not in live: {}, {}", consumerFilterData, cqExtUnit);
            return true;
        }

        byte[] filterBitMap = cqExtUnit.getFilterBitMap();
        BloomFilter bloomFilter = this.consumerFilterManager.getBloomFilter();
        if (filterBitMap == null || !this.bloomDataValid
            || filterBitMap.length * Byte.SIZE != consumerFilterData.getBloomFilterData().getBitNum()) {
            return true;
        }

        BitsArray bitsArray = null;
        try {
            bitsArray = BitsArray.create(filterBitMap);
            boolean ret = bloomFilter.isHit(consumerFilterData.getBloomFilterData(), bitsArray);
            log.debug("Pull {} by bit map:{}, {}, {}", ret, consumerFilterData, bitsArray, cqExtUnit);
            return ret;
        } catch (Throwable e) {
            log.error("bloom filter error, sub=" + subscriptionData
                + ", filter=" + consumerFilterData + ", bitMap=" + bitsArray, e);
        }
    }

    return true;
}
Example 15
Source File: CommitLogDispatcherCalcBitMapTest.java    From DDMQ with Apache License 2.0 4 votes vote down vote up 
@Test
public void testDispatch_filterDataIllegal() {
    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);

    ConsumerFilterManager filterManager = new ConsumerFilterManager();

    filterManager.register("topic0", "CID_0", "a is not null and a >= 5",
        ExpressionType.SQL92, System.currentTimeMillis());

    filterManager.register("topic0", "CID_1", "a is not null and a >= 15",
        ExpressionType.SQL92, System.currentTimeMillis());

    ConsumerFilterData nullExpression = filterManager.get("topic0", "CID_0");
    nullExpression.setExpression(null);
    nullExpression.setCompiledExpression(null);
    ConsumerFilterData nullBloomData = filterManager.get("topic0", "CID_1");
    nullBloomData.setBloomFilterData(null);

    CommitLogDispatcherCalcBitMap calcBitMap = new CommitLogDispatcherCalcBitMap(brokerConfig,
        filterManager);

    for (int i = 0; i < 1; i++) {
        Map<String, String> properties = new HashMap<String, String>(4);
        properties.put("a", String.valueOf(i * 10 + 5));

        String topic = "topic" + i;

        DispatchRequest dispatchRequest = new DispatchRequest(
            topic,
            0,
            i * 100 + 123,
            100,
            (long) ("tags" + i).hashCode(),
            System.currentTimeMillis(),
            i,
            null,
            UUID.randomUUID().toString(),
            0,
            0,
            properties
        );

        calcBitMap.dispatch(dispatchRequest);

        assertThat(dispatchRequest.getBitMap()).isNotNull();

        BitsArray bitsArray = BitsArray.create(dispatchRequest.getBitMap(),
            filterManager.getBloomFilter().getM());

        for (int j = 0; j < bitsArray.bitLength(); j++) {
            assertThat(bitsArray.getBit(j)).isFalse();
        }
    }
}
Example 16
Source File: CommitLogDispatcherCalcBitMapTest.java    From rocketmq-all-4.1.0-incubating with Apache License 2.0 4 votes vote down vote up 
@Test
public void testDispatch_filterDataIllegal() {
    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);

    ConsumerFilterManager filterManager = new ConsumerFilterManager();

    filterManager.register("topic0", "CID_0", "a is not null and a >= 5",
        ExpressionType.SQL92, System.currentTimeMillis());

    filterManager.register("topic0", "CID_1", "a is not null and a >= 15",
        ExpressionType.SQL92, System.currentTimeMillis());

    ConsumerFilterData nullExpression = filterManager.get("topic0", "CID_0");
    nullExpression.setExpression(null);
    nullExpression.setCompiledExpression(null);
    ConsumerFilterData nullBloomData = filterManager.get("topic0", "CID_1");
    nullBloomData.setBloomFilterData(null);


    CommitLogDispatcherCalcBitMap calcBitMap = new CommitLogDispatcherCalcBitMap(brokerConfig,
        filterManager);

    for (int i = 0; i < 1; i++) {
        Map<String, String> properties = new HashMap<String, String>(4);
        properties.put("a", String.valueOf(i * 10 + 5));

        String topic = "topic" + i;

        DispatchRequest dispatchRequest = new DispatchRequest(
            topic,
            0,
            i * 100 + 123,
            100,
            (long) ("tags" + i).hashCode(),
            System.currentTimeMillis(),
            i,
            null,
            UUID.randomUUID().toString(),
            0,
            0,
            properties
        );

        calcBitMap.dispatch(dispatchRequest);

        assertThat(dispatchRequest.getBitMap()).isNotNull();

        BitsArray bitsArray = BitsArray.create(dispatchRequest.getBitMap(),
            filterManager.getBloomFilter().getM());

        for (int j = 0; j < bitsArray.bitLength(); j++) {
            assertThat(bitsArray.getBit(j)).isFalse();
        }
    }
}
Example 17
Source File: BloomFilterTest.java    From rocketmq with Apache License 2.0 4 votes vote down vote up 
@Test
public void testHashTo() {
    String cid = "CID_abc_efg";

    BloomFilter bloomFilter = BloomFilter.createByFn(10, 20);

    BitsArray bits = BitsArray.create(bloomFilter.getM());

    int[] bitPos = bloomFilter.calcBitPositions(cid);

    bloomFilter.hashTo(cid, bits);

    for (int bit : bitPos) {
        assertThat(bits.getBit(bit)).isTrue();
    }
}
Example 18
Source File: BloomFilterTest.java    From DDMQ with Apache License 2.0 4 votes vote down vote up 
@Test
public void testIsHit() {
    String cid = "CID_abc_efg";
    String cid2 = "CID_abc_123";

    BloomFilter bloomFilter = BloomFilter.createByFn(10, 20);

    BitsArray bits = BitsArray.create(bloomFilter.getM());

    bloomFilter.hashTo(cid, bits);

    assertThat(bloomFilter.isHit(cid, bits)).isTrue();
    assertThat(!bloomFilter.isHit(cid2, bits)).isTrue();

    bloomFilter.hashTo(cid2, bits);

    assertThat(bloomFilter.isHit(cid, bits)).isTrue();
    assertThat(bloomFilter.isHit(cid2, bits)).isTrue();
}
Example 19
Source File: BitsArrayTest.java    From DDMQ with Apache License 2.0 3 votes vote down vote up 
@Test
public void testConstructor() {
    BitsArray bitsArray = BitsArray.create(8);

    assertThat(bitsArray.byteLength() == 1 && bitsArray.bitLength() == 8).isTrue();

    bitsArray = BitsArray.create(9);

    assertThat(bitsArray.byteLength() == 2 && bitsArray.bitLength() == 9).isTrue();

    bitsArray = BitsArray.create(7);

    assertThat(bitsArray.byteLength() == 1 && bitsArray.bitLength() == 7).isTrue();
}
Example 20
Source File: BitsArrayTest.java    From rocketmq with Apache License 2.0 3 votes vote down vote up 
@Test
public void testConstructor() {
    BitsArray bitsArray = BitsArray.create(8);

    assertThat(bitsArray.byteLength() == 1 && bitsArray.bitLength() == 8).isTrue();

    bitsArray = BitsArray.create(9);

    assertThat(bitsArray.byteLength() == 2 && bitsArray.bitLength() == 9).isTrue();

    bitsArray = BitsArray.create(7);

    assertThat(bitsArray.byteLength() == 1 && bitsArray.bitLength() == 7).isTrue();
}