com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter Java Exaples

Source File: AbstractCanalInstance.java From canal-1.1.3 with Apache License 2.0

6 votes

@Override
public boolean subscribeChange(ClientIdentity identity) {
    if (StringUtils.isNotEmpty(identity.getFilter())) {
        logger.info("subscribe filter change to " + identity.getFilter());
        AviaterRegexFilter aviaterFilter = new AviaterRegexFilter(identity.getFilter());

        boolean isGroup = (eventParser instanceof GroupEventParser);
        if (isGroup) {
            // 处理group的模式
            List<CanalEventParser> eventParsers = ((GroupEventParser) eventParser).getEventParsers();
            for (CanalEventParser singleEventParser : eventParsers) {// 需要遍历启动
                ((AbstractEventParser) singleEventParser).setEventFilter(aviaterFilter);
            }
        } else {
            ((AbstractEventParser) eventParser).setEventFilter(aviaterFilter);
        }

    }

    // filter的处理规则
    // a. parser处理数据过滤处理
    // b. sink处理数据的路由&分发,一份parse数据经过sink后可以分发为多份，每份的数据可以根据自己的过滤规则不同而有不同的数据
    // 后续内存版的一对多分发，可以考虑
    return true;
}

Source File: MQMessageUtils.java From canal-1.1.3 with Apache License 2.0

6 votes

public List<DynamicTopicData> apply(String pkHashConfigs) {
    List<DynamicTopicData> datas = Lists.newArrayList();
    String[] dynamicTopicArray = StringUtils.split(StringUtils.replace(pkHashConfigs,
        ",",
        ";"),
        ";");
    // schema.table
    for (String dynamicTopic : dynamicTopicArray) {
        DynamicTopicData data = new DynamicTopicData();

        if (!isWildCard(dynamicTopic)) {
            data.simpleName = dynamicTopic;
        } else {
            if (dynamicTopic.contains("\\.")) {
                data.tableRegexFilter = new AviaterRegexFilter(dynamicTopic);
            } else {
                data.schemaRegexFilter = new AviaterRegexFilter(dynamicTopic);
            }
        }
        datas.add(data);
    }

    return datas;
}

Source File: AviaterFilterTest.java From canal with Apache License 2.0

6 votes

@Test
public void testDisordered() {
    AviaterRegexFilter filter = new AviaterRegexFilter("u\\..*,uvw\\..*,uv\\..*,a\\.x,a\\.xyz,a\\.xy,abc\\.x,abc\\.xyz,abc\\.xy,ab\\.x,ab\\.xyz,ab\\.xy");

    boolean result = filter.filter("u.abc");
    Assert.assertEquals(true, result);

    result = filter.filter("ab.x");
    Assert.assertEquals(true, result);

    result = filter.filter("ab.xyz1");
    Assert.assertEquals(false, result);

    result = filter.filter("abc.xyz");
    Assert.assertEquals(true, result);

    result = filter.filter("uv.xyz");
    Assert.assertEquals(true, result);

}

Source File: AbstractMysqlEventParser.java From canal-1.1.3 with Apache License 2.0

6 votes

protected BinlogParser buildParser() {
    LogEventConvert convert = new LogEventConvert();
    if (eventFilter != null && eventFilter instanceof AviaterRegexFilter) {
        convert.setNameFilter((AviaterRegexFilter) eventFilter);
    }

    if (eventBlackFilter != null && eventBlackFilter instanceof AviaterRegexFilter) {
        convert.setNameBlackFilter((AviaterRegexFilter) eventBlackFilter);
    }

    convert.setCharset(connectionCharset);
    convert.setFilterQueryDcl(filterQueryDcl);
    convert.setFilterQueryDml(filterQueryDml);
    convert.setFilterQueryDdl(filterQueryDdl);
    convert.setFilterRows(filterRows);
    convert.setFilterTableError(filterTableError);
    convert.setUseDruidDdlFilter(useDruidDdlFilter);
    return convert;
}

Source File: AbstractMysqlEventParser.java From canal with Apache License 2.0

6 votes

protected BinlogParser buildParser() {
    LogEventConvert convert = new LogEventConvert();
    if (eventFilter != null && eventFilter instanceof AviaterRegexFilter) {
        convert.setNameFilter((AviaterRegexFilter) eventFilter);
    }

    if (eventBlackFilter != null && eventBlackFilter instanceof AviaterRegexFilter) {
        convert.setNameBlackFilter((AviaterRegexFilter) eventBlackFilter);
    }
    
    convert.setFieldFilterMap(getFieldFilterMap());
    convert.setFieldBlackFilterMap(getFieldBlackFilterMap());

    convert.setCharset(connectionCharset);
    convert.setFilterQueryDcl(filterQueryDcl);
    convert.setFilterQueryDml(filterQueryDml);
    convert.setFilterQueryDdl(filterQueryDdl);
    convert.setFilterRows(filterRows);
    convert.setFilterTableError(filterTableError);
    convert.setUseDruidDdlFilter(useDruidDdlFilter);
    return convert;
}

Source File: AviaterFilterTest.java From canal-1.1.3 with Apache License 2.0

6 votes

@Test
public void testDisordered() {
    AviaterRegexFilter filter = new AviaterRegexFilter("u\\..*,uvw\\..*,uv\\..*,a\\.x,a\\.xyz,a\\.xy,abc\\.x,abc\\.xyz,abc\\.xy,ab\\.x,ab\\.xyz,ab\\.xy");

    boolean result = filter.filter("u.abc");
    Assert.assertEquals(true, result);

    result = filter.filter("ab.x");
    Assert.assertEquals(true, result);

    result = filter.filter("ab.xyz1");
    Assert.assertEquals(false, result);

    result = filter.filter("abc.xyz");
    Assert.assertEquals(true, result);

    result = filter.filter("uv.xyz");
    Assert.assertEquals(true, result);

}

Source File: MQMessageUtils.java From canal-1.1.3 with Apache License 2.0

5 votes

public List<PartitionData> apply(String pkHashConfigs) {
    List<PartitionData> datas = Lists.newArrayList();

    String[] pkHashConfigArray = StringUtils.split(StringUtils.replace(pkHashConfigs,
        ",",
        ";"),
        ";");
    // schema.table:id^name
    for (String pkHashConfig : pkHashConfigArray) {
        PartitionData data = new PartitionData();
        int i = pkHashConfig.lastIndexOf(":");
        if (i > 0) {
            String pkStr = pkHashConfig.substring(i + 1);
            if (pkStr.equalsIgnoreCase("$pk$")) {
                data.hashMode.autoPkHash = true;
            } else {
                data.hashMode.pkNames = Lists.newArrayList(StringUtils.split(pkStr,
                    '^'));
            }

            pkHashConfig = pkHashConfig.substring(0,
                i);
        } else {
            data.hashMode.tableHash = true;
        }

        if (!isWildCard(pkHashConfig)) {
            data.simpleName = pkHashConfig;
        } else {
            data.regexFilter = new AviaterRegexFilter(pkHashConfig);
        }
        datas.add(data);
    }

    return datas;
}

Source File: MutliAviaterFilterTest.java From canal with Apache License 2.0

5 votes

private void doRegexTest() {
    AviaterRegexFilter filter3 = new AviaterRegexFilter("otter2.otter_stability1|otter1.otter_stability1|"
                                                        + RandomStringUtils.randomAlphabetic(200));
    boolean result = filter3.filter("otter1.otter_stability1");
    Assert.assertEquals(true, result);
    result = filter3.filter("otter2.otter_stability1");
    Assert.assertEquals(true, result);
}

Source File: AbstractMysqlEventParser.java From canal with Apache License 2.0

5 votes

public void setEventBlackFilter(CanalEventFilter eventBlackFilter) {
    super.setEventBlackFilter(eventBlackFilter);

    // 触发一下filter变更
    if (eventBlackFilter != null && eventBlackFilter instanceof AviaterRegexFilter) {
        if (binlogParser instanceof LogEventConvert) {
            ((LogEventConvert) binlogParser).setNameBlackFilter((AviaterRegexFilter) eventBlackFilter);
        }

        if (tableMetaTSDB != null && tableMetaTSDB instanceof DatabaseTableMeta) {
            ((DatabaseTableMeta) tableMetaTSDB).setBlackFilter(eventBlackFilter);
        }
    }
}

Source File: AbstractMysqlEventParser.java From canal with Apache License 2.0

5 votes

public void setEventFilter(CanalEventFilter eventFilter) {
    super.setEventFilter(eventFilter);

    // 触发一下filter变更
    if (eventFilter != null && eventFilter instanceof AviaterRegexFilter) {
        if (binlogParser instanceof LogEventConvert) {
            ((LogEventConvert) binlogParser).setNameFilter((AviaterRegexFilter) eventFilter);
        }

        if (tableMetaTSDB != null && tableMetaTSDB instanceof DatabaseTableMeta) {
            ((DatabaseTableMeta) tableMetaTSDB).setFilter(eventFilter);
        }
    }
}

Source File: AbstractCanalInstance.java From canal with Apache License 2.0

5 votes

@Override
public boolean subscribeChange(ClientIdentity identity) {
    if (StringUtils.isNotEmpty(identity.getFilter())) {
        logger.info("subscribe filter change to " + identity.getFilter());
        AviaterRegexFilter aviaterFilter = new AviaterRegexFilter(identity.getFilter());

        boolean isGroup = (eventParser instanceof GroupEventParser);
        if (isGroup) {
            // 处理group的模式
            List<CanalEventParser> eventParsers = ((GroupEventParser) eventParser).getEventParsers();
            for (CanalEventParser singleEventParser : eventParsers) {// 需要遍历启动
                if(singleEventParser instanceof AbstractEventParser) {
                    ((AbstractEventParser) singleEventParser).setEventFilter(aviaterFilter);
                }
            }
        } else {
            if(eventParser instanceof AbstractEventParser) {
                ((AbstractEventParser) eventParser).setEventFilter(aviaterFilter);
            }
        }

    }

    // filter的处理规则
    // a. parser处理数据过滤处理
    // b. sink处理数据的路由&分发,一份parse数据经过sink后可以分发为多份，每份的数据可以根据自己的过滤规则不同而有不同的数据
    // 后续内存版的一对多分发，可以考虑
    return true;
}

Source File: Binlog.java From jlogstash-input-plugin with Apache License 2.0

5 votes

@Override
public void prepare() {
    logger.info("binlog prepare started..");

    parseCategories();

    controller = new MysqlEventParser();
    controller.setConnectionCharset(Charset.forName("UTF-8"));
    controller.setSlaveId(slaveId);
    controller.setDetectingEnable(false);
    controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress(host, port), username, password));
    controller.setEnableTsdb(true);
    controller.setDestination("example");
    controller.setParallel(true);
    controller.setParallelBufferSize(256);
    controller.setParallelThreadSize(2);
    controller.setIsGTIDMode(false);
    controller.setEventSink(new BinlogEventSink(this));

    controller.setLogPositionManager(new BinlogPositionManager(this));

    EntryPosition startPosition = findStartPosition();
    if (startPosition != null) {
       controller.setMasterPosition(startPosition);
    }


    if (filter != null) {
        controller.setEventFilter(new AviaterRegexFilter(filter));
    }

    logger.info("binlog prepare ended..");
}

Source File: MutliAviaterFilterTest.java From canal-1.1.3 with Apache License 2.0

5 votes

private void doRegexTest() {
    AviaterRegexFilter filter3 = new AviaterRegexFilter("otter2.otter_stability1|otter1.otter_stability1|"
                                                        + RandomStringUtils.randomAlphabetic(200));
    boolean result = filter3.filter("otter1.otter_stability1");
    Assert.assertEquals(true, result);
    result = filter3.filter("otter2.otter_stability1");
    Assert.assertEquals(true, result);
}

Source File: AbstractMysqlEventParser.java From canal-1.1.3 with Apache License 2.0

5 votes

public void setEventBlackFilter(CanalEventFilter eventBlackFilter) {
    super.setEventBlackFilter(eventBlackFilter);

    // 触发一下filter变更
    if (eventBlackFilter != null && eventBlackFilter instanceof AviaterRegexFilter) {
        if (binlogParser instanceof LogEventConvert) {
            ((LogEventConvert) binlogParser).setNameBlackFilter((AviaterRegexFilter) eventBlackFilter);
        }

        if (tableMetaTSDB != null && tableMetaTSDB instanceof DatabaseTableMeta) {
            ((DatabaseTableMeta) tableMetaTSDB).setBlackFilter(eventBlackFilter);
        }
    }
}

Source File: AbstractMysqlEventParser.java From canal-1.1.3 with Apache License 2.0

5 votes

public void setEventFilter(CanalEventFilter eventFilter) {
    super.setEventFilter(eventFilter);

    // 触发一下filter变更
    if (eventFilter != null && eventFilter instanceof AviaterRegexFilter) {
        if (binlogParser instanceof LogEventConvert) {
            ((LogEventConvert) binlogParser).setNameFilter((AviaterRegexFilter) eventFilter);
        }

        if (tableMetaTSDB != null && tableMetaTSDB instanceof DatabaseTableMeta) {
            ((DatabaseTableMeta) tableMetaTSDB).setFilter(eventFilter);
        }
    }
}

Source File: LogEventConvert.java From DBus with Apache License 2.0

4 votes

public void setNameBlackFilter(AviaterRegexFilter nameBlackFilter) {
    this.nameBlackFilter = nameBlackFilter;
}

Source File: LogEventConvert_old.java From DBus with Apache License 2.0

4 votes

public void setNameFilter(AviaterRegexFilter nameFilter) {
    this.nameFilter = nameFilter;
}

Source File: LogEventConvert_old.java From DBus with Apache License 2.0

4 votes

public void setNameBlackFilter(AviaterRegexFilter nameBlackFilter) {
    this.nameBlackFilter = nameBlackFilter;
}

Source File: LogEventConvert.java From DBus with Apache License 2.0

4 votes

public void setNameFilter(AviaterRegexFilter nameFilter) {
    this.nameFilter = nameFilter;
}

Source File: LogEventConvert.java From DBus with Apache License 2.0

4 votes

public void setNameBlackFilter(AviaterRegexFilter nameBlackFilter) {
    this.nameBlackFilter = nameBlackFilter;
}

Source File: LogEventConvert.java From DBus with Apache License 2.0

4 votes

public void setNameFilter(AviaterRegexFilter nameFilter) {
    this.nameFilter = nameFilter;
}

Source File: LogEventConvert.java From canal with Apache License 2.0

4 votes

public void setNameFilter(AviaterRegexFilter nameFilter) {
    this.nameFilter = nameFilter;
    logger.warn("--> init table filter : " + nameFilter.toString());
}

Source File: LogEventConvert.java From canal with Apache License 2.0

4 votes

public void setNameBlackFilter(AviaterRegexFilter nameBlackFilter) {
    this.nameBlackFilter = nameBlackFilter;
    logger.warn("--> init table black filter : " + nameBlackFilter.toString());
}

Source File: AviaterFilterTest.java From canal-1.1.3 with Apache License 2.0

4 votes

@Test
public void test_regex() {
    AviaterRegexFilter filter = new AviaterRegexFilter("s1\\..*,s2\\..*");
    boolean result = filter.filter("s1.t1");
    Assert.assertEquals(true, result);

    result = filter.filter("s1.t2");
    Assert.assertEquals(true, result);

    result = filter.filter("");
    Assert.assertEquals(true, result);

    result = filter.filter("s12.t1");
    Assert.assertEquals(false, result);

    result = filter.filter("s2.t2");
    Assert.assertEquals(true, result);

    result = filter.filter("s3.t2");
    Assert.assertEquals(false, result);

    result = filter.filter("S1.S2");
    Assert.assertEquals(true, result);

    result = filter.filter("S2.S1");
    Assert.assertEquals(true, result);

    AviaterRegexFilter filter2 = new AviaterRegexFilter("s1\\..*,s2.t1");

    result = filter2.filter("s1.t1");
    Assert.assertEquals(true, result);

    result = filter2.filter("s1.t2");
    Assert.assertEquals(true, result);

    result = filter2.filter("s2.t1");
    Assert.assertEquals(true, result);

    AviaterRegexFilter filter3 = new AviaterRegexFilter("foooo,f.*t");

    result = filter3.filter("fooooot");
    Assert.assertEquals(true, result);

    AviaterRegexFilter filter4 = new AviaterRegexFilter("otter2.otter_stability1|otter1.otter_stability1|retl.retl_mark|retl.retl_buffer|retl.xdual");
    result = filter4.filter("otter1.otter_stability1");
    Assert.assertEquals(true, result);
}

Source File: AviaterFilterTest.java From canal with Apache License 2.0

4 votes

@Test
public void test_regex() {
    AviaterRegexFilter filter = new AviaterRegexFilter("s1\\..*,s2\\..*");
    boolean result = filter.filter("s1.t1");
    Assert.assertEquals(true, result);

    result = filter.filter("s1.t2");
    Assert.assertEquals(true, result);

    result = filter.filter("");
    Assert.assertEquals(true, result);

    result = filter.filter("s12.t1");
    Assert.assertEquals(false, result);

    result = filter.filter("s2.t2");
    Assert.assertEquals(true, result);

    result = filter.filter("s3.t2");
    Assert.assertEquals(false, result);

    result = filter.filter("S1.S2");
    Assert.assertEquals(true, result);

    result = filter.filter("S2.S1");
    Assert.assertEquals(true, result);

    AviaterRegexFilter filter2 = new AviaterRegexFilter("s1\\..*,s2.t1");

    result = filter2.filter("s1.t1");
    Assert.assertEquals(true, result);

    result = filter2.filter("s1.t2");
    Assert.assertEquals(true, result);

    result = filter2.filter("s2.t1");
    Assert.assertEquals(true, result);

    AviaterRegexFilter filter3 = new AviaterRegexFilter("foooo,f.*t");

    result = filter3.filter("fooooot");
    Assert.assertEquals(true, result);

    AviaterRegexFilter filter4 = new AviaterRegexFilter("otter2.otter_stability1|otter1.otter_stability1|retl.retl_mark|retl.retl_buffer|retl.xdual");
    result = filter4.filter("otter1.otter_stability1");
    Assert.assertEquals(true, result);
}

Source File: LogEventConvert.java From canal-1.1.3 with Apache License 2.0

4 votes

public void setNameBlackFilter(AviaterRegexFilter nameBlackFilter) {
    this.nameBlackFilter = nameBlackFilter;
    logger.warn("--> init table black filter : " + nameBlackFilter.toString());
}

Source File: LogEventConvert.java From canal-1.1.3 with Apache License 2.0

4 votes

public void setNameFilter(AviaterRegexFilter nameFilter) {
    this.nameFilter = nameFilter;
    logger.warn("--> init table filter : " + nameFilter.toString());
}

com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter Java Examples