com.alibaba.otter.canal.client.CanalConnector Java Examples

The following examples show how to use com.alibaba.otter.canal.client.CanalConnector. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: MutiCanalFactory.java    From database-transform-tool with Apache License 2.0 6 votes vote down vote up 
public List<MonitorInfo> service(){
	List<MonitorInfo> data = new ArrayList<MonitorInfo>();
	try {
		if(cache==null||cache.isEmpty()){
			init(destinations, servers, username, password, filter_regex, isZookeeper, batch_size);
		}
		if(!cache.isEmpty()){
			for (CanalConnector connector : cache.values()) {
				List<MonitorInfo> list = execute(connector);
				if(list!=null&&list.size()>0){
					data.addAll(list);
				}
			}
		}
	} catch (Exception e) {
		logger.error("--Muti Canal监控失败!",e);
	}
	return data;
}
Example #2
Source File: ClusterCanalClientTest.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
public static void main(String args[]) {
    String destination = "example";

    // 基于固定canal server的地址，建立链接，其中一台server发生crash，可以支持failover
    // CanalConnector connector = CanalConnectors.newClusterConnector(
    // Arrays.asList(new InetSocketAddress(
    // AddressUtils.getHostIp(),
    // 11111)),
    // "stability_test", "", "");

    // 基于zookeeper动态获取canal server的地址，建立链接，其中一台server发生crash，可以支持failover
    CanalConnector connector = CanalConnectors.newClusterConnector("127.0.0.1:2181", destination, "", "");

    final ClusterCanalClientTest clientTest = new ClusterCanalClientTest(destination);
    clientTest.setConnector(connector);
    clientTest.start();

    Runtime.getRuntime().addShutdownHook(new Thread() {

        public void run() {
            try {
                logger.info("## stop the canal client");
                clientTest.stop();
            } catch (Throwable e) {
                logger.warn("##something goes wrong when stopping canal:", e);
            } finally {
                logger.info("## canal client is down.");
            }
        }

    });
}
Example #3
Source File: SimpleCanalClientTest.java    From canal with Apache License 2.0 5 votes vote down vote up 
public static void main(String args[]) {
    // 根据ip，直接创建链接，无HA的功能
    String destination = "example";
    String ip = AddressUtils.getHostIp();
    CanalConnector connector = CanalConnectors.newSingleConnector(new InetSocketAddress(ip, 11111),
            destination,
            "canal",
            "canal");

    final SimpleCanalClientTest clientTest = new SimpleCanalClientTest(destination);
    clientTest.setConnector(connector);
    clientTest.start();
    Runtime.getRuntime().addShutdownHook(new Thread() {

        public void run() {
            try {
                logger.info("## stop the canal client");
                clientTest.stop();
            } catch (Throwable e) {
                logger.warn("##something goes wrong when stopping canal:", e);
            } finally {
                logger.info("## canal client is down.");
            }
        }

    });
}
Example #4
Source File: ClusterCanalClientTest.java    From canal with Apache License 2.0 5 votes vote down vote up 
public static void main(String args[]) {
    String destination = "example";

    // 基于固定canal server的地址，建立链接，其中一台server发生crash，可以支持failover
    // CanalConnector connector = CanalConnectors.newClusterConnector(
    // Arrays.asList(new InetSocketAddress(
    // AddressUtils.getHostIp(),
    // 11111)),
    // "stability_test", "", "");

    // 基于zookeeper动态获取canal server的地址，建立链接，其中一台server发生crash，可以支持failover
    CanalConnector connector = CanalConnectors.newClusterConnector("127.0.0.1:2181", destination, "canal", "canal");

    final ClusterCanalClientTest clientTest = new ClusterCanalClientTest(destination);
    clientTest.setConnector(connector);
    clientTest.start();

    Runtime.getRuntime().addShutdownHook(new Thread() {

        public void run() {
            try {
                logger.info("## stop the canal client");
                clientTest.stop();
            } catch (Throwable e) {
                logger.warn("##something goes wrong when stopping canal:", e);
            } finally {
                logger.info("## canal client is down.");
            }
        }

    });
}
Example #5
Source File: CaseController.java    From skywalking with Apache License 2.0 5 votes vote down vote up 
private void wrapCreateConnector(String destination, Consumer<CanalConnector> consumer) {
    CanalConnector connector = CanalConnectors.newSingleConnector(new InetSocketAddress(address, port), destination, "", "");
    connector.connect();
    try {
        consumer.accept(connector);
    } finally {
        connector.disconnect();
    }
}
Example #6
Source File: CanalClient.java    From canal_mysql_elasticsearch_sync with Apache License 2.0 5 votes vote down vote up 
@Bean
public CanalConnector getCanalConnector() {
    canalConnector = CanalConnectors.newClusterConnector(Lists.newArrayList(new InetSocketAddress(canalHost, Integer.valueOf(canalPort))), canalDestination, canalUsername, canalPassword);
    canalConnector.connect();
    // 指定filter，格式 {database}.{table}，这里不做过滤，过滤操作留给用户
    canalConnector.subscribe();
    // 回滚寻找上次中断的位置
    canalConnector.rollback();
    logger.info("canal客户端启动成功");
    return canalConnector;
}
Example #7
Source File: LockTest.java    From canal-elasticsearch with Apache License 2.0 5 votes vote down vote up 
public static void main(String[] args) throws InterruptedException {

        CanalConnector connector = CanalConnectors.newSingleConnector(new InetSocketAddress("127.0.0.1", 11111),
                "totoro",
                "",
                "");

        connector.connect();
        connector.subscribe();
        int emptyTimes = 0;

        while (running) {
            Message message = connector.getWithoutAck(5 * 1024);

            if (message == null || message.getId() == -1L) {
                applyWait(emptyTimes++);
            } else {
                //logger.info(message.toString());
                long messageId = message.getId();
                System.out.println("消息id：" + messageId);
                Thread.sleep(1000);

                connector.rollback();
            }


        }

    }
Example #8
Source File: CanalInitHandler.java    From canal-mongo with Apache License 2.0 5 votes vote down vote up 
public void initCanalStart() {
    List<String> destinations = canalProperties.getDestination();
    if (destinations != null && destinations.size() > 0) {
        for (String destination : destinations) {
            logger.info("## start the canal client : {}", destination);
            // 基于zookeeper动态获取canal server的地址，建立链接，其中一台server发生crash，可以支持failover
            CanalConnector connector = CanalConnectors.newClusterConnector(canalProperties.getZkServers(), destination, "", "");
            CanalClient client = new CanalClient(destination, connector);
            client.start();
            canalClientList.add(client);
        }
    }
}
Example #9
Source File: AbstractCanalClient.java    From spring-boot-starter-canal with MIT License 5 votes vote down vote up 
private CanalConnector processInstanceEntry(Map.Entry<String, CanalConfig.Instance> instanceEntry) {
    CanalConfig.Instance instance = instanceEntry.getValue();
    CanalConnector connector;
    if (instance.isClusterEnabled()) {
        List<SocketAddress> addresses = new ArrayList<>();
        for (String s : instance.getZookeeperAddress()) {
            String[] entry = s.split(":");
            if (entry.length != 2)
                throw new CanalClientException("error parsing zookeeper address:" + s);
            addresses.add(new InetSocketAddress(entry[0], Integer.parseInt(entry[1])));
        }
        connector = CanalConnectors.newClusterConnector(addresses, instanceEntry.getKey(),
                instance.getUserName(),
                instance.getPassword());
    } else {
        connector = CanalConnectors.newSingleConnector(new InetSocketAddress(instance.getHost(), instance.getPort()),
                instanceEntry.getKey(),
                instance.getUserName(),
                instance.getPassword());
    }
    connector.connect();
    if (!StringUtils.isEmpty(instance.getFilter())) {
        connector.subscribe(instance.getFilter());
    } else {
        connector.subscribe();
    }

    connector.rollback();
    return connector;
}
Example #10
Source File: SimpleCanalClientTest.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
public static void main(String args[]) {
        // 根据ip，直接创建链接，无HA的功能
        String destination = "example";
        String ip = AddressUtils.getHostIp();
//        创建单机canal connector
        CanalConnector connector = CanalConnectors.newSingleConnector(new InetSocketAddress(ip, 11111),
            destination,
            "",
            "");

        final SimpleCanalClientTest clientTest = new SimpleCanalClientTest(destination);
        clientTest.setConnector(connector);
        clientTest.start();
        Runtime.getRuntime().addShutdownHook(new Thread() {

            public void run() {
                try {
                    logger.info("## stop the canal client");
                    clientTest.stop();
                } catch (Throwable e) {
                    logger.warn("##something goes wrong when stopping canal:", e);
                } finally {
                    logger.info("## canal client is down.");
                }
            }

        });
    }
Example #11
Source File: MutiCanalFactory.java    From database-transform-tool with Apache License 2.0 5 votes vote down vote up 
/**
 * 关闭服务
 */
public static void close(){
	if(!cache.isEmpty()){
		for (CanalConnector connector : cache.values()) {
			connector.disconnect();
		}
	}
}
Example #12
Source File: AbstractMessageTransponder.java    From spring-boot-starter-canal with MIT License 5 votes vote down vote up 
public AbstractMessageTransponder(CanalConnector connector,
                                  Map.Entry<String, CanalConfig.Instance> config,
                                  List<CanalEventListener> listeners,
                                  List<ListenerPoint> annoListeners) {
    Objects.requireNonNull(connector, "connector can not be null!");
    Objects.requireNonNull(config, "config can not be null!");
    this.connector = connector;
    this.destination = config.getKey();
    this.config = config.getValue();
    if (listeners != null)
        this.listeners.addAll(listeners);
    if (annoListeners != null)
        this.annoListeners.addAll(annoListeners);
}
Example #13
Source File: CanalClient.java    From flume-canal-source with Apache License 2.0 4 votes vote down vote up 
private CanalConnector getConnector(SocketAddress address, String destination, String username,
                                    String password) {
    return CanalConnectors.newSingleConnector(address, destination, username, password);
}
Example #14
Source File: BaseCanalClientTest.java    From canal with Apache License 2.0 4 votes vote down vote up 
public void setConnector(CanalConnector connector) {
    this.connector = connector;
}
Example #15
Source File: AbstractCanalClientTest.java    From canal-1.1.3 with Apache License 2.0 4 votes vote down vote up 
public AbstractCanalClientTest(String destination, CanalConnector connector){
    this.destination = destination;
    this.connector = connector;
}
Example #16
Source File: AbstractCanalClientTest.java    From canal with Apache License 2.0 4 votes vote down vote up 
public AbstractCanalClientTest(String destination, CanalConnector connector){
    this.destination = destination;
    this.connector = connector;
}
Example #17
Source File: BaseCanalClientTest.java    From canal-1.1.3 with Apache License 2.0 4 votes vote down vote up 
public void setConnector(CanalConnector connector) {
    this.connector = connector;
}
Example #18
Source File: CanalAbstractSource.java    From pulsar with Apache License 2.0 4 votes vote down vote up 
public CanalRecord(CanalConnector connector) {
    this.connector = connector;
}
Example #19
Source File: Schemas.java    From search-commons with Apache License 2.0 4 votes vote down vote up 
@Override
public CanalConnector create(String instanceName) {
    return CanalConnectors.newClusterConnector(zkServers, instanceName, null, null);
}
Example #20
Source File: Schemas.java    From search-commons with Apache License 2.0 4 votes vote down vote up 
@Override
public CanalConnector create(String instanceName) {
    return CanalConnectors.newClusterConnector(addresses, instanceName, null, null);
}
Example #21
Source File: Schemas.java    From search-commons with Apache License 2.0 4 votes vote down vote up 
@Override
public CanalConnector create(String instanceName) {
    return CanalConnectors.newSingleConnector(address, instanceName, null, null);
}
Example #22
Source File: MutiCanalFactory.java    From database-transform-tool with Apache License 2.0 4 votes vote down vote up 
/**
 * @description Canal服务配置
 * @author yi.zhang
 * @time 2017年4月19日 上午10:38:42
 * @throws Exception
 */
public void init(String destinations,String servers,String username,String password,String filter_regex,boolean isZookeeper,Integer batch_size){
	try {
		if(filter_regex!=null){
			CANAL_FILTER_REGEX = filter_regex;
		}
		if(batch_size!=null){
			BATCH_SIZE = batch_size;
		}
		if(servers==null||"".equals(servers)){
			return;
		}
		if(destinations!=null&&!"".equals(destinations)){
			for(String destination:destinations.split(",")){
				if(destination==null||"".equals(destination)){
					continue;
				}
				CanalConnector connector = null;
				if(isZookeeper){
					connector = CanalConnectors.newClusterConnector(servers, destination, username, password);
				}else{
					List<SocketAddress> addresses = new ArrayList<SocketAddress>();
					for(String address : servers.split(",")){
						String[] ips = address.split(":");
						String ip = ips[0];
						int port=11111;
						if(ips.length>1){
							port = Integer.valueOf(ips[1]);
						}
						addresses.add(new InetSocketAddress(ip, port));
					}
					if(addresses!=null&&addresses.size()==1){
						connector = CanalConnectors.newSingleConnector(addresses.get(0), destination, username, password);
					}else{
						connector = CanalConnectors.newClusterConnector(addresses, destination, username, password);
					}
				}
				connector.connect();
		        connector.subscribe(CANAL_FILTER_REGEX);
		        connector.rollback();
		        cache.put(destination, connector);
			}
		}
	} catch (Exception e) {
		logger.error("-----Muti Canal Config init Error-----", e);
	}
}
Example #23
Source File: CanalClient.java    From flume-canal-source with Apache License 2.0 4 votes vote down vote up 
private CanalConnector getConnector(List<? extends SocketAddress> addresses, String destination,
                                    String username, String password) {
    return CanalConnectors.newClusterConnector(addresses, destination, username, password);
}
Example #24
Source File: CanalClient.java    From flume-canal-source with Apache License 2.0 4 votes vote down vote up 
private CanalConnector getConnector(String zkServers, String destination, String username, String password) {
    return CanalConnectors.newClusterConnector(zkServers, destination, username, password);
}
Example #25
Source File: SimpleCanalClientExample.java    From DBus with Apache License 2.0 4 votes vote down vote up 
public static void main(String args[]) {
    //args = new String[]{"vdbus-4", "10000", "mysql_db2"};
    args = new String[]{"vdbus-4", "10000", "mysql_db2"};
    if (args.length != 3) {
        System.out.println("args: dbus-n1 11111 testdb");
        return;
    }
    String ip = args[0];
    int port = Integer.parseInt(args[1]);
    String dbname = args[2];

    // 创建链接
    CanalConnector connector = null;
    int batchSize = 1000;
    int emptyCount = 0;
    try {
        connector = CanalConnectors.newSingleConnector(new InetSocketAddress(ip, port), dbname, "", "");
        //connector = CanalConnectors.newClusterConnector("vdbus-7:2181/DBus/Canal/mysql_db1", dbname, "", "");
        connector.connect();
        connector.subscribe("");
        connector.rollback();
        int totalEmtryCount = 120;
        while (emptyCount < totalEmtryCount) {
            Message message = connector.getWithoutAck(batchSize); // 获取指定数量的数据
            long batchId = message.getId();
            int size = message.getEntries().size();

            if (batchId == -1 || size == 0) {
                emptyCount++;

                System.out.print(".");
                try {
                    Thread.sleep(1000);
                } catch (InterruptedException e) {
                }
            } else {
                emptyCount = 0;
                // System.out.printf("message[batchId=%s,size=%s] \n", batchId, size);


                System.out.println("");
                printEntry(message.getEntries(), batchId);
            }

            connector.ack(batchId); // 提交确认
            // connector.rollback(batchId); // 处理失败, 回滚数据
        }

        System.out.println("empty too many times, exit");
    } finally {
        if (connector != null) {
            connector.disconnect();
        }
    }
}
Example #26
Source File: CanalClient.java    From canal-mongo with Apache License 2.0 4 votes vote down vote up 
public CanalClient(String destination, CanalConnector connector) {
    this.destination = destination;
    this.connector = connector;
}
Example #27
Source File: SimpleCanalClient.java    From spring-boot-starter-canal with MIT License 4 votes vote down vote up 
@Override
protected void process(CanalConnector connector, Map.Entry<String, CanalConfig.Instance> config) {
    executor.submit(factory.newTransponder(connector, config, listeners, annoListeners));
}
Example #28
Source File: AbstractBasicMessageTransponder.java    From spring-boot-starter-canal with MIT License 4 votes vote down vote up 
public AbstractBasicMessageTransponder(CanalConnector connector, Map.Entry<String, CanalConfig.Instance> config, List<CanalEventListener> listeners, List<ListenerPoint> annoListeners) {
    super(connector, config, listeners, annoListeners);
}
Example #29
Source File: DefaultTransponderFactory.java    From spring-boot-starter-canal with MIT License 4 votes vote down vote up 
@Override
public MessageTransponder newTransponder(CanalConnector connector, Map.Entry<String, CanalConfig.Instance> config, List<CanalEventListener> listeners,
                                         List<ListenerPoint> annoListeners) {
    return new DefaultMessageTransponder(connector, config, listeners, annoListeners);
}
Example #30
Source File: DefaultMessageTransponder.java    From spring-boot-starter-canal with MIT License 4 votes vote down vote up 
public DefaultMessageTransponder(CanalConnector connector,
                      Map.Entry<String, CanalConfig.Instance> config,
                      List<CanalEventListener> listeners,
                      List<ListenerPoint> annoListeners) {
    super(connector, config, listeners, annoListeners);
}