com.alibaba.otter.canal.client.CanalConnectors Java Examples
The following examples show how to use
com.alibaba.otter.canal.client.CanalConnectors.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: CanalAbstractSource.java From pulsar with Apache License 2.0 | 6 votes |
|
@Override
public void open(Map<String, Object> config, SourceContext sourceContext) throws Exception {
canalSourceConfig = CanalSourceConfig.load(config);
if (canalSourceConfig.getCluster()) {
connector = CanalConnectors.newClusterConnector(canalSourceConfig.getZkServers(),
canalSourceConfig.getDestination(), canalSourceConfig.getUsername(), canalSourceConfig.getPassword());
log.info("Start canal connect in cluster mode, canal cluster info {}", canalSourceConfig.getZkServers());
} else {
connector = CanalConnectors.newSingleConnector(
new InetSocketAddress(canalSourceConfig.getSingleHostname(), canalSourceConfig.getSinglePort()),
canalSourceConfig.getDestination(), canalSourceConfig.getUsername(), canalSourceConfig.getPassword());
log.info("Start canal connect in standalone mode, canal server info {}:{}",
canalSourceConfig.getSingleHostname(), canalSourceConfig.getSinglePort());
}
log.info("canal source destination {}", canalSourceConfig.getDestination());
this.start();
}
Example #2
| Source File: SimpleCanalClientTest.java From canal-1.1.3 with Apache License 2.0 | 5 votes |
|
public static void main(String args[]) {
// 根据ip，直接创建链接，无HA的功能
String destination = "example";
String ip = AddressUtils.getHostIp();
// 创建单机canal connector
CanalConnector connector = CanalConnectors.newSingleConnector(new InetSocketAddress(ip, 11111),
destination,
"",
"");
final SimpleCanalClientTest clientTest = new SimpleCanalClientTest(destination);
clientTest.setConnector(connector);
clientTest.start();
Runtime.getRuntime().addShutdownHook(new Thread() {
public void run() {
try {
logger.info("## stop the canal client");
clientTest.stop();
} catch (Throwable e) {
logger.warn("##something goes wrong when stopping canal:", e);
} finally {
logger.info("## canal client is down.");
}
}
});
}
Example #3
| Source File: SimpleCanalClientTest.java From canal with Apache License 2.0 | 5 votes |
|
public static void main(String args[]) {
// 根据ip，直接创建链接，无HA的功能
String destination = "example";
String ip = AddressUtils.getHostIp();
CanalConnector connector = CanalConnectors.newSingleConnector(new InetSocketAddress(ip, 11111),
destination,
"canal",
"canal");
final SimpleCanalClientTest clientTest = new SimpleCanalClientTest(destination);
clientTest.setConnector(connector);
clientTest.start();
Runtime.getRuntime().addShutdownHook(new Thread() {
public void run() {
try {
logger.info("## stop the canal client");
clientTest.stop();
} catch (Throwable e) {
logger.warn("##something goes wrong when stopping canal:", e);
} finally {
logger.info("## canal client is down.");
}
}
});
}
Example #4
| Source File: ClusterCanalClientTest.java From canal with Apache License 2.0 | 5 votes |
|
public static void main(String args[]) {
String destination = "example";
// 基于固定canal server的地址，建立链接，其中一台server发生crash，可以支持failover
// CanalConnector connector = CanalConnectors.newClusterConnector(
// Arrays.asList(new InetSocketAddress(
// AddressUtils.getHostIp(),
// 11111)),
// "stability_test", "", "");
// 基于zookeeper动态获取canal server的地址，建立链接，其中一台server发生crash，可以支持failover
CanalConnector connector = CanalConnectors.newClusterConnector("127.0.0.1:2181", destination, "canal", "canal");
final ClusterCanalClientTest clientTest = new ClusterCanalClientTest(destination);
clientTest.setConnector(connector);
clientTest.start();
Runtime.getRuntime().addShutdownHook(new Thread() {
public void run() {
try {
logger.info("## stop the canal client");
clientTest.stop();
} catch (Throwable e) {
logger.warn("##something goes wrong when stopping canal:", e);
} finally {
logger.info("## canal client is down.");
}
}
});
}
Example #5
| Source File: CaseController.java From skywalking with Apache License 2.0 | 5 votes |
|
private void wrapCreateConnector(String destination, Consumer<CanalConnector> consumer) {
CanalConnector connector = CanalConnectors.newSingleConnector(new InetSocketAddress(address, port), destination, "", "");
connector.connect();
try {
consumer.accept(connector);
} finally {
connector.disconnect();
}
}
Example #6
| Source File: CanalClient.java From canal_mysql_elasticsearch_sync with Apache License 2.0 | 5 votes |
|
@Bean
public CanalConnector getCanalConnector() {
canalConnector = CanalConnectors.newClusterConnector(Lists.newArrayList(new InetSocketAddress(canalHost, Integer.valueOf(canalPort))), canalDestination, canalUsername, canalPassword);
canalConnector.connect();
// 指定filter，格式 {database}.{table}，这里不做过滤，过滤操作留给用户
canalConnector.subscribe();
// 回滚寻找上次中断的位置
canalConnector.rollback();
logger.info("canal客户端启动成功");
return canalConnector;
}
Example #7
| Source File: CustomSimpleCanalClient.java From canal-client with Apache License 2.0 | 5 votes |
|
public void initConector() {
if (canalConfig == null) {
throw new IllegalArgumentException("CustomSimpleCanalClient canalConfig property is empty!");
}
if ((invokeMap == null || invokeMap.isEmpty())&& this.globalInvoke==null) {
throw new IllegalArgumentException("CustomSimpleCanalClient invokeMap property is empty!");
}
if (canalConfig instanceof SingleCanalConfig) {
String socketStr = ((SingleCanalConfig) canalConfig).getSocketAddress();
logger.info("canal will connection to {}.", socketStr);
connector = CanalConnectors.newSingleConnector(this.getSocketAddressByString(socketStr),
canalConfig.getDestination(), canalConfig.getUsername(), canalConfig.getPassword());
} else if (canalConfig instanceof SocketsClusterCanalConfig) {
List<SocketAddress> socketAddressList = new ArrayList<SocketAddress>();
for (String sok : ((SocketsClusterCanalConfig) canalConfig).getSocketAddress()) {
logger.info("canal will connection to {}.", sok);
socketAddressList.add(this.getSocketAddressByString(sok));
}
connector = CanalConnectors.newClusterConnector(socketAddressList,
canalConfig.getDestination(), canalConfig.getUsername(), canalConfig.getPassword());
} else if (canalConfig instanceof ZkClusterCanalConfig) {
String zkAddress = ((ZkClusterCanalConfig) canalConfig).getZkAddress();
logger.info("canal will connection to {}.", zkAddress);
connector = CanalConnectors.newClusterConnector(zkAddress,
canalConfig.getDestination(), canalConfig.getUsername(), canalConfig.getPassword());
}
connector.connect();
connector.subscribe(canalConfig.getSubscribeChannel());
connector.rollback();
}
Example #8
| Source File: LockTest.java From canal-elasticsearch with Apache License 2.0 | 5 votes |
|
public static void main(String[] args) throws InterruptedException {
CanalConnector connector = CanalConnectors.newSingleConnector(new InetSocketAddress("127.0.0.1", 11111),
"totoro",
"",
"");
connector.connect();
connector.subscribe();
int emptyTimes = 0;
while (running) {
Message message = connector.getWithoutAck(5 * 1024);
if (message == null || message.getId() == -1L) {
applyWait(emptyTimes++);
} else {
//logger.info(message.toString());
long messageId = message.getId();
System.out.println("消息id：" + messageId);
Thread.sleep(1000);
connector.rollback();
}
}
}
Example #9
| Source File: CanalEmbedSelector.java From canal-elasticsearch with Apache License 2.0 | 5 votes |
|
public CanalEmbedSelector(CanalConf conf) {
logger.info("TotoroSelector init start , conf :{}", conf.toString());
this.mode = conf.getMode();
this.destination = conf.getDestination();
this.filterPatten = conf.getFilterPatten();
String userName = conf.getUserName();
String passWord = conf.getPassWord();
if (Mode.SIGN.equals(mode)) {
String address = conf.getAddress();
String[] hostPort = address.split(":");
String ip = hostPort[0];
Integer port = Integer.valueOf(hostPort[1]);
SocketAddress socketAddress = new InetSocketAddress(ip, port);
connector = CanalConnectors.newSingleConnector(socketAddress,
destination,
userName,
passWord);
} else if (Mode.CLUSTER.equals(mode)) {
String zkAddress = conf.getZkAddress();
connector = CanalConnectors.newClusterConnector(zkAddress, destination, userName, passWord);
} else {
throw new TotoroException("Invalid mode");
}
logger.info("TotoroSelector init complete .......");
}
Example #10
| Source File: CanalInitHandler.java From canal-mongo with Apache License 2.0 | 5 votes |
|
public void initCanalStart() {
List<String> destinations = canalProperties.getDestination();
if (destinations != null && destinations.size() > 0) {
for (String destination : destinations) {
logger.info("## start the canal client : {}", destination);
// 基于zookeeper动态获取canal server的地址，建立链接，其中一台server发生crash，可以支持failover
CanalConnector connector = CanalConnectors.newClusterConnector(canalProperties.getZkServers(), destination, "", "");
CanalClient client = new CanalClient(destination, connector);
client.start();
canalClientList.add(client);
}
}
}
Example #11
| Source File: AbstractCanalClient.java From spring-boot-starter-canal with MIT License | 5 votes |
|
private CanalConnector processInstanceEntry(Map.Entry<String, CanalConfig.Instance> instanceEntry) {
CanalConfig.Instance instance = instanceEntry.getValue();
CanalConnector connector;
if (instance.isClusterEnabled()) {
List<SocketAddress> addresses = new ArrayList<>();
for (String s : instance.getZookeeperAddress()) {
String[] entry = s.split(":");
if (entry.length != 2)
throw new CanalClientException("error parsing zookeeper address:" + s);
addresses.add(new InetSocketAddress(entry[0], Integer.parseInt(entry[1])));
}
connector = CanalConnectors.newClusterConnector(addresses, instanceEntry.getKey(),
instance.getUserName(),
instance.getPassword());
} else {
connector = CanalConnectors.newSingleConnector(new InetSocketAddress(instance.getHost(), instance.getPort()),
instanceEntry.getKey(),
instance.getUserName(),
instance.getPassword());
}
connector.connect();
if (!StringUtils.isEmpty(instance.getFilter())) {
connector.subscribe(instance.getFilter());
} else {
connector.subscribe();
}
connector.rollback();
return connector;
}
Example #12
| Source File: CanalFactory.java From database-transform-tool with Apache License 2.0 | 5 votes |
|
/**
* @description Canal服务配置
* @author yi.zhang
* @time 2017年4月19日 上午10:38:42
* @throws Exception
*/
public void init(String destination,String servers,String username,String password,String filter_regex,boolean isZookeeper,Integer batch_size){
try {
if(filter_regex!=null){
CANAL_FILTER_REGEX = filter_regex;
}
if(batch_size!=null){
BATCH_SIZE = batch_size;
}
if(isZookeeper){
connector = CanalConnectors.newClusterConnector(servers, destination, username, password);
}else{
List<SocketAddress> addresses = new ArrayList<SocketAddress>();
for(String address : servers.split(",")){
String[] ips = address.split(":");
String ip = ips[0];
int port=11111;
if(ips.length>1){
port = Integer.valueOf(ips[1]);
}
addresses.add(new InetSocketAddress(ip, port));
}
connector = CanalConnectors.newClusterConnector(addresses, destination, username, password);
}
connector.connect();
connector.subscribe(CANAL_FILTER_REGEX);
connector.rollback();
} catch (Exception e) {
logger.error("-----Canal Config init Error-----", e);
}
}
Example #13
| Source File: CanalAdapterWorker.java From canal-1.1.3 with Apache License 2.0 | 5 votes |
|
/**
* HA模式下client适配器worker的构造方法
*
* @param canalDestination canal实例名
* @param zookeeperHosts zookeeper地址
* @param canalOuterAdapters 外部适配器组
*/
public CanalAdapterWorker(CanalClientConfig canalClientConfig, String canalDestination, String zookeeperHosts,
List<List<OuterAdapter>> canalOuterAdapters){
super(canalOuterAdapters);
this.canalDestination = canalDestination;
this.canalClientConfig = canalClientConfig;
connector = CanalConnectors.newClusterConnector(zookeeperHosts, canalDestination, "", "");
((ClusterCanalConnector) connector).setSoTimeout(SO_TIMEOUT);
}
Example #14
| Source File: CanalAdapterWorker.java From canal-1.1.3 with Apache License 2.0 | 5 votes |
|
/**
* 单台client适配器worker的构造方法
*
* @param canalDestination canal实例名
* @param address canal-server地址
* @param canalOuterAdapters 外部适配器组
*/
public CanalAdapterWorker(CanalClientConfig canalClientConfig, String canalDestination, SocketAddress address,
List<List<OuterAdapter>> canalOuterAdapters){
super(canalOuterAdapters);
this.canalClientConfig = canalClientConfig;
this.canalDestination = canalDestination;
connector = CanalConnectors.newSingleConnector(address, canalDestination, "", "");
}
Example #15
| Source File: ClusterCanalClientTest.java From canal-1.1.3 with Apache License 2.0 | 5 votes |
|
public static void main(String args[]) {
String destination = "example";
// 基于固定canal server的地址，建立链接，其中一台server发生crash，可以支持failover
// CanalConnector connector = CanalConnectors.newClusterConnector(
// Arrays.asList(new InetSocketAddress(
// AddressUtils.getHostIp(),
// 11111)),
// "stability_test", "", "");
// 基于zookeeper动态获取canal server的地址，建立链接，其中一台server发生crash，可以支持failover
CanalConnector connector = CanalConnectors.newClusterConnector("127.0.0.1:2181", destination, "", "");
final ClusterCanalClientTest clientTest = new ClusterCanalClientTest(destination);
clientTest.setConnector(connector);
clientTest.start();
Runtime.getRuntime().addShutdownHook(new Thread() {
public void run() {
try {
logger.info("## stop the canal client");
clientTest.stop();
} catch (Throwable e) {
logger.warn("##something goes wrong when stopping canal:", e);
} finally {
logger.info("## canal client is down.");
}
}
});
}
Example #16
| Source File: SimpleCanalClientExample.java From DBus with Apache License 2.0 | 4 votes |
|
public static void main(String args[]) {
//args = new String[]{"vdbus-4", "10000", "mysql_db2"};
args = new String[]{"vdbus-4", "10000", "mysql_db2"};
if (args.length != 3) {
System.out.println("args: dbus-n1 11111 testdb");
return;
}
String ip = args[0];
int port = Integer.parseInt(args[1]);
String dbname = args[2];
// 创建链接
CanalConnector connector = null;
int batchSize = 1000;
int emptyCount = 0;
try {
connector = CanalConnectors.newSingleConnector(new InetSocketAddress(ip, port), dbname, "", "");
//connector = CanalConnectors.newClusterConnector("vdbus-7:2181/DBus/Canal/mysql_db1", dbname, "", "");
connector.connect();
connector.subscribe("");
connector.rollback();
int totalEmtryCount = 120;
while (emptyCount < totalEmtryCount) {
Message message = connector.getWithoutAck(batchSize); // 获取指定数量的数据
long batchId = message.getId();
int size = message.getEntries().size();
if (batchId == -1 || size == 0) {
emptyCount++;
System.out.print(".");
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
} else {
emptyCount = 0;
// System.out.printf("message[batchId=%s,size=%s] \n", batchId, size);
System.out.println("");
printEntry(message.getEntries(), batchId);
}
connector.ack(batchId); // 提交确认
// connector.rollback(batchId); // 处理失败, 回滚数据
}
System.out.println("empty too many times, exit");
} finally {
if (connector != null) {
connector.disconnect();
}
}
}
Example #17
| Source File: CanalClient.java From flume-canal-source with Apache License 2.0 | 4 votes |
|
private CanalConnector getConnector(String zkServers, String destination, String username, String password) {
return CanalConnectors.newClusterConnector(zkServers, destination, username, password);
}
Example #18
| Source File: CanalClient.java From flume-canal-source with Apache License 2.0 | 4 votes |
|
private CanalConnector getConnector(List<? extends SocketAddress> addresses, String destination,
String username, String password) {
return CanalConnectors.newClusterConnector(addresses, destination, username, password);
}
Example #19
| Source File: CanalClient.java From flume-canal-source with Apache License 2.0 | 4 votes |
|
private CanalConnector getConnector(SocketAddress address, String destination, String username,
String password) {
return CanalConnectors.newSingleConnector(address, destination, username, password);
}
Example #20
| Source File: CanalClientSpout.java From DBus with Apache License 2.0 | 4 votes |
|
/********************************************************************************/
private void reloadConfig(String reloadJson) {
logger.info("spout: canal client reload starting......");
ZKHelper zkHelper = null;
try {
ContainerMng.clearAllContainer();//清除所有container中存在的信息
zkHelper = new ZKHelper(zkServers, extractorRoot, extractorName);
zkHelper.loadJdbcConfig();
zkHelper.loadExtractorConifg();
zkHelper.loadOutputTopic();
zkHelper.loadFilter();
zkHelper.loadKafkaProducerConfig();
zkHelper.loadKafkaConsumerConfig();
logger.info("spout: 1 reload zk OK!");
Integer kafkaBatchSize = ExtractorConfigContainer.getInstances().getExtractorConfig().getKafkaSendBatchSize();
if (kafkaBatchSize != null)
kafkaSendBatchSize = kafkaBatchSize;
batchSize = ExtractorConfigContainer.getInstances().getExtractorConfig().getCanalBatchSize();
flowSize = ExtractorConfigContainer.getInstances().getExtractorConfig().getCanalFlowSize();
/****************************初始化控制reload的kafka consumer************************/
if (consumer != null) {
consumer.close();
consumer = null;
}
//不需要for循环 20180307 review
for (OutputTopicVo vo : ExtractorConfigContainer.getInstances().getOutputTopic()) {
extractorControlTopic = vo.getControlTopic();
break;
}
consumer = DbusHelper.createConsumer(ExtractorConfigContainer.getInstances().getKafkaConsumerConfig(),
extractorControlTopic);
logger.info("spout: 2 reload kafka consumer OK!");
/***********************************************************************************/
/***判断连接canal connect的zkconnectstr和destination是否改变，如果改变则断开重新连接，否则只进行后续订阅即可***/
String canalZkPath = ExtractorConfigContainer.getInstances().getExtractorConfig().getCanalZkPath();
String newZkConnectStr = zkServers + canalZkPath;
String newDestination = ExtractorConfigContainer.getInstances().getExtractorConfig().getCanalInstanceName();
if (connector == null || (connector != null && connector.checkValid() == false)) {
//连接不可用
if (connector != null && connector.checkValid() == false) {
//有连接，连接不可用
logger.error("spout: connect is not valid!");
connector.disconnect();
if (softStopProcess()) {
MsgStatusContainer.getInstance().clear();
}
}
assert (zkConnectStr == null && destination == null);
zkConnectStr = newZkConnectStr;
destination = newDestination;
connector = CanalConnectors.newClusterConnector(newZkConnectStr, newDestination, "", "");
connector.connect();
logger.info("spout: 3 canal reload OK!");
} else {
//连接是可以用的
if (!newZkConnectStr.equals(zkConnectStr) || !newDestination.equals(destination)) {
//配置发生变化
logger.info("spout: canal reconnected, zk config have been changed!");
connector.disconnect();
if (softStopProcess()) {
MsgStatusContainer.getInstance().clear();
}
connector = CanalConnectors.newClusterConnector(newZkConnectStr, newDestination, "", "");
connector.connect();
zkConnectStr = newZkConnectStr;
destination = newDestination;
logger.info("spout: 3 canal reload OK!");
} else {
logger.info("spout: 3 canal reload OK! zk config not changed!");
}
}
filter = ExtractorConfigContainer.getInstances().getFilter();
connector.subscribe(filter);
logger.info("spout: canal subscribe filter is: {}", filter);
//写reload状态到zookeeper
zkHelper.saveReloadStatus(reloadJson, "extractor-canal-client-spout", true);
logger.info("spout: reload all_OK!");
} catch (Exception ex) {
logger.error("spout reloadConfig()", ex);
collector.reportError(ex);
//throw new RuntimeException(ex);
} finally {
if (zkHelper != null) {
zkHelper.close();
}
}
}
Example #21
| Source File: Schemas.java From search-commons with Apache License 2.0 | 4 votes |
|
@Override
public CanalConnector create(String instanceName) {
return CanalConnectors.newSingleConnector(address, instanceName, null, null);
}
Example #22
| Source File: Schemas.java From search-commons with Apache License 2.0 | 4 votes |
|
@Override
public CanalConnector create(String instanceName) {
return CanalConnectors.newClusterConnector(addresses, instanceName, null, null);
}
Example #23
| Source File: Schemas.java From search-commons with Apache License 2.0 | 4 votes |
|
@Override
public CanalConnector create(String instanceName) {
return CanalConnectors.newClusterConnector(zkServers, instanceName, null, null);
}
Example #24
| Source File: MutiCanalFactory.java From database-transform-tool with Apache License 2.0 | 4 votes |
|
/**
* @description Canal服务配置
* @author yi.zhang
* @time 2017年4月19日 上午10:38:42
* @throws Exception
*/
public void init(String destinations,String servers,String username,String password,String filter_regex,boolean isZookeeper,Integer batch_size){
try {
if(filter_regex!=null){
CANAL_FILTER_REGEX = filter_regex;
}
if(batch_size!=null){
BATCH_SIZE = batch_size;
}
if(servers==null||"".equals(servers)){
return;
}
if(destinations!=null&&!"".equals(destinations)){
for(String destination:destinations.split(",")){
if(destination==null||"".equals(destination)){
continue;
}
CanalConnector connector = null;
if(isZookeeper){
connector = CanalConnectors.newClusterConnector(servers, destination, username, password);
}else{
List<SocketAddress> addresses = new ArrayList<SocketAddress>();
for(String address : servers.split(",")){
String[] ips = address.split(":");
String ip = ips[0];
int port=11111;
if(ips.length>1){
port = Integer.valueOf(ips[1]);
}
addresses.add(new InetSocketAddress(ip, port));
}
if(addresses!=null&&addresses.size()==1){
connector = CanalConnectors.newSingleConnector(addresses.get(0), destination, username, password);
}else{
connector = CanalConnectors.newClusterConnector(addresses, destination, username, password);
}
}
connector.connect();
connector.subscribe(CANAL_FILTER_REGEX);
connector.rollback();
cache.put(destination, connector);
}
}
} catch (Exception e) {
logger.error("-----Muti Canal Config init Error-----", e);
}
}
