com.alibaba.otter.canal.common.CanalException Java Examples

public CanalInstance generate(String destination) {
    synchronized (CanalInstanceGenerator.class) {
        try {
            // 设置当前正在加载的通道，加载spring查找文件时会用到该变量
            System.setProperty("canal.instance.destination", destination);
            this.beanFactory = getBeanFactory(springXml);
            String beanName = destination;
            if (!beanFactory.containsBean(beanName)) {
                beanName = defaultName;
            }

            return (CanalInstance) beanFactory.getBean(beanName);
        } catch (Throwable e) {
            logger.error("generator instance failed.", e);
            throw new CanalException(e);
        } finally {
            System.setProperty("canal.instance.destination", "");
        }
    }
}
 

public CanalInstance generate(String destination) {
    synchronized (CanalInstanceGenerator.class) {
        try {
            PlainCanal canal = canalConfigClient.findInstance(destination, null);
            if (canal == null) {
                throw new CanalException("instance : " + destination + " config is not found");
            }
            Properties properties = canal.getProperties();
            // merge local
            properties.putAll(canalConfig);

            // 设置动态properties,替换掉本地properties
            com.alibaba.otter.canal.instance.spring.support.PropertyPlaceholderConfigurer.propertiesLocal.set(properties);
            // 设置当前正在加载的通道，加载spring查找文件时会用到该变量
            System.setProperty("canal.instance.destination", destination);
            this.beanFactory = getBeanFactory(springXml);
            String beanName = destination;
            if (!beanFactory.containsBean(beanName)) {
                beanName = defaultName;
            }

            return (CanalInstance) beanFactory.getBean(beanName);
        } catch (Throwable e) {
            logger.error("generator instance failed.", e);
            throw new CanalException(e);
        } finally {
            System.setProperty("canal.instance.destination", "");
        }
    }
}
 

private ResponseModel<CanalConfig> doQuery(String url) {
    Map<String, String> heads = new HashMap<>();
    heads.put("user", user);
    heads.put("passwd", passwd);
    String response = httpHelper.get(url, heads, REQUEST_TIMEOUT);
    ResponseModel<CanalConfig> resp = JSONObject.parseObject(response,
        new TypeReference<ResponseModel<CanalConfig>>() {
        });

    if (!HttpHelper.REST_STATE_OK.equals(resp.code)) {
        throw new CanalException("requestGet for canal config error: " + resp.message);
    }

    return resp;
}
 

private PlainCanal queryConfig(String url) {
    try {
        ResponseModel<CanalConfig> config = doQuery(url);
        return processData(config.data);
    } catch (Throwable e) {
        throw new CanalException("load manager config failed.", e);
    }
}
 

protected CanalLogPositionManager initLogPositionManager() {
    logger.info("init logPositionPersistManager begin...");
    IndexMode indexMode = parameters.getIndexMode();
    CanalLogPositionManager logPositionManager;
    if (indexMode.isMemory()) {
        logPositionManager = new MemoryLogPositionManager();
    } else if (indexMode.isZookeeper()) {
        logPositionManager = new ZooKeeperLogPositionManager(getZkclientx());
    } else if (indexMode.isMixed()) {
        MemoryLogPositionManager memoryLogPositionManager = new MemoryLogPositionManager();
        ZooKeeperLogPositionManager zooKeeperLogPositionManager = new ZooKeeperLogPositionManager(getZkclientx());
        logPositionManager = new PeriodMixedLogPositionManager(memoryLogPositionManager,
            zooKeeperLogPositionManager,
            1000L);
    } else if (indexMode.isMeta()) {
        logPositionManager = new MetaLogPositionManager(metaManager);
    } else if (indexMode.isMemoryMetaFailback()) {
        MemoryLogPositionManager primary = new MemoryLogPositionManager();
        MetaLogPositionManager secondary = new MetaLogPositionManager(metaManager);

        logPositionManager = new FailbackLogPositionManager(primary, secondary);
    } else {
        throw new CanalException("unsupport indexMode for " + indexMode);
    }

    logger.info("init logPositionManager end! \n\t load CanalLogPositionManager:{}", logPositionManager.getClass()
        .getName());

    return logPositionManager;
}
 

protected CanalHAController initHaController() {
    logger.info("init haController begin...");
    HAMode haMode = parameters.getHaMode();
    CanalHAController haController = null;
    if (haMode.isHeartBeat()) {
        haController = new HeartBeatHAController();
        ((HeartBeatHAController) haController).setDetectingRetryTimes(parameters.getDetectingRetryTimes());
        ((HeartBeatHAController) haController).setSwitchEnable(parameters.getHeartbeatHaEnable());
    } else {
        throw new CanalException("unsupport HAMode for " + haMode);
    }
    logger.info("init haController end! \n\t load CanalHAController:{}", haController.getClass().getName());

    return haController;
}
 

protected void initEventParser() {
    logger.info("init eventParser begin...");
    SourcingType type = parameters.getSourcingType();

    List<List<DataSourcing>> groupDbAddresses = parameters.getGroupDbAddresses();
    if (!CollectionUtils.isEmpty(groupDbAddresses)) {
        int size = groupDbAddresses.get(0).size();// 取第一个分组的数量，主备分组的数量必须一致
        List<CanalEventParser> eventParsers = new ArrayList<CanalEventParser>();
        for (int i = 0; i < size; i++) {
            List<InetSocketAddress> dbAddress = new ArrayList<InetSocketAddress>();
            SourcingType lastType = null;
            for (List<DataSourcing> groupDbAddress : groupDbAddresses) {
                if (lastType != null && !lastType.equals(groupDbAddress.get(i).getType())) {
                    throw new CanalException(String.format("master/slave Sourcing type is unmatch. %s vs %s",
                        lastType,
                        groupDbAddress.get(i).getType()));
                }

                lastType = groupDbAddress.get(i).getType();
                dbAddress.add(groupDbAddress.get(i).getDbAddress());
            }

            // 初始化其中的一个分组parser
            eventParsers.add(doInitEventParser(lastType, dbAddress));
        }

        if (eventParsers.size() > 1) { // 如果存在分组，构造分组的parser
            GroupEventParser groupEventParser = new GroupEventParser();
            groupEventParser.setEventParsers(eventParsers);
            this.eventParser = groupEventParser;
        } else {
            this.eventParser = eventParsers.get(0);
        }
    } else {
        // 创建一个空数据库地址的parser，可能使用了tddl指定地址，启动的时候才会从tddl获取地址
        this.eventParser = doInitEventParser(type, new ArrayList<InetSocketAddress>());
    }

    logger.info("init eventParser end! \n\t load CanalEventParser:{}", eventParser.getClass().getName());
}
 

protected void initEventStore() {
    logger.info("init eventStore begin...");
    StorageMode mode = parameters.getStorageMode();
    if (mode.isMemory()) {
        MemoryEventStoreWithBuffer memoryEventStore = new MemoryEventStoreWithBuffer();
        memoryEventStore.setBufferSize(parameters.getMemoryStorageBufferSize());
        memoryEventStore.setBufferMemUnit(parameters.getMemoryStorageBufferMemUnit());
        memoryEventStore.setBatchMode(BatchMode.valueOf(parameters.getStorageBatchMode().name()));
        memoryEventStore.setDdlIsolation(parameters.getDdlIsolation());
        memoryEventStore.setRaw(parameters.getMemoryStorageRawEntry());
        eventStore = memoryEventStore;
    } else if (mode.isFile()) {
        // 后续版本支持
        throw new CanalException("unsupport MetaMode for " + mode);
    } else if (mode.isMixed()) {
        // 后续版本支持
        throw new CanalException("unsupport MetaMode for " + mode);
    } else {
        throw new CanalException("unsupport MetaMode for " + mode);
    }

    if (eventStore instanceof AbstractCanalStoreScavenge) {
        StorageScavengeMode scavengeMode = parameters.getStorageScavengeMode();
        AbstractCanalStoreScavenge eventScavengeStore = (AbstractCanalStoreScavenge) eventStore;
        eventScavengeStore.setDestination(destination);
        eventScavengeStore.setCanalMetaManager(metaManager);
        eventScavengeStore.setOnAck(scavengeMode.isOnAck());
        eventScavengeStore.setOnFull(scavengeMode.isOnFull());
        eventScavengeStore.setOnSchedule(scavengeMode.isOnSchedule());
        if (scavengeMode.isOnSchedule()) {
            eventScavengeStore.setScavengeSchedule(parameters.getScavengeSchdule());
        }
    }
    logger.info("init eventStore end! \n\t load CanalEventStore:{}", eventStore.getClass().getName());
}
 

protected void initMetaManager() {
    logger.info("init metaManager begin...");
    MetaMode mode = parameters.getMetaMode();
    if (mode.isMemory()) {
        metaManager = new MemoryMetaManager();
    } else if (mode.isZookeeper()) {
        metaManager = new ZooKeeperMetaManager();
        ((ZooKeeperMetaManager) metaManager).setZkClientx(getZkclientx());
    } else if (mode.isMixed()) {
        // metaManager = new MixedMetaManager();
        metaManager = new PeriodMixedMetaManager();// 换用优化过的mixed, at
                                                   // 2012-09-11
        // 设置内嵌的zk metaManager
        ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
        zooKeeperMetaManager.setZkClientx(getZkclientx());
        ((PeriodMixedMetaManager) metaManager).setZooKeeperMetaManager(zooKeeperMetaManager);
    } else if (mode.isLocalFile()) {
        FileMixedMetaManager fileMixedMetaManager = new FileMixedMetaManager();
        fileMixedMetaManager.setDataDir(parameters.getDataDir());
        fileMixedMetaManager.setPeriod(parameters.getMetaFileFlushPeriod());
        metaManager = fileMixedMetaManager;
    } else {
        throw new CanalException("unsupport MetaMode for " + mode);
    }

    logger.info("init metaManager end! \n\t load CanalMetaManager:{} ", metaManager.getClass().getName());
}
 

protected void initMetaManager() {
    logger.info("init metaManager begin...");
    MetaMode mode = parameters.getMetaMode();
    if (mode.isMemory()) {
        metaManager = new MemoryMetaManager();
    } else if (mode.isZookeeper()) {
        metaManager = new ZooKeeperMetaManager();
        ((ZooKeeperMetaManager) metaManager).setZkClientx(getZkclientx());
    } else if (mode.isMixed()) {
        // metaManager = new MixedMetaManager();
        metaManager = new PeriodMixedMetaManager();// 换用优化过的mixed, at
                                                   // 2012-09-11
        // 设置内嵌的zk metaManager
        ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
        zooKeeperMetaManager.setZkClientx(getZkclientx());
        ((PeriodMixedMetaManager) metaManager).setZooKeeperMetaManager(zooKeeperMetaManager);
    } else if (mode.isLocalFile()) {
        FileMixedMetaManager fileMixedMetaManager = new FileMixedMetaManager();
        fileMixedMetaManager.setDataDir(parameters.getDataDir());
        fileMixedMetaManager.setPeriod(parameters.getMetaFileFlushPeriod());
        metaManager = fileMixedMetaManager;
    } else {
        throw new CanalException("unsupport MetaMode for " + mode);
    }

    logger.info("init metaManager end! \n\t load CanalMetaManager:{} ", metaManager.getClass().getName());
}
 

@Override
public void stop() {
    logger.info("## Stop RabbitMQ producer##");
    try {
        this.connect.close();
        this.channel.close();
        super.stop();
    } catch (IOException | TimeoutException ex) {
        throw new CanalException("Stop RabbitMQ producer error", ex);
    }

    super.stop();
}
 

@Override
public void init(Properties properties) {
    RabbitMQProducerConfig rabbitMQProperties = new RabbitMQProducerConfig();
    this.mqProperties = rabbitMQProperties;
    super.init(properties);
    loadRabbitMQProperties(properties);

    ConnectionFactory factory = new ConnectionFactory();
    String servers = rabbitMQProperties.getHost();
    if (servers.contains(":")) {
        String[] serverHostAndPort = servers.split(":");
        factory.setHost(serverHostAndPort[0]);
        factory.setPort(Integer.parseInt(serverHostAndPort[1]));
    } else {
        factory.setHost(servers);
    }

    if (mqProperties.getAliyunAccessKey().length() > 0 && mqProperties.getAliyunSecretKey().length() > 0
        && mqProperties.getAliyunUid() > 0) {
        factory.setCredentialsProvider(new AliyunCredentialsProvider(mqProperties.getAliyunAccessKey(),
            mqProperties.getAliyunSecretKey(),
            mqProperties.getAliyunUid()));
    } else {
        factory.setUsername(rabbitMQProperties.getUsername());
        factory.setPassword(rabbitMQProperties.getPassword());
    }
    factory.setVirtualHost(rabbitMQProperties.getVirtualHost());
    try {
        connect = factory.newConnection();
        channel = connect.createChannel();
        // channel.exchangeDeclare(mqProperties.getExchange(), "topic");
    } catch (IOException | TimeoutException ex) {
        throw new CanalException("Start RabbitMQ producer error", ex);
    }
}
 

protected CanalLogPositionManager initLogPositionManager() {
    logger.info("init logPositionPersistManager begin...");
    IndexMode indexMode = parameters.getIndexMode();
    CanalLogPositionManager logPositionManager;
    if (indexMode.isMemory()) {
        logPositionManager = new MemoryLogPositionManager();
    } else if (indexMode.isZookeeper()) {
        logPositionManager = new ZooKeeperLogPositionManager(getZkclientx());
    } else if (indexMode.isMixed()) {
        MemoryLogPositionManager memoryLogPositionManager = new MemoryLogPositionManager();
        ZooKeeperLogPositionManager zooKeeperLogPositionManager = new ZooKeeperLogPositionManager(getZkclientx());
        logPositionManager = new PeriodMixedLogPositionManager(memoryLogPositionManager,
            zooKeeperLogPositionManager,
            1000L);
    } else if (indexMode.isMeta()) {
        logPositionManager = new MetaLogPositionManager(metaManager);
    } else if (indexMode.isMemoryMetaFailback()) {
        MemoryLogPositionManager primary = new MemoryLogPositionManager();
        MetaLogPositionManager secondary = new MetaLogPositionManager(metaManager);

        logPositionManager = new FailbackLogPositionManager(primary, secondary);
    } else {
        throw new CanalException("unsupport indexMode for " + indexMode);
    }

    logger.info("init logPositionManager end! \n\t load CanalLogPositionManager:{}", logPositionManager.getClass()
        .getName());

    return logPositionManager;
}
 

protected CanalHAController initHaController() {
    logger.info("init haController begin...");
    HAMode haMode = parameters.getHaMode();
    CanalHAController haController = null;
    if (haMode.isHeartBeat()) {
        haController = new HeartBeatHAController();
        ((HeartBeatHAController) haController).setDetectingRetryTimes(parameters.getDetectingRetryTimes());
        ((HeartBeatHAController) haController).setSwitchEnable(parameters.getHeartbeatHaEnable());
    } else {
        throw new CanalException("unsupport HAMode for " + haMode);
    }
    logger.info("init haController end! \n\t load CanalHAController:{}", haController.getClass().getName());

    return haController;
}
 

protected void initEventParser() {
    logger.info("init eventParser begin...");
    SourcingType type = parameters.getSourcingType();

    List<List<DataSourcing>> groupDbAddresses = parameters.getGroupDbAddresses();
    if (!CollectionUtils.isEmpty(groupDbAddresses)) {
        int size = groupDbAddresses.get(0).size();// 取第一个分组的数量，主备分组的数量必须一致
        List<CanalEventParser> eventParsers = new ArrayList<CanalEventParser>();
        for (int i = 0; i < size; i++) {
            List<InetSocketAddress> dbAddress = new ArrayList<InetSocketAddress>();
            SourcingType lastType = null;
            for (List<DataSourcing> groupDbAddress : groupDbAddresses) {
                if (lastType != null && !lastType.equals(groupDbAddress.get(i).getType())) {
                    throw new CanalException(String.format("master/slave Sourcing type is unmatch. %s vs %s",
                        lastType,
                        groupDbAddress.get(i).getType()));
                }

                lastType = groupDbAddress.get(i).getType();
                dbAddress.add(groupDbAddress.get(i).getDbAddress());
            }

            // 初始化其中的一个分组parser
            eventParsers.add(doInitEventParser(lastType, dbAddress));
        }

        if (eventParsers.size() > 1) { // 如果存在分组，构造分组的parser
            GroupEventParser groupEventParser = new GroupEventParser();
            groupEventParser.setEventParsers(eventParsers);
            this.eventParser = groupEventParser;
        } else {
            this.eventParser = eventParsers.get(0);
        }
    } else {
        // 创建一个空数据库地址的parser，可能使用了tddl指定地址，启动的时候才会从tddl获取地址
        this.eventParser = doInitEventParser(type, new ArrayList<InetSocketAddress>());
    }

    logger.info("init eventParser end! \n\t load CanalEventParser:{}", eventParser.getClass().getName());
}
 

protected void initEventStore() {
    logger.info("init eventStore begin...");
    StorageMode mode = parameters.getStorageMode();
    if (mode.isMemory()) {
        MemoryEventStoreWithBuffer memoryEventStore = new MemoryEventStoreWithBuffer();
        memoryEventStore.setBufferSize(parameters.getMemoryStorageBufferSize());
        memoryEventStore.setBufferMemUnit(parameters.getMemoryStorageBufferMemUnit());
        memoryEventStore.setBatchMode(BatchMode.valueOf(parameters.getStorageBatchMode().name()));
        memoryEventStore.setDdlIsolation(parameters.getDdlIsolation());
        memoryEventStore.setRaw(parameters.getMemoryStorageRawEntry());
        eventStore = memoryEventStore;
    } else if (mode.isFile()) {
        // 后续版本支持
        throw new CanalException("unsupport MetaMode for " + mode);
    } else if (mode.isMixed()) {
        // 后续版本支持
        throw new CanalException("unsupport MetaMode for " + mode);
    } else {
        throw new CanalException("unsupport MetaMode for " + mode);
    }

    if (eventStore instanceof AbstractCanalStoreScavenge) {
        StorageScavengeMode scavengeMode = parameters.getStorageScavengeMode();
        AbstractCanalStoreScavenge eventScavengeStore = (AbstractCanalStoreScavenge) eventStore;
        eventScavengeStore.setDestination(destination);
        eventScavengeStore.setCanalMetaManager(metaManager);
        eventScavengeStore.setOnAck(scavengeMode.isOnAck());
        eventScavengeStore.setOnFull(scavengeMode.isOnFull());
        eventScavengeStore.setOnSchedule(scavengeMode.isOnSchedule());
        if (scavengeMode.isOnSchedule()) {
            eventScavengeStore.setScavengeSchedule(parameters.getScavengeSchdule());
        }
    }
    logger.info("init eventStore end! \n\t load CanalEventStore:{}", eventStore.getClass().getName());
}
 

@SuppressWarnings("resource")
protected void initAlarmHandler() {
    logger.info("init alarmHandler begin...");
    String alarmHandlerClass = parameters.getAlarmHandlerClass();
    String alarmHandlerPluginDir = parameters.getAlarmHandlerPluginDir();
    if (alarmHandlerClass == null || alarmHandlerPluginDir == null) {
        alarmHandler = new LogAlarmHandler();
    } else {
        try {
            File externalLibDir = new File(alarmHandlerPluginDir);
            File[] jarFiles = externalLibDir.listFiles(new FilenameFilter() {

                @Override
                public boolean accept(File dir, String name) {
                    return name.endsWith(".jar");
                }
            });
            if (jarFiles == null || jarFiles.length == 0) {
                throw new IllegalStateException(String.format("alarmHandlerPluginDir [%s] can't find any name endswith \".jar\" file.",
                    alarmHandlerPluginDir));
            }
            URL[] urls = new URL[jarFiles.length];
            for (int i = 0; i < jarFiles.length; i++) {
                urls[i] = jarFiles[i].toURI().toURL();
            }
            ClassLoader currentClassLoader = new URLClassLoader(urls,
                CanalInstanceWithManager.class.getClassLoader());
            Class<CanalAlarmHandler> _alarmClass = (Class<CanalAlarmHandler>) currentClassLoader.loadClass(alarmHandlerClass);
            alarmHandler = _alarmClass.newInstance();
            logger.info("init [{}] alarm handler success.", alarmHandlerClass);
        } catch (Throwable e) {
            String errorMsg = String.format("init alarmHandlerPluginDir [%s] alarm handler [%s] error: %s",
                alarmHandlerPluginDir,
                alarmHandlerClass,
                ExceptionUtils.getFullStackTrace(e));
            logger.error(errorMsg);
            throw new CanalException(errorMsg, e);
        }
    }
    logger.info("init alarmHandler end! \n\t load CanalAlarmHandler:{} ", alarmHandler.getClass().getName());
}
 

@Override
protected void initEventParser() {
    logger.info("init eventParser begin...");
    CanalParameter.SourcingType type = parameters.getSourcingType();

    List<List<CanalParameter.DataSourcing>> groupDbAddresses = parameters.getGroupDbAddresses();
    if (!CollectionUtils.isEmpty(groupDbAddresses)) {
        int size = groupDbAddresses.get(0).size();// 取第一个分组的数量，主备分组的数量必须一致
        List<CanalEventParser> eventParsers = new ArrayList<CanalEventParser>();
        for (int i = 0; i < size; i++) {
            List<InetSocketAddress> dbAddress = new ArrayList<InetSocketAddress>();
            CanalParameter.SourcingType lastType = null;
            for (List<CanalParameter.DataSourcing> groupDbAddress : groupDbAddresses) {
                if (lastType != null && !lastType.equals(groupDbAddress.get(i).getType())) {
                    throw new CanalException(String.format("master/slave Sourcing type is unmatch. %s vs %s",
                            lastType,
                            groupDbAddress.get(i).getType()));
                }

                lastType = groupDbAddress.get(i).getType();
                dbAddress.add(groupDbAddress.get(i).getDbAddress());
            }

            // 初始化其中的一个分组parser
            eventParsers.add(doInitEventParser(lastType, dbAddress, (size > 1) ? true : false));//@@changes
        }

        if (eventParsers.size() > 1) { // 如果存在分组，构造分组的parser
            GroupEventParser groupEventParser = new GroupEventParser();
            groupEventParser.setEventParsers(eventParsers);
            this.eventParser = groupEventParser;
        } else {
            this.eventParser = eventParsers.get(0);
        }
    } else {
        // 创建一个空数据库地址的parser，可能使用了tddl指定地址，启动的时候才会从tddl获取地址
        this.eventParser = doInitEventParser(type, new ArrayList<InetSocketAddress>(), false);//@@changes
    }

    logger.info("init eventParser end! \n\t load CanalEventParser:{}", eventParser.getClass().getName());
}