com.alibaba.otter.canal.instance.core.CanalInstance Java Exaples

Source File: EntryCollector.java From canal with Apache License 2.0

6 votes

@Override
public void register(CanalInstance instance) {
    final String destination = instance.getDestination();
    EntryMetricsHolder holder = new EntryMetricsHolder();
    holder.destLabelValues = Collections.singletonList(destination);
    CanalEventSink sink = instance.getEventSink();
    if (!(sink instanceof EntryEventSink)) {
        throw new IllegalArgumentException("CanalEventSink must be EntryEventSink");
    }
    EntryEventSink entrySink = (EntryEventSink) sink;
    PrometheusCanalEventDownStreamHandler handler = assembleHandler(entrySink);
    holder.latestExecTime = handler.getLatestExecuteTime();
    holder.transactionCounter = handler.getTransactionCounter();
    Preconditions.checkNotNull(holder.latestExecTime);
    Preconditions.checkNotNull(holder.transactionCounter);
    EntryMetricsHolder old = instances.put(destination, holder);
    if (old != null) {
        logger.warn("Remove stale EntryCollector for instance {}.", destination);
    }
}

Source File: CanalServerWithEmbedded.java From canal with Apache License 2.0

6 votes

public void start() {
    if (!isStart()) {
        super.start();
        // 如果存在provider,则启动metrics service
        loadCanalMetrics();
        metrics.setServerPort(metricsPort);
        metrics.initialize();
        canalInstances = MigrateMap.makeComputingMap(new Function<String, CanalInstance>() {

            public CanalInstance apply(String destination) {
                return canalInstanceGenerator.generate(destination);
            }
        });

        // lastRollbackPostions = new MapMaker().makeMap();
    }
}

Source File: CanalServerWithEmbedded.java From canal with Apache License 2.0

6 votes

public void stop() {
    super.stop();
    for (Map.Entry<String, CanalInstance> entry : canalInstances.entrySet()) {
        try {
            CanalInstance instance = entry.getValue();
            if (instance.isStart()) {
                try {
                    String destination = entry.getKey();
                    MDC.put("destination", destination);
                    entry.getValue().stop();
                    logger.info("stop CanalInstances[{}] successfully", destination);
                } finally {
                    MDC.remove("destination");
                }
            }
        } catch (Exception e) {
            logger.error(String.format("stop CanalInstance[%s] has an error", entry.getKey()), e);
        }
    }
    metrics.terminate();
}

Source File: CanalServerWithEmbedded.java From canal with Apache License 2.0

6 votes

public void stop(String destination) {
    CanalInstance canalInstance = canalInstances.remove(destination);
    if (canalInstance != null) {
        if (canalInstance.isStart()) {
            try {
                MDC.put("destination", destination);
                canalInstance.stop();
                if (metrics.isRunning()) {
                    metrics.unregister(canalInstance);
                }
                logger.info("stop CanalInstances[{}] successfully", destination);
            } finally {
                MDC.remove("destination");
            }
        }
    }
}

Source File: CanalServerWithEmbedded.java From canal with Apache License 2.0

6 votes

/**
 * 客户端订阅，重复订阅时会更新对应的filter信息
 */
@Override
public void subscribe(ClientIdentity clientIdentity) throws CanalServerException {
    checkStart(clientIdentity.getDestination());

    CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
    if (!canalInstance.getMetaManager().isStart()) {
        canalInstance.getMetaManager().start();
    }

    canalInstance.getMetaManager().subscribe(clientIdentity); // 执行一下meta订阅

    Position position = canalInstance.getMetaManager().getCursor(clientIdentity);
    if (position == null) {
        position = canalInstance.getEventStore().getFirstPosition();// 获取一下store中的第一条
        if (position != null) {
            canalInstance.getMetaManager().updateCursor(clientIdentity, position); // 更新一下cursor
        }
        logger.info("subscribe successfully, {} with first position:{} ", clientIdentity, position);
    } else {
        logger.info("subscribe successfully, use last cursor position:{} ", clientIdentity, position);
    }

    // 通知下订阅关系变化
    canalInstance.subscribeChange(clientIdentity);
}

Source File: SinkCollector.java From canal-1.1.3 with Apache License 2.0

6 votes

@Override
public void register(CanalInstance instance) {
    final String destination = instance.getDestination();
    SinkMetricsHolder holder = new SinkMetricsHolder();
    holder.destLabelValues = Collections.singletonList(destination);
    CanalEventSink sink = instance.getEventSink();
    if (!(sink instanceof EntryEventSink)) {
        throw new IllegalArgumentException("CanalEventSink must be EntryEventSink");
    }
    EntryEventSink entrySink = (EntryEventSink) sink;
    holder.eventsSinkBlockingTime = entrySink.getEventsSinkBlockingTime();
    Preconditions.checkNotNull(holder.eventsSinkBlockingTime);
    SinkMetricsHolder old = instances.put(destination, holder);
    if (old != null) {
        logger.warn("Remote stale SinkCollector for instance {}.", destination);
    }
}

Source File: CanalServerWithEmbedded.java From canal-1.1.3 with Apache License 2.0

6 votes

/**
 * 回滚到未进行 {@link #ack} 的地方，下次fetch的时候，可以从最后一个没有 {@link #ack} 的地方开始拿
 */
@Override
public void rollback(ClientIdentity clientIdentity) throws CanalServerException {
    checkStart(clientIdentity.getDestination());
    CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
    // 因为存在第一次链接时自动rollback的情况，所以需要忽略未订阅
    boolean hasSubscribe = canalInstance.getMetaManager().hasSubscribe(clientIdentity);
    if (!hasSubscribe) {
        return;
    }

    synchronized (canalInstance) {
        // 清除batch信息
        canalInstance.getMetaManager().clearAllBatchs(clientIdentity);
        // rollback eventStore中的状态信息
        canalInstance.getEventStore().rollback();
        logger.info("rollback successfully, clientId:{}", new Object[] { clientIdentity.getClientId() });
    }
}

Source File: CanalServerWithEmbedded.java From canal with Apache License 2.0

6 votes

/**
 * 回滚到未进行 {@link #ack} 的地方，下次fetch的时候，可以从最后一个没有 {@link #ack} 的地方开始拿
 */
@Override
public void rollback(ClientIdentity clientIdentity) throws CanalServerException {
    checkStart(clientIdentity.getDestination());
    CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
    // 因为存在第一次链接时自动rollback的情况，所以需要忽略未订阅
    boolean hasSubscribe = canalInstance.getMetaManager().hasSubscribe(clientIdentity);
    if (!hasSubscribe) {
        return;
    }

    synchronized (canalInstance) {
        // 清除batch信息
        canalInstance.getMetaManager().clearAllBatchs(clientIdentity);
        // rollback eventStore中的状态信息
        canalInstance.getEventStore().rollback();
        logger.info("rollback successfully, clientId:{}", new Object[] { clientIdentity.getClientId() });
    }
}

Source File: SpringCanalInstanceGenerator.java From canal with Apache License 2.0

6 votes

public CanalInstance generate(String destination) {
    synchronized (CanalInstanceGenerator.class) {
        try {
            // 设置当前正在加载的通道，加载spring查找文件时会用到该变量
            System.setProperty("canal.instance.destination", destination);
            this.beanFactory = getBeanFactory(springXml);
            String beanName = destination;
            if (!beanFactory.containsBean(beanName)) {
                beanName = defaultName;
            }

            return (CanalInstance) beanFactory.getBean(beanName);
        } catch (Throwable e) {
            logger.error("generator instance failed.", e);
            throw new CanalException(e);
        } finally {
            System.setProperty("canal.instance.destination", "");
        }
    }
}

Source File: GatewayInstance.java From DataLink with Apache License 2.0

6 votes

/**
 * 并发量不大，直接用互斥锁就行了
 */
public synchronized void registerEndpointInstance(String destination, CanalInstance canalInstance, KickoutListener kickoutListener, String filter) {
    String oldMDC = MDC.get(Constants.MDC_TASKID);
    try {
        MDC.put(Constants.MDC_TASKID, gwCanal.getName());

        logger.info("register an endpoint instance with name {} begin.", destination);
        this.endpointInstances.put(destination, new RegisterItem(canalInstance, kickoutListener, filter));
        logger.info("register an endpoint instance with name {} end.", destination);
    } finally {
        if (StringUtils.isNotBlank(oldMDC)) {
            MDC.put(Constants.MDC_TASKID, oldMDC);
        } else {
            MDC.remove(Constants.MDC_TASKID);
        }
    }
}

Source File: CanalServerWithEmbedded.java From canal-1.1.3 with Apache License 2.0

6 votes

/**
 * 客户端订阅，重复订阅时会更新对应的filter信息
 */
@Override
public void subscribe(ClientIdentity clientIdentity) throws CanalServerException {
    checkStart(clientIdentity.getDestination());

    CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
    if (!canalInstance.getMetaManager().isStart()) {
        canalInstance.getMetaManager().start();
    }

    canalInstance.getMetaManager().subscribe(clientIdentity); // 执行一下meta订阅

    Position position = canalInstance.getMetaManager().getCursor(clientIdentity);
    if (position == null) {
        position = canalInstance.getEventStore().getFirstPosition();// 获取一下store中的第一条
        if (position != null) {
            canalInstance.getMetaManager().updateCursor(clientIdentity, position); // 更新一下cursor
        }
        logger.info("subscribe successfully, {} with first position:{} ", clientIdentity, position);
    } else {
        logger.info("subscribe successfully, use last cursor position:{} ", clientIdentity, position);
    }

    // 通知下订阅关系变化
    canalInstance.subscribeChange(clientIdentity);
}

Source File: EntryCollector.java From canal-1.1.3 with Apache License 2.0

6 votes

@Override
public void register(CanalInstance instance) {
    final String destination = instance.getDestination();
    EntryMetricsHolder holder = new EntryMetricsHolder();
    holder.destLabelValues = Collections.singletonList(destination);
    CanalEventSink sink = instance.getEventSink();
    if (!(sink instanceof EntryEventSink)) {
        throw new IllegalArgumentException("CanalEventSink must be EntryEventSink");
    }
    EntryEventSink entrySink = (EntryEventSink) sink;
    PrometheusCanalEventDownStreamHandler handler = assembleHandler(entrySink);
    holder.latestExecTime = handler.getLatestExecuteTime();
    holder.transactionCounter = handler.getTransactionCounter();
    Preconditions.checkNotNull(holder.latestExecTime);
    Preconditions.checkNotNull(holder.transactionCounter);
    EntryMetricsHolder old = instances.put(destination, holder);
    if (old != null) {
        logger.warn("Remove stale EntryCollector for instance {}.", destination);
    }
}

Source File: CanalServerWithEmbedded.java From canal-1.1.3 with Apache License 2.0

6 votes

public void stop() {
    super.stop();
    for (Map.Entry<String, CanalInstance> entry : canalInstances.entrySet()) {
        try {
            CanalInstance instance = entry.getValue();
            if (instance.isStart()) {
                try {
                    String destination = entry.getKey();
                    MDC.put("destination", destination);
                    entry.getValue().stop();
                    logger.info("stop CanalInstances[{}] successfully", destination);
                } finally {
                    MDC.remove("destination");
                }
            }
        } catch (Exception e) {
            logger.error(String.format("stop CanalInstance[%s] has an error", entry.getKey()), e);
        }
    }
    metrics.terminate();
}

Source File: CanalServerWithEmbedded.java From canal-1.1.3 with Apache License 2.0

6 votes

public void start() {
    if (!isStart()) {
        super.start();
        // 如果存在provider,则启动metrics service
        loadCanalMetrics();
        metrics.setServerPort(metricsPort);
        metrics.initialize();
        canalInstances = MigrateMap.makeComputingMap(new Function<String, CanalInstance>() {

            public CanalInstance apply(String destination) {
                return canalInstanceGenerator.generate(destination);
            }
        });

        // lastRollbackPostions = new MapMaker().makeMap();
    }
}

Source File: ParserCollector.java From canal with Apache License 2.0

6 votes

@Override
public void register(CanalInstance instance) {
    final String destination = instance.getDestination();
    ParserMetricsHolder holder;
    CanalEventParser parser = instance.getEventParser();
    if (parser instanceof AbstractMysqlEventParser) {
        holder = singleHolder(destination, (AbstractMysqlEventParser)parser, "0");
    } else if (parser instanceof GroupEventParser) {
        holder = groupHolder(destination, (GroupEventParser)parser);
    } else {
        throw new IllegalArgumentException("CanalEventParser must be either AbstractMysqlEventParser or GroupEventParser.");
    }
    Preconditions.checkNotNull(holder);
    ParserMetricsHolder old = instances.put(destination, holder);
    if (old != null) {
        logger.warn("Remove stale ParserCollector for instance {}.", destination);
    }
}

Source File: CanalServerTest.java From canal with Apache License 2.0

6 votes

@Before
public void setUp() {
    CanalServerWithEmbedded embeddedServer = new CanalServerWithEmbedded();
    embeddedServer.setCanalInstanceGenerator(new CanalInstanceGenerator() {

        public CanalInstance generate(String destination) {
            Canal canal = buildCanal();
            return new CanalInstanceWithManager(canal, FILTER);
        }
    });

    nettyServer = CanalServerWithNetty.instance();
    nettyServer.setEmbeddedServer(embeddedServer);
    nettyServer.setPort(1088);
    nettyServer.start();
}

Source File: CanalAdminController.java From canal with Apache License 2.0

6 votes

@Override
public String getRunningInstances() {
    try {
        Map<String, CanalInstance> instances = CanalServerWithEmbedded.instance().getCanalInstances();
        List<String> runningInstances = new ArrayList<String>();
        instances.forEach((destination, instance) -> {
            if (instance.isStart()) {
                runningInstances.add(destination);
            }
        });

        return Joiner.on(",").join(runningInstances);
    } catch (Throwable e) {
        logger.error(e.getMessage(), e);
    }
    return "";
}

Source File: ParserCollector.java From canal-1.1.3 with Apache License 2.0

6 votes

@Override
public void register(CanalInstance instance) {
    final String destination = instance.getDestination();
    ParserMetricsHolder holder;
    CanalEventParser parser = instance.getEventParser();
    if (parser instanceof AbstractMysqlEventParser) {
        holder = singleHolder(destination, (AbstractMysqlEventParser)parser, "0");
    } else if (parser instanceof GroupEventParser) {
        holder = groupHolder(destination, (GroupEventParser)parser);
    } else {
        throw new IllegalArgumentException("CanalEventParser must be either AbstractMysqlEventParser or GroupEventParser.");
    }
    Preconditions.checkNotNull(holder);
    ParserMetricsHolder old = instances.put(destination, holder);
    if (old != null) {
        logger.warn("Remove stale ParserCollector for instance {}.", destination);
    }
}

Source File: SinkCollector.java From canal with Apache License 2.0

6 votes

@Override
public void register(CanalInstance instance) {
    final String destination = instance.getDestination();
    SinkMetricsHolder holder = new SinkMetricsHolder();
    holder.destLabelValues = Collections.singletonList(destination);
    CanalEventSink sink = instance.getEventSink();
    if (!(sink instanceof EntryEventSink)) {
        throw new IllegalArgumentException("CanalEventSink must be EntryEventSink");
    }
    EntryEventSink entrySink = (EntryEventSink) sink;
    holder.eventsSinkBlockingTime = entrySink.getEventsSinkBlockingTime();
    Preconditions.checkNotNull(holder.eventsSinkBlockingTime);
    SinkMetricsHolder old = instances.put(destination, holder);
    if (old != null) {
        logger.warn("Remote stale SinkCollector for instance {}.", destination);
    }
}

Source File: CanalServerWithEmbedded.java From canal with Apache License 2.0

5 votes

private void checkSubscribe(ClientIdentity clientIdentity) {
    CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
    boolean hasSubscribe = canalInstance.getMetaManager().hasSubscribe(clientIdentity);
    if (!hasSubscribe) {
        throw new CanalServerException(String.format("ClientIdentity:%s should subscribe first",
            clientIdentity.toString()));
    }
}

Source File: BaseCanalServerWithEmbededTest.java From canal with Apache License 2.0

5 votes

@Before
public void setUp() {
    server = CanalServerWithEmbedded.instance();
    server.setCanalInstanceGenerator(new CanalInstanceGenerator() {

        public CanalInstance generate(String destination) {
            Canal canal = buildCanal();
            return new CanalInstanceWithManager(canal, FILTER);
        }
    });
    server.start();
    server.start(DESTINATION);
}

Source File: CanalMQStarter.java From canal with Apache License 2.0

5 votes

public synchronized void startDestination(String destination) {
    CanalInstance canalInstance = canalServer.getCanalInstances().get(destination);
    if (canalInstance != null) {
        stopDestination(destination);
        CanalMQRunnable canalMQRunnable = new CanalMQRunnable(destination);
        canalMQWorks.put(canalInstance.getDestination(), canalMQRunnable);
        executorService.execute(canalMQRunnable);
        logger.info("## Start the MQ work of destination:" + destination);
    }
}

Source File: CanalInstanceExports.java From canal with Apache License 2.0

5 votes

void register(CanalInstance instance) {
    requiredInstanceRegistry(storeCollector).register(instance);
    requiredInstanceRegistry(entryCollector).register(instance);
    requiredInstanceRegistry(metaCollector).register(instance);
    requiredInstanceRegistry(sinkCollector).register(instance);
    requiredInstanceRegistry(parserCollector).register(instance);
    logger.info("Successfully register metrics for instance {}.", instance.getDestination());
}

Source File: PrometheusService.java From canal with Apache License 2.0

5 votes

@Override
public void register(CanalInstance instance) {
    if (instance.isStart()) {
        logger.warn("Cannot register metrics for destination {} that is running.", instance.getDestination());
        return;
    }
    try {
        instanceExports.register(instance);
    } catch (Throwable t) {
        logger.warn("Unable to register instance exports for {}.", instance.getDestination(), t);
    }
    logger.info("Register metrics for destination {}.", instance.getDestination());
}

Source File: PrometheusService.java From canal-1.1.3 with Apache License 2.0

5 votes

@Override
public void unregister(CanalInstance instance) {
    if (instance.isStart()) {
        logger.warn("Try unregister metrics after destination {} is stopped.", instance.getDestination());
    }
    try {
        instanceExports.unregister(instance);
    } catch (Throwable t) {
        logger.warn("Unable to unregister instance exports for {}.", instance.getDestination(), t);
    }
    logger.info("Unregister metrics for destination {}.", instance.getDestination());
}

Source File: DefaultSpringInstanceTest.java From canal with Apache License 2.0

5 votes

@Test
public void testInstance() {
    CanalInstanceGenerator generator = (CanalInstanceGenerator) context.getBean("canalInstanceGenerator");
    CanalInstance canalInstance = generator.generate("instance");
    Assert.notNull(canalInstance);

    canalInstance.start();
    try {
        Thread.sleep(10 * 1000);
    } catch (InterruptedException e) {
    }
    canalInstance.stop();
}

Source File: PrometheusService.java From canal-1.1.3 with Apache License 2.0

5 votes

@Override
public void register(CanalInstance instance) {
    if (instance.isStart()) {
        logger.warn("Cannot register metrics for destination {} that is running.", instance.getDestination());
        return;
    }
    try {
        instanceExports.register(instance);
    } catch (Throwable t) {
        logger.warn("Unable to register instance exports for {}.", instance.getDestination(), t);
    }
    logger.info("Register metrics for destination {}.", instance.getDestination());
}

Source File: GroupSpringInstanceTest.java From canal with Apache License 2.0

5 votes

@Test
public void testInstance() {
    CanalInstanceGenerator generator = (CanalInstanceGenerator) context.getBean("canalInstanceGenerator");
    CanalInstance canalInstance = generator.generate("instance");
    Assert.notNull(canalInstance);

    canalInstance.start();
    try {
        Thread.sleep(10 * 1000);
    } catch (InterruptedException e) {
    }
    canalInstance.stop();
}

Source File: MemorySpringInstanceTest.java From canal with Apache License 2.0

5 votes

@Test
public void testInstance() {
    CanalInstanceGenerator generator = (CanalInstanceGenerator) context.getBean("canalInstanceGenerator");
    CanalInstance canalInstance = generator.generate("instance");
    Assert.notNull(canalInstance);

    canalInstance.start();
    try {
        Thread.sleep(10 * 1000);
    } catch (InterruptedException e) {
    }
    canalInstance.stop();
}

Source File: SpringCanalInstanceGenerator.java From canal-1.1.3 with Apache License 2.0

5 votes

public CanalInstance generate(String destination) {
    String beanName = destination;
    if (!beanFactory.containsBean(beanName)) {
        beanName = defaultName;
    }

    return (CanalInstance) beanFactory.getBean(beanName);
}

com.alibaba.otter.canal.instance.core.CanalInstance Java Examples