com.alibaba.otter.canal.parse.exception.CanalParseException Java Exaples

Source File: AbstractEventParser.java From canal-1.1.3 with Apache License 2.0

6 votes

public AbstractEventParser(){
    // 初始化一下
    transactionBuffer = new EventTransactionBuffer(new TransactionFlushCallback() {

        public void flush(List<CanalEntry.Entry> transaction) throws InterruptedException {
            boolean successed = consumeTheEventAndProfilingIfNecessary(transaction);
            if (!running) {
                return;
            }

            if (!successed) {
                throw new CanalParseException("consume failed!");
            }

            LogPosition position = buildLastTransactionPosition(transaction);
            if (position != null) { // 可能position为空
                logPositionManager.persistLogPosition(AbstractEventParser.this.destination, position);
            }
        }
    });
}

Source File: MysqlEventParser.java From canal with Apache License 2.0

6 votes

protected void afterDump(ErosaConnection connection) {
    super.afterDump(connection);

    if (connection == null) {
        throw new CanalParseException("illegal connection is null");
    }

    if (!(connection instanceof MysqlConnection)) {
        throw new CanalParseException("Unsupported connection type : " + connection.getClass().getSimpleName());
    }

    if (metaConnection != null) {
        try {
            metaConnection.disconnect();
        } catch (IOException e) {
            logger.error("ERROR # disconnect meta connection for address:{}", metaConnection.getConnector()
                .getAddress(), e);
        }
    }
}

Source File: MysqlConnection.java From canal-1.1.3 with Apache License 2.0

6 votes

/**
 * 获取一下binlog format格式
 */
private void loadBinlogFormat() {
    ResultSetPacket rs = null;
    try {
        rs = query("show variables like 'binlog_format'");
    } catch (IOException e) {
        throw new CanalParseException(e);
    }

    List<String> columnValues = rs.getFieldValues();
    if (columnValues == null || columnValues.size() != 2) {
        logger.warn("unexpected binlog format query result, this may cause unexpected result, so throw exception to request network to io shutdown.");
        throw new IllegalStateException("unexpected binlog format query result:" + rs.getFieldValues());
    }

    binlogFormat = BinlogFormat.valuesOf(columnValues.get(1));
    if (binlogFormat == null) {
        throw new IllegalStateException("unexpected binlog format query result:" + rs.getFieldValues());
    }
}

Source File: LogEventConvert.java From canal with Apache License 2.0

6 votes

private Entry parseRowsQueryEvent(RowsQueryLogEvent event) {
    if (filterQueryDml) {
        return null;
    }
    // mysql5.6支持，需要设置binlog-rows-query-log-events=1，可详细打印原始DML语句
    String queryString = null;
    try {
        queryString = new String(event.getRowsQuery().getBytes(ISO_8859_1), charset.name());
        String tableName = null;
        if (useDruidDdlFilter) {
            List<DdlResult> results = DruidDdlParser.parse(queryString, null);
            if (results.size() > 0) {
                tableName = results.get(0).getTableName();
            }
        }

        return buildQueryEntry(queryString, event.getHeader(), tableName);
    } catch (UnsupportedEncodingException e) {
        throw new CanalParseException(e);
    }
}

Source File: LocalBinlogEventParser.java From canal with Apache License 2.0

6 votes

@Override
protected void preDump(ErosaConnection connection) {
    metaConnection = buildMysqlConnection();
    try {
        metaConnection.connect();
    } catch (IOException e) {
        throw new CanalParseException(e);
    }

    if (tableMetaTSDB != null && tableMetaTSDB instanceof DatabaseTableMeta) {
        ((DatabaseTableMeta) tableMetaTSDB).setConnection(metaConnection);
        ((DatabaseTableMeta) tableMetaTSDB).setFilter(eventFilter);
        ((DatabaseTableMeta) tableMetaTSDB).setBlackFilter(eventBlackFilter);
        ((DatabaseTableMeta) tableMetaTSDB).setSnapshotInterval(tsdbSnapshotInterval);
        ((DatabaseTableMeta) tableMetaTSDB).setSnapshotExpire(tsdbSnapshotExpire);
        ((DatabaseTableMeta) tableMetaTSDB).init(destination);
    }

    tableMetaCache = new TableMetaCache(metaConnection, tableMetaTSDB);
    ((LogEventConvert) binlogParser).setTableMetaCache(tableMetaCache);
}

Source File: MysqlConnection.java From canal with Apache License 2.0

6 votes

/**
 * 获取一下binlog image格式
 */
private void loadBinlogImage() {
    ResultSetPacket rs = null;
    try {
        rs = query("show variables like 'binlog_row_image'");
    } catch (IOException e) {
        throw new CanalParseException(e);
    }

    List<String> columnValues = rs.getFieldValues();
    if (columnValues == null || columnValues.size() != 2) {
        // 可能历时版本没有image特性
        binlogImage = BinlogImage.FULL;
    } else {
        binlogImage = BinlogImage.valuesOf(columnValues.get(1));
    }

    if (binlogFormat == null) {
        throw new IllegalStateException("unexpected binlog image query result:" + rs.getFieldValues());
    }
}

Source File: LocalBinlogEventParser.java From canal-1.1.3 with Apache License 2.0

6 votes

@Override
protected void preDump(ErosaConnection connection) {
    metaConnection = buildMysqlConnection();
    try {
        metaConnection.connect();
    } catch (IOException e) {
        throw new CanalParseException(e);
    }

    if (tableMetaTSDB != null && tableMetaTSDB instanceof DatabaseTableMeta) {
        ((DatabaseTableMeta) tableMetaTSDB).setConnection(metaConnection);
        ((DatabaseTableMeta) tableMetaTSDB).setFilter(eventFilter);
        ((DatabaseTableMeta) tableMetaTSDB).setBlackFilter(eventBlackFilter);
        ((DatabaseTableMeta) tableMetaTSDB).setSnapshotInterval(tsdbSnapshotInterval);
        ((DatabaseTableMeta) tableMetaTSDB).setSnapshotExpire(tsdbSnapshotExpire);
        ((DatabaseTableMeta) tableMetaTSDB).init(destination);
    }

    tableMetaCache = new TableMetaCache(metaConnection, tableMetaTSDB);
    ((LogEventConvert) binlogParser).setTableMetaCache(tableMetaCache);
}

Source File: MysqlMultiStageCoprocessor.java From canal with Apache License 2.0

6 votes

public void onEvent(MessageEvent event, long sequence, boolean endOfBatch) throws Exception {
    try {
        if (event.getEntry() != null) {
            transactionBuffer.add(event.getEntry());
        }

        LogEvent logEvent = event.getEvent();
        if (connection instanceof MysqlConnection && logEvent.getSemival() == 1) {
            // semi ack回报
            ((MysqlConnection) connection).sendSemiAck(logEvent.getHeader().getLogFileName(),
                logEvent.getHeader().getLogPos());
        }

        // clear for gc
        event.setBuffer(null);
        event.setEvent(null);
        event.setTable(null);
        event.setEntry(null);
        event.setNeedDmlParse(false);
    } catch (Throwable e) {
        exception = new CanalParseException(e);
        throw exception;
    }
}

Source File: MysqlEventParser.java From canal with Apache License 2.0

6 votes

private final long generateUniqueServerId() {
    try {
        // a=`echo $masterip|cut -d\. -f1`
        // b=`echo $masterip|cut -d\. -f2`
        // c=`echo $masterip|cut -d\. -f3`
        // d=`echo $masterip|cut -d\. -f4`
        // #server_id=`expr $a \* 256 \* 256 \* 256 + $b \* 256 \* 256 + $c
        // \* 256 + $d `
        // #server_id=$b$c$d
        // server_id=`expr $b \* 256 \* 256 + $c \* 256 + $d `
        InetAddress localHost = InetAddress.getLocalHost();
        byte[] addr = localHost.getAddress();
        int salt = (destination != null) ? destination.hashCode() : 0;
        return ((0x7f & salt) << 24) + ((0xff & (int) addr[1]) << 16) // NL
               + ((0xff & (int) addr[2]) << 8) // NL
               + (0xff & (int) addr[3]);
    } catch (UnknownHostException e) {
        throw new CanalParseException("Unknown host", e);
    }
}

Source File: MysqlConnection.java From canal-1.1.3 with Apache License 2.0

6 votes

/**
 * 获取一下binlog image格式
 */
private void loadBinlogImage() {
    ResultSetPacket rs = null;
    try {
        rs = query("show variables like 'binlog_row_image'");
    } catch (IOException e) {
        throw new CanalParseException(e);
    }

    List<String> columnValues = rs.getFieldValues();
    if (columnValues == null || columnValues.size() != 2) {
        // 可能历时版本没有image特性
        binlogImage = BinlogImage.FULL;
    } else {
        binlogImage = BinlogImage.valuesOf(columnValues.get(1));
    }

    if (binlogFormat == null) {
        throw new IllegalStateException("unexpected binlog image query result:" + rs.getFieldValues());
    }
}

Source File: LogEventConvert.java From DBus with Apache License 2.0

6 votes

private boolean isUpdate(List<Column> bfColumns, String newValue, int index) {
    if (bfColumns == null) {
        throw new CanalParseException("ERROR ## the bfColumns is null");
    }

    if (index < 0) {
        return false;
    }

    for (Column column : bfColumns) {
        if (column.getIndex() == index) {// 比较before / after的column index
            if (column.getIsNull() && newValue == null) {
                // 如果全是null
                return false;
            } else if (newValue != null && (!column.getIsNull() && column.getValue().equals(newValue))) {
                // fixed issue #135, old column is Null
                // 如果不为null，并且相等
                return false;
            }
        }
    }

    // 比如nolob/minial模式下,可能找不到before记录,认为是有变化
    return true;
}

Source File: LogEventConvert.java From canal-1.1.3 with Apache License 2.0

6 votes

private Entry parseRowsQueryEvent(RowsQueryLogEvent event) {
    if (filterQueryDml) {
        return null;
    }
    // mysql5.6支持，需要设置binlog-rows-query-log-events=1，可详细打印原始DML语句
    String queryString = null;
    try {
        queryString = new String(event.getRowsQuery().getBytes(ISO_8859_1), charset.name());
        String tableName = null;
        if (useDruidDdlFilter) {
            List<DdlResult> results = DruidDdlParser.parse(queryString, null);
            if (results.size() > 0) {
                tableName = results.get(0).getTableName();
            }
        }

        return buildQueryEntry(queryString, event.getHeader(), tableName);
    } catch (UnsupportedEncodingException e) {
        throw new CanalParseException(e);
    }
}

Source File: LogEventConvert.java From canal-1.1.3 with Apache License 2.0

6 votes

private TableMeta getTableMeta(String dbName, String tbName, boolean useCache, EntryPosition position) {
    try {
        return tableMetaCache.getTableMeta(dbName, tbName, useCache, position);
    } catch (Throwable e) {
        String message = ExceptionUtils.getRootCauseMessage(e);
        if (filterTableError) {
            if (StringUtils.contains(message, "errorNumber=1146") && StringUtils.contains(message, "doesn't exist")) {
                return null;
            } else if (StringUtils.contains(message, "errorNumber=1142")
                       && StringUtils.contains(message, "command denied")) {
                return null;
            }
        }

        throw new CanalParseException(e);
    }
}

Source File: MysqlEventParser.java From canal-1.1.3 with Apache License 2.0

6 votes

protected void afterDump(ErosaConnection connection) {
    super.afterDump(connection);

    if (connection == null) {
        throw new CanalParseException("illegal connection is null");
    }

    if (!(connection instanceof MysqlConnection)) {
        throw new CanalParseException("Unsupported connection type : " + connection.getClass().getSimpleName());
    }

    if (metaConnection != null) {
        try {
            metaConnection.disconnect();
        } catch (IOException e) {
            logger.error("ERROR # disconnect meta connection for address:{}", metaConnection.getConnector()
                .getAddress(), e);
        }
    }
}

Source File: LogEventConvert_old.java From DBus with Apache License 2.0

6 votes

private boolean isUpdate(List<Column> bfColumns, String newValue, int index) {
    if (bfColumns == null) {
        throw new CanalParseException("ERROR ## the bfColumns is null");
    }

    if (index < 0) {
        return false;
    }

    for (Column column : bfColumns) {
        if (column.getIndex() == index) {// �Ƚ�before / after��column index
            if (column.getIsNull() && newValue == null) {
                // ���ȫ��null
                return false;
            } else if (newValue != null && (!column.getIsNull() && column.getValue().equals(newValue))) {
                // fixed issue #135, old column is Null
                // �����Ϊnull���������
                return false;
            }
        }
    }

    // ����nolob/minialģʽ��,�����Ҳ���before��¼,��Ϊ���б仯
    return true;
}

Source File: MysqlEventParser.java From canal-1.1.3 with Apache License 2.0

6 votes

private final long generateUniqueServerId() {
    try {
        // a=`echo $masterip|cut -d\. -f1`
        // b=`echo $masterip|cut -d\. -f2`
        // c=`echo $masterip|cut -d\. -f3`
        // d=`echo $masterip|cut -d\. -f4`
        // #server_id=`expr $a \* 256 \* 256 \* 256 + $b \* 256 \* 256 + $c
        // \* 256 + $d `
        // #server_id=$b$c$d
        // server_id=`expr $b \* 256 \* 256 + $c \* 256 + $d `
        InetAddress localHost = InetAddress.getLocalHost();
        byte[] addr = localHost.getAddress();
        int salt = (destination != null) ? destination.hashCode() : 0;
        return ((0x7f & salt) << 24) + ((0xff & (int) addr[1]) << 16) // NL
               + ((0xff & (int) addr[2]) << 8) // NL
               + (0xff & (int) addr[3]);
    } catch (UnknownHostException e) {
        throw new CanalParseException("Unknown host", e);
    }
}

Source File: LogEventConvert.java From DBus with Apache License 2.0

6 votes

private boolean isUpdate(List<Column> bfColumns, String newValue, int index) {
    if (bfColumns == null) {
        throw new CanalParseException("ERROR ## the bfColumns is null");
    }

    if (index < 0) {
        return false;
    }

    for (Column column : bfColumns) {
        if (column.getIndex() == index) {// 比较before / after的column index
            if (column.getIsNull() && newValue == null) {
                // 如果全是null
                return false;
            } else if (newValue != null && (!column.getIsNull() && column.getValue().equals(newValue))) {
                // fixed issue #135, old column is Null
                // 如果不为null，并且相等
                return false;
            }
        }
    }

    // 比如nolob/minial模式下,可能找不到before记录,认为是有变化
    return true;
}

Source File: MysqlEventParser.java From canal-1.1.3 with Apache License 2.0

6 votes

/**
 * 查询当前的binlog位置
 */
private EntryPosition findEndPosition(MysqlConnection mysqlConnection) {
    try {
        ResultSetPacket packet = mysqlConnection.query("show master status");
        List<String> fields = packet.getFieldValues();
        if (CollectionUtils.isEmpty(fields)) {
            throw new CanalParseException("command : 'show master status' has an error! pls check. you need (at least one of) the SUPER,REPLICATION CLIENT privilege(s) for this operation");
        }
        EntryPosition endPosition = new EntryPosition(fields.get(0), Long.valueOf(fields.get(1)));
        if (isGTIDMode() && fields.size() > 4) {
            endPosition.setGtid(fields.get(4));
        }
        return endPosition;
    } catch (IOException e) {
        throw new CanalParseException("command : 'show master status' has an error!", e);
    }
}

Source File: AbstractEventParser.java From DBus with Apache License 2.0

6 votes

public AbstractEventParser(){
    // 初始化一下
    transactionBuffer = new EventTransactionBuffer(new TransactionFlushCallback() {

        public void flush(List<CanalEntry.Entry> transaction) throws InterruptedException {
            boolean successed = consumeTheEventAndProfilingIfNecessary(transaction);
            if (!running) {
                return;
            }

            if (!successed) {
                throw new CanalParseException("consume failed!");
            }

            LogPosition position = buildLastTransactionPosition(transaction);
            if (position != null) { // 可能position为空
                logPositionManager.persistLogPosition(AbstractEventParser.this.destination, position);
            }
        }
    });
}

Source File: MysqlMultiStageCoprocessor.java From canal-1.1.3 with Apache License 2.0

6 votes

public void onEvent(MessageEvent event, long sequence, boolean endOfBatch) throws Exception {
    try {
        if (event.getEntry() != null) {
            transactionBuffer.add(event.getEntry());
        }

        LogEvent logEvent = event.getEvent();
        if (connection instanceof MysqlConnection && logEvent.getSemival() == 1) {
            // semi ack回报
            ((MysqlConnection) connection).sendSemiAck(logEvent.getHeader().getLogFileName(),
                logEvent.getHeader().getLogPos());
        }

        // clear for gc
        event.setBuffer(null);
        event.setEvent(null);
        event.setTable(null);
        event.setEntry(null);
        event.setNeedDmlParse(false);
    } catch (Throwable e) {
        exception = new CanalParseException(e);
        throw exception;
    }
}

Source File: LogEventConvert.java From canal-1.1.3 with Apache License 2.0

6 votes

private boolean isUpdate(List<Column> bfColumns, String newValue, int index) {
    if (bfColumns == null) {
        throw new CanalParseException("ERROR ## the bfColumns is null");
    }

    if (index < 0) {
        return false;
    }

    for (Column column : bfColumns) {
        if (column.getIndex() == index) {// 比较before / after的column index
            if (column.getIsNull() && newValue == null) {
                // 如果全是null
                return false;
            } else if (newValue != null && (!column.getIsNull() && column.getValue().equals(newValue))) {
                // fixed issue #135, old column is Null
                // 如果不为null，并且相等
                return false;
            }
        }
    }

    // 比如nolob/minial模式下,可能找不到before记录,认为是有变化
    return true;
}

Source File: LogEventConvert.java From canal with Apache License 2.0

6 votes

private TableMeta getTableMeta(String dbName, String tbName, boolean useCache, EntryPosition position) {
    try {
        return tableMetaCache.getTableMeta(dbName, tbName, useCache, position);
    } catch (Throwable e) {
        String message = ExceptionUtils.getRootCauseMessage(e);
        if (filterTableError) {
            if (StringUtils.contains(message, "errorNumber=1146") && StringUtils.contains(message, "doesn't exist")) {
                return null;
            } else if (StringUtils.contains(message, "errorNumber=1142")
                       && StringUtils.contains(message, "command denied")) {
                return null;
            }
        }

        throw new CanalParseException(e);
    }
}

Source File: AbstractMysqlEventParser.java From canal-1.1.3 with Apache License 2.0

6 votes

public void start() throws CanalParseException {
    if (enableTsdb) {
        if (tableMetaTSDB == null) {
            synchronized (CanalEventParser.class) {
                try {
                    // 设置当前正在加载的通道，加载spring查找文件时会用到该变量
                    System.setProperty("canal.instance.destination", destination);
                    // 初始化
                    tableMetaTSDB = tableMetaTSDBFactory.build(destination, tsdbSpringXml);
                } finally {
                    System.setProperty("canal.instance.destination", "");
                }
            }
        }
    }

    super.start();
}

Source File: MysqlEventParser.java From canal with Apache License 2.0

6 votes

public void stop() throws CanalParseException {
    if (metaConnection != null) {
        try {
            metaConnection.disconnect();
        } catch (IOException e) {
            logger.error("ERROR # disconnect meta connection for address:{}", metaConnection.getConnector()
                .getAddress(), e);
        }
    }

    if (tableMetaCache != null) {
        tableMetaCache.clearTableMeta();
    }

    super.stop();
}

Source File: FailbackLogPositionManager.java From canal with Apache License 2.0

5 votes

@Override
public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
    try {
        primary.persistLogPosition(destination, logPosition);
    } catch (CanalParseException e) {
        logger.warn("persistLogPosition use primary log position manager exception. destination: {}, logPosition: {}",
            destination,
            logPosition,
            e);
        secondary.persistLogPosition(destination, logPosition);
    }
}

Source File: MixedLogPositionManager.java From canal-1.1.3 with Apache License 2.0

5 votes

@Override
public void persistLogPosition(final String destination, final LogPosition logPosition) throws CanalParseException {
    memoryLogPositionManager.persistLogPosition(destination, logPosition);
    executor.submit(new Runnable() {

        public void run() {
            try {
                zooKeeperLogPositionManager.persistLogPosition(destination, logPosition);
            } catch (Exception e) {
                logger.error("ERROR # persist to zookeeper has an error", e);
            }
        }
    });
}

Source File: DirectLogFetcherTest.java From canal with Apache License 2.0

5 votes

private void loadBinlogChecksum(MysqlConnector connector) {
    ResultSetPacket rs = null;
    try {
        rs = query("select @@global.binlog_checksum", connector);
    } catch (IOException e) {
        throw new CanalParseException(e);
    }

    List<String> columnValues = rs.getFieldValues();
    if (columnValues != null && columnValues.size() >= 1 && columnValues.get(0).toUpperCase().equals("CRC32")) {
        binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_CRC32;
    } else {
        binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_OFF;
    }
}

Source File: DatabaseTableMeta.java From canal with Apache License 2.0

5 votes

private EntryPosition buildMemFromSnapshot(EntryPosition position) {
    try {
        MetaSnapshotDO snapshotDO = metaSnapshotDAO.findByTimestamp(destination, position.getTimestamp());
        if (snapshotDO == null) {
            return null;
        }
        String binlogFile = snapshotDO.getBinlogFile();
        Long binlogOffest = snapshotDO.getBinlogOffest();
        String binlogMasterId = snapshotDO.getBinlogMasterId();
        Long binlogTimestamp = snapshotDO.getBinlogTimestamp();

        EntryPosition snapshotPosition = new EntryPosition(binlogFile,
            binlogOffest == null ? 0l : binlogOffest,
            binlogTimestamp == null ? 0l : binlogTimestamp,
            Long.valueOf(binlogMasterId == null ? "-2" : binlogMasterId));
        // data存储为Map<String,String>，每个分库一套建表
        String sqlData = snapshotDO.getData();
        JSONObject jsonObj = JSON.parseObject(sqlData);
        for (Map.Entry entry : jsonObj.entrySet()) {
            // 记录到内存
            if (!memoryTableMeta.apply(snapshotPosition,
                ObjectUtils.toString(entry.getKey()),
                ObjectUtils.toString(entry.getValue()),
                null)) {
                return null;
            }
        }

        return snapshotPosition;
    } catch (Throwable e) {
        throw new CanalParseException("apply failed caused by : " + e.getMessage(), e);
    }
}

Source File: DirectLogFetcherTest.java From canal-1.1.3 with Apache License 2.0

5 votes

private void loadBinlogChecksum(MysqlConnector connector) {
    ResultSetPacket rs = null;
    try {
        rs = query("select @@global.binlog_checksum", connector);
    } catch (IOException e) {
        throw new CanalParseException(e);
    }

    List<String> columnValues = rs.getFieldValues();
    if (columnValues != null && columnValues.size() >= 1 && columnValues.get(0).toUpperCase().equals("CRC32")) {
        binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_CRC32;
    } else {
        binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_OFF;
    }
}

Source File: FailbackLogPositionManager.java From canal-1.1.3 with Apache License 2.0

5 votes

@Override
public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
    try {
        primary.persistLogPosition(destination, logPosition);
    } catch (CanalParseException e) {
        logger.warn("persistLogPosition use primary log position manager exception. destination: {}, logPosition: {}",
            destination,
            logPosition,
            e);
        secondary.persistLogPosition(destination, logPosition);
    }
}

com.alibaba.otter.canal.parse.exception.CanalParseException Java Examples