Java Code Examples for com.alibaba.otter.canal.protocol.CanalEntry#Column

public void insert(List<CanalEntry.Column> data, String schemaName, String tableName) {
    DBObject obj = DBConvertUtil.columnToJson(data);
    logger.debug("insert ：{}", obj.toString());
    //订单库单独处理
    if (schemaName.equals("order")) {
        //保存原始数据
        if (tableName.startsWith("order_base_info")) {
            tableName = "order_base_info";
        } else if (tableName.startsWith("order_detail_info")) {
            tableName = "order_detail_info";
        } else {
            logger.info("unknown data ：{}.{}:{}", schemaName, tableName, obj);
            return;
        }
        insertData(schemaName, tableName, obj, obj);
    } else {
        DBObject newObj = (DBObject) ObjectUtils.clone(obj);
        if (newObj.containsField("id")) {
            newObj.put("_id", newObj.get("id"));
            newObj.removeField("id");
        }
        insertData(schemaName, tableName, newObj, obj);
    }
}
 

public void update(List<CanalEntry.Column> data, String schemaName, String tableName) {
    DBObject obj = DBConvertUtil.columnToJson(data);
    logger.debug("update：{}", obj.toString());
    //订单库单独处理
    if (schemaName.equals("order")) {
        if (tableName.startsWith("order_base_info")) {
            tableName = "order_base_info";
        } else if (tableName.startsWith("order_detail_info")) {
            tableName = "order_detail_info";
        } else {
            logger.info("unknown data：{}.{}:{}", schemaName, tableName, obj);
        }
        updateData(schemaName, tableName, new BasicDBObject("orderId", obj.get("orderId")), obj);
    } else {
        if (obj.containsField("id")) {
            updateData(schemaName, tableName, new BasicDBObject("_id", obj.get("id")), obj);
        } else {
            logger.info("unknown data structure");
        }
    }
}
 

public static Map<String, String> genColumn(CanalEntry.Column column) {
    Map<String, String> row = new LinkedHashMap<>();
    if (column.getIsKey()) {
        row.put("isKey", "1");
    } else {
        row.put("isKey", "0");
    }
    if (column.getIsNull()) {
        row.put("isNull", "1");
    } else {
        row.put("isNull", "0");
    }
    row.put("index", Integer.toString(column.getIndex()));
    row.put("mysqlType", column.getMysqlType());
    row.put("columnName", column.getName());
    if (column.getIsNull()) {
        row.put("columnValue", null);
    } else {
        row.put("columnValue", column.getValue());
    }
    return row;
}
 

public void delete(List<CanalEntry.Column> data, String schemaName, String tableName) {
    DBObject obj = DBConvertUtil.columnToJson(data);
    logger.debug("delete：{}", obj.toString());
    if (schemaName.equals("order")) {
        logger.info("not support delete：{}.{}:{}", schemaName, tableName, obj);
    } else {
        deleteData(schemaName, tableName, obj);
    }
}
 

/**
 * binlog行数据列转换成DBObject
 *
 * @param columns
 * @return
 */
public static BasicDBObject columnToJson(List<CanalEntry.Column> columns) {
    BasicDBObject obj = new BasicDBObject();
    for (CanalEntry.Column column : columns) {
        Object value = dataTypeConvert(column.getMysqlType(), column.getValue());
        obj.put(column.getName(), value);
    }
    return obj;
}
 

@Override
public List<CanalEntry.Column> getColumnList(int esEventType, CanalEntry.RowData rowData) {

    List<CanalEntry.Column> columnList;
    if (esEventType == ElasticsearchMetadata.DELETE) {
        columnList = rowData.getBeforeColumnsList();
    } else {
        columnList = rowData.getAfterColumnsList();
    }

    return columnList;

}
 

private ElasticsearchMetadata.EsEntry getElasticsearchMetadata(CanalEntry.Entry entry) throws InvalidProtocolBufferException {

        final String database = entry.getHeader().getSchemaName(); // => index
        final String table = entry.getHeader().getTableName();// => type
        final CanalEntry.RowChange change = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
        final List<CanalEntry.RowData> rowDataList = change.getRowDatasList();

        CanalEntry.EventType eventType = entry.getHeader().getEventType();
        final int esEventType = esAdapter.getEsEventType(eventType);


        EsRowDataArrayList esRowDataList = TotoroObjectPool.esRowDataArrayList();

        for (CanalEntry.RowData rowData : rowDataList) {
            List<CanalEntry.Column> columnList = esAdapter.getColumnList(esEventType, rowData);
            ElasticsearchMetadata.EsRowData esRowData = TotoroObjectPool.esRowData();
            EsColumnHashMap columnMap = TotoroObjectPool.esColumnHashMap();
            columnList.forEach(column -> columnMap.put(column.getName(), column.getValue()));
            esRowData.setRowData(columnMap);
            esRowData.setIdColumn(esAdapter.getEsIdColumn(database, table));//获取es对应的id Column
            esRowDataList.add(esRowData);
        }

        ElasticsearchMetadata.EsEntry esEntry = TotoroObjectPool.esEntry();

        esEntry.setIndex(database)
                .setType(table)
                .setEsRowDatas(esRowDataList)
                .setEventType(esEventType);

        return esEntry;
    }
 

protected void printColumn(List<CanalEntry.Column> columns) {
    for (CanalEntry.Column column : columns) {
        StringBuilder builder = new StringBuilder();
        builder.append(column.getName() + " : " + column.getValue());
        builder.append("    type=" + column.getMysqlType());
        if (column.getUpdated()) {
            builder.append("    update=" + column.getUpdated());
        }
        builder.append(SEP);
        logger.info(builder.toString());
    }
}
 

private Map<String,Object> processColumnList(List<CanalEntry.Column> columnList) {
    Map<String,Object> map = new HashMap<>();
    for (CanalEntry.Column column : columnList) {
        map.put(column.getName(), column.getValue());
    }
    return map;
}
 

@Override
public int processFullPullerMessage(Map<String, List<IGenericMessage>> map, IGenericMessage obj) throws
        IOException {
    MysqlGenericMessage message = (MysqlGenericMessage) obj;
    CanalEntry.Entry entry = message.getEntry();

    CanalEntry.EventType eventType = entry.getHeader().getEventType();
    if (eventType != CanalEntry.EventType.INSERT) {
        //skip it.
        logger.info("Skipped a FULL_PULL_REQUESTS message which is not INSERT Type! :" + eventType.toString());
        return -1;
    }

    CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
    List<CanalEntry.RowData> dataList = rowChange.getRowDatasList();
    if (dataList.size() != 1) {
        throw new RuntimeException(String.format("解压FULL_PULL_REQUESTS 发现 %d 条bach数据，应该只有一条", dataList.size()));
    }

    //为了向下兼容，使用PULL_REMARK 保存dsName
    String dsName = null;
    String schema = null;
    String table = null;
    List<CanalEntry.Column> columns = dataList.get(0).getAfterColumnsList();
    for (CanalEntry.Column column : columns) {
        if (column.getName().equalsIgnoreCase("PULL_REMARK")) {
            dsName = column.getValue();
        } else if (column.getName().equalsIgnoreCase("SCHEMA_NAME")) {
            schema = column.getValue();
        } else if (column.getName().equalsIgnoreCase("TABLE_NAME")) {
            table = column.getValue();
        }
    }

    if (dsName == null || schema == null || table == null) {
        throw new RuntimeException("解压FULL_PULL_REQUESTS 发现 dsName 或 schema 或 table为空.");
    }

    if (!dsName.equalsIgnoreCase(dsInfo.getDbSourceName())) {
        logger.info("Skipped other datasource FULL_PULL_REQUESTS! : {}.{}.{}", dsName, schema, table);
        return -1;
    }

    logger.info(String.format("Get FULL_PULL_REQUESTS message : %s.%s.%s", dsName, schema, table));

    //单独发送一条 拉全量的消息
    List<IGenericMessage> subList = new ArrayList<>();
    subList.add(message);
    map.put(schema, subList);
    return 0;
}
 

@Override
    public int processHeartBeatMessage(Map<String, List<IGenericMessage>> map, IGenericMessage obj) throws
            IOException {
        MysqlGenericMessage message = (MysqlGenericMessage) obj;
        CanalEntry.Entry entry = message.getEntry();

        CanalEntry.EventType eventType = entry.getHeader().getEventType();
        if (eventType != CanalEntry.EventType.INSERT) {
            //skip it.
            logger.info("Skipped a DB_HEARTBEAT_MONITOR message which is not INSERT Type! :" + eventType.toString());
            return -1;
        }

        CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
        List<CanalEntry.RowData> dataList = rowChange.getRowDatasList();
        /**
         * 以前代码dataList.size() 必须是1条数据，发现生产中有奇怪数据，心跳表数据居然不止一个心跳数据，推测是 insert ... select ...
         * 这种情况我们不处理，打算直接将这个心跳包抛弃。
         */
//        if (dataList.size() != 1) {
//            throw new RuntimeException(String.format("DB_HEARTBEAT_MONITOR 发现 %d 条bach数据，应该只有一条", dataList.size()));
//        }
        if (dataList.size() != 1) {
            logger.error(String.format("Skipped a DB_HEARTBEAT_MONITOR message. DB_HEARTBEAT_MONITOR 发现 %d 条bach数据，应该只有一条, 语句是%s",
                    dataList.size(), rowChange.getSql()));
            return -1;
        }

        String dsName = null;
        String schemaName = null;
        String tableName = null;
        String packetJson = null;
        List<CanalEntry.Column> columns = dataList.get(0).getAfterColumnsList();
        for (CanalEntry.Column column : columns) {
            if (column.getName().equalsIgnoreCase("DS_NAME")) {
                dsName = column.getValue();
            } else if (column.getName().equalsIgnoreCase("SCHEMA_NAME")) {
                schemaName = column.getValue();
            } else if (column.getName().equalsIgnoreCase("TABLE_NAME")) {
                tableName = column.getValue();
            } else if (column.getName().equalsIgnoreCase("PACKET"))
                packetJson = column.getValue();
        }

        if (dsName == null || schemaName == null || tableName == null || packetJson == null) {
            throw new RuntimeException("DB_HEARTBEAT_MONITOR 发现 dsName 或 schema 或 table, 或 packetJson 为空.");
        }

        if (!dsName.equalsIgnoreCase(dsInfo.getDbSourceName())) {
            logger.info("Skipped other datasource HeartBeat! : {}.{}.{}", dsName, schemaName, tableName);
            return -1;
        }

        logger.debug(String.format("Get DB_HEARTBEAT_MONITOR message : %s.%s, packetJson: %s", schemaName, tableName, packetJson));
        if (packetJson.indexOf("checkpoint") >= 0) {
            logger.debug(String.format("Get DB_HEARTBEAT_MONITOR message, prepare set stat message. "));
            HeartBeatPacket packet = HeartBeatPacket.parse(packetJson);
            statMeter(schemaName, tableName, packet.getTime(), packet.getTxtime());
        }

        List<IGenericMessage> subList = map.get(schemaName);
        if (subList != null) {
            subList.add(message);
        } else {
            subList = new ArrayList<>();
            subList.add(message);
            map.put(schemaName, subList);
        }
        return 0;
    }
 

List<CanalEntry.Column> getColumnList(int esEventType, CanalEntry.RowData rowData);