com.alibaba.otter.canal.common.utils.JsonUtils Java Exaples

Source File: ZooKeeperMetaManager.java From canal with Apache License 2.0

5 votes

public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils.getCursorPath(clientIdentity.getDestination(), clientIdentity.getClientId());

    byte[] data = zkClientx.readData(path, true);
    if (data == null || data.length == 0) {
        return null;
    }

    return JsonUtils.unmarshalFromByte(data, Position.class);
}

Source File: FileMixedLogPositionManager.java From canal with Apache License 2.0

5 votes

private LogPosition loadDataFromFile(File dataFile) {
    try {
        if (!dataFile.exists()) {
            return null;
        }

        String json = FileUtils.readFileToString(dataFile, charset.name());
        return JsonUtils.unmarshalFromString(json, LogPosition.class);
    } catch (IOException e) {
        throw new CanalMetaManagerException(e);
    }
}

Source File: FileMixedLogPositionManager.java From canal with Apache License 2.0

5 votes

private void flushDataToFile(String destination, File dataFile) {
    LogPosition position = memoryLogPositionManager.getLatestIndexBy(destination);
    if (position != null && position != nullPosition) {
        String json = JsonUtils.marshalToString(position);
        try {
            FileUtils.writeStringToFile(dataFile, json);
        } catch (IOException e) {
            throw new CanalMetaManagerException(e);
        }
    }
}

Source File: ZooKeeperLogPositionManager.java From canal with Apache License 2.0

5 votes

@Override
public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
    String path = ZookeeperPathUtils.getParsePath(destination);
    byte[] data = JsonUtils.marshalToByte(logPosition);
    try {
        zkClientx.writeData(path, data);
    } catch (ZkNoNodeException e) {
        zkClientx.createPersistent(path, data, true);
    }
}

Source File: ZooKeeperLogPositionManager.java From canal with Apache License 2.0

5 votes

@Override
public LogPosition getLatestIndexBy(String destination) {
    String path = ZookeeperPathUtils.getParsePath(destination);
    byte[] data = zkClientx.readData(path, true);
    if (data == null || data.length == 0) {
        return null;
    }

    return JsonUtils.unmarshalFromByte(data, LogPosition.class);
}

Source File: ClusterNodeAccessStrategy.java From canal with Apache License 2.0

5 votes

private void initRunning(Object data) {
    if (data == null) {
        return;
    }

    ServerRunningData runningData = JsonUtils.unmarshalFromByte((byte[]) data, ServerRunningData.class);
    String[] strs = StringUtils.split(runningData.getAddress(), ':');
    if (strs.length == 2) {
        runningAddress = new InetSocketAddress(strs[0], Integer.valueOf(strs[1]));
    }
}

Source File: ClientRunningMonitor.java From canal with Apache License 2.0

5 votes

private void processActiveEnter() {
    if (listener != null) {
        // 触发回调，建立与server的socket链接
        InetSocketAddress connectAddress = listener.processActiveEnter();
        String address = connectAddress.getAddress().getHostAddress() + ":" + connectAddress.getPort();
        this.clientData.setAddress(address);

        String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination,
            this.clientData.getClientId());
        // 序列化
        byte[] bytes = JsonUtils.marshalToByte(clientData);
        zkClient.writeData(path, bytes);
    }
}

Source File: FileMixedMetaManager.java From canal with Apache License 2.0

5 votes

private void flushDataToFile(String destination, File dataFile) {
    FileMetaInstanceData data = new FileMetaInstanceData();
    if (destinations.containsKey(destination)) {
        synchronized (destination.intern()) { // 基于destination控制一下并发更新
            data.setDestination(destination);

            List<FileMetaClientIdentityData> clientDatas = Lists.newArrayList();
            List<ClientIdentity> clientIdentitys = destinations.get(destination);
            for (ClientIdentity clientIdentity : clientIdentitys) {
                FileMetaClientIdentityData clientData = new FileMetaClientIdentityData();
                clientData.setClientIdentity(clientIdentity);
                Position position = cursors.get(clientIdentity);
                if (position != null && position != nullCursor) {
                    clientData.setCursor((LogPosition) position);
                }

                clientDatas.add(clientData);
            }

            data.setClientDatas(clientDatas);
        }

        String json = JsonUtils.marshalToString(data);
        try {
            FileUtils.writeStringToFile(dataFile, json);
        } catch (IOException e) {
            throw new CanalMetaManagerException(e);
        }
    }
}

Source File: FileMixedMetaManager.java From canal with Apache License 2.0

5 votes

private FileMetaInstanceData loadDataFromFile(File dataFile) {
    try {
        if (!dataFile.exists()) {
            return null;
        }

        String json = FileUtils.readFileToString(dataFile, charset.name());
        return JsonUtils.unmarshalFromString(json, FileMetaInstanceData.class);
    } catch (IOException e) {
        throw new CanalMetaManagerException(e);
    }
}

Source File: ZooKeeperMetaManager.java From canal with Apache License 2.0

5 votes

public PositionRange getBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils
        .getBatchMarkWithIdPath(clientIdentity.getDestination(), clientIdentity.getClientId(), batchId);
    byte[] data = zkClientx.readData(path, true);
    if (data == null) {
        return null;
    }

    PositionRange positionRange = JsonUtils.unmarshalFromByte(data, PositionRange.class);
    return positionRange;
}

Source File: ZooKeeperMetaManager.java From canal with Apache License 2.0

5 votes

public void addBatch(ClientIdentity clientIdentity, PositionRange positionRange,
                     Long batchId) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils
        .getBatchMarkWithIdPath(clientIdentity.getDestination(), clientIdentity.getClientId(), batchId);
    byte[] data = JsonUtils.marshalToByte(positionRange, SerializerFeature.WriteClassName);
    zkClientx.createPersistent(path, data, true);
}

Source File: ZooKeeperMetaManager.java From canal with Apache License 2.0

5 votes

public Long addBatch(ClientIdentity clientIdentity, PositionRange positionRange) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
        clientIdentity.getClientId());
    byte[] data = JsonUtils.marshalToByte(positionRange, SerializerFeature.WriteClassName);
    String batchPath = zkClientx
        .createPersistentSequential(path + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR, data, true);
    String batchIdString = StringUtils.substringAfterLast(batchPath, ZookeeperPathUtils.ZOOKEEPER_SEPARATOR);
    return ZookeeperPathUtils.getBatchMarkId(batchIdString);
}

Source File: ZooKeeperMetaManager.java From canal with Apache License 2.0

5 votes

public void updateCursor(ClientIdentity clientIdentity, Position position) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils.getCursorPath(clientIdentity.getDestination(), clientIdentity.getClientId());
    byte[] data = JsonUtils.marshalToByte(position, SerializerFeature.WriteClassName);
    try {
        zkClientx.writeData(path, data);
    } catch (ZkNoNodeException e) {
        zkClientx.createPersistent(path, data, true);// 第一次节点不存在，则尝试重建
    }
}

Source File: ZooKeeperMetaManager.java From canal-1.1.3 with Apache License 2.0

5 votes

public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils.getCursorPath(clientIdentity.getDestination(), clientIdentity.getClientId());

    byte[] data = zkClientx.readData(path, true);
    if (data == null || data.length == 0) {
        return null;
    }

    return JsonUtils.unmarshalFromByte(data, Position.class);
}

Source File: FileMixedLogPositionManager.java From canal-1.1.3 with Apache License 2.0

5 votes

private LogPosition loadDataFromFile(File dataFile) {
    try {
        if (!dataFile.exists()) {
            return null;
        }

        String json = FileUtils.readFileToString(dataFile, charset.name());
        return JsonUtils.unmarshalFromString(json, LogPosition.class);
    } catch (IOException e) {
        throw new CanalMetaManagerException(e);
    }
}

Source File: FileMixedLogPositionManager.java From canal-1.1.3 with Apache License 2.0

5 votes

private void flushDataToFile(String destination, File dataFile) {
    LogPosition position = memoryLogPositionManager.getLatestIndexBy(destination);
    if (position != null && position != nullPosition) {
        String json = JsonUtils.marshalToString(position);
        try {
            FileUtils.writeStringToFile(dataFile, json);
        } catch (IOException e) {
            throw new CanalMetaManagerException(e);
        }
    }
}

Source File: ZooKeeperLogPositionManager.java From canal-1.1.3 with Apache License 2.0

5 votes

@Override
public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
    String path = ZookeeperPathUtils.getParsePath(destination);
    byte[] data = JsonUtils.marshalToByte(logPosition);
    try {
        zkClientx.writeData(path, data);
    } catch (ZkNoNodeException e) {
        zkClientx.createPersistent(path, data, true);
    }
}

Source File: ZooKeeperLogPositionManager.java From canal-1.1.3 with Apache License 2.0

5 votes

@Override
public LogPosition getLatestIndexBy(String destination) {
    String path = ZookeeperPathUtils.getParsePath(destination);
    byte[] data = zkClientx.readData(path, true);
    if (data == null || data.length == 0) {
        return null;
    }

    return JsonUtils.unmarshalFromByte(data, LogPosition.class);
}

Source File: ClusterNodeAccessStrategy.java From canal-1.1.3 with Apache License 2.0

5 votes

private void initRunning(Object data) {
    if (data == null) {
        return;
    }

    ServerRunningData runningData = JsonUtils.unmarshalFromByte((byte[]) data, ServerRunningData.class);
    String[] strs = StringUtils.split(runningData.getAddress(), ':');
    if (strs.length == 2) {
        runningAddress = new InetSocketAddress(strs[0], Integer.valueOf(strs[1]));
    }
}

Source File: ClientRunningMonitor.java From canal-1.1.3 with Apache License 2.0

5 votes

private void processActiveEnter() {
    if (listener != null) {
        // 触发回调，建立与server的socket链接
        InetSocketAddress connectAddress = listener.processActiveEnter();
        String address = connectAddress.getAddress().getHostAddress() + ":" + connectAddress.getPort();
        this.clientData.setAddress(address);

        String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination,
            this.clientData.getClientId());
        // 序列化
        byte[] bytes = JsonUtils.marshalToByte(clientData);
        zkClient.writeData(path, bytes);
    }
}

Source File: FileMixedMetaManager.java From canal-1.1.3 with Apache License 2.0

5 votes

private void flushDataToFile(String destination, File dataFile) {
    FileMetaInstanceData data = new FileMetaInstanceData();
    if (destinations.containsKey(destination)) {
        synchronized (destination.intern()) { // 基于destination控制一下并发更新
            data.setDestination(destination);

            List<FileMetaClientIdentityData> clientDatas = Lists.newArrayList();
            List<ClientIdentity> clientIdentitys = destinations.get(destination);
            for (ClientIdentity clientIdentity : clientIdentitys) {
                FileMetaClientIdentityData clientData = new FileMetaClientIdentityData();
                clientData.setClientIdentity(clientIdentity);
                Position position = cursors.get(clientIdentity);
                if (position != null && position != nullCursor) {
                    clientData.setCursor((LogPosition) position);
                }

                clientDatas.add(clientData);
            }

            data.setClientDatas(clientDatas);
        }

        String json = JsonUtils.marshalToString(data);
        try {
            FileUtils.writeStringToFile(dataFile, json);
        } catch (IOException e) {
            throw new CanalMetaManagerException(e);
        }
    }
}

Source File: FileMixedMetaManager.java From canal-1.1.3 with Apache License 2.0

5 votes

private FileMetaInstanceData loadDataFromFile(File dataFile) {
    try {
        if (!dataFile.exists()) {
            return null;
        }

        String json = FileUtils.readFileToString(dataFile, charset.name());
        return JsonUtils.unmarshalFromString(json, FileMetaInstanceData.class);
    } catch (IOException e) {
        throw new CanalMetaManagerException(e);
    }
}

Source File: ZooKeeperMetaManager.java From canal-1.1.3 with Apache License 2.0

5 votes

public PositionRange getBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils
        .getBatchMarkWithIdPath(clientIdentity.getDestination(), clientIdentity.getClientId(), batchId);
    byte[] data = zkClientx.readData(path, true);
    if (data == null) {
        return null;
    }

    PositionRange positionRange = JsonUtils.unmarshalFromByte(data, PositionRange.class);
    return positionRange;
}

Source File: ZooKeeperMetaManager.java From canal-1.1.3 with Apache License 2.0

5 votes

public void addBatch(ClientIdentity clientIdentity, PositionRange positionRange,
                     Long batchId) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils
        .getBatchMarkWithIdPath(clientIdentity.getDestination(), clientIdentity.getClientId(), batchId);
    byte[] data = JsonUtils.marshalToByte(positionRange, SerializerFeature.WriteClassName);
    zkClientx.createPersistent(path, data, true);
}

Source File: ZooKeeperMetaManager.java From canal-1.1.3 with Apache License 2.0

5 votes

public Long addBatch(ClientIdentity clientIdentity, PositionRange positionRange) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
        clientIdentity.getClientId());
    byte[] data = JsonUtils.marshalToByte(positionRange, SerializerFeature.WriteClassName);
    String batchPath = zkClientx
        .createPersistentSequential(path + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR, data, true);
    String batchIdString = StringUtils.substringAfterLast(batchPath, ZookeeperPathUtils.ZOOKEEPER_SEPARATOR);
    return ZookeeperPathUtils.getBatchMarkId(batchIdString);
}

Source File: ZooKeeperMetaManager.java From canal-1.1.3 with Apache License 2.0

5 votes

public void updateCursor(ClientIdentity clientIdentity, Position position) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils.getCursorPath(clientIdentity.getDestination(), clientIdentity.getClientId());
    byte[] data = JsonUtils.marshalToByte(position, SerializerFeature.WriteClassName);
    try {
        zkClientx.writeData(path, data);
    } catch (ZkNoNodeException e) {
        zkClientx.createPersistent(path, data, true);// 第一次节点不存在，则尝试重建
    }
}

Source File: MysqlEventParser.java From canal with Apache License 2.0

4 votes

protected EntryPosition findStartPositionInternal(ErosaConnection connection) {
    MysqlConnection mysqlConnection = (MysqlConnection) connection;
    LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
    if (logPosition == null) {// 找不到历史成功记录
        EntryPosition entryPosition = null;
        if (masterInfo != null && mysqlConnection.getConnector().getAddress().equals(masterInfo.getAddress())) {
            entryPosition = masterPosition;
        } else if (standbyInfo != null
                   && mysqlConnection.getConnector().getAddress().equals(standbyInfo.getAddress())) {
            entryPosition = standbyPosition;
        }

        if (entryPosition == null) {
            entryPosition = findEndPositionWithMasterIdAndTimestamp(mysqlConnection); // 默认从当前最后一个位置进行消费
        }

        // 判断一下是否需要按时间订阅
        if (StringUtils.isEmpty(entryPosition.getJournalName())) {
            // 如果没有指定binlogName，尝试按照timestamp进行查找
            if (entryPosition.getTimestamp() != null && entryPosition.getTimestamp() > 0L) {
                logger.warn("prepare to find start position {}:{}:{}",
                    new Object[] { "", "", entryPosition.getTimestamp() });
                return findByStartTimeStamp(mysqlConnection, entryPosition.getTimestamp());
            } else {
                logger.warn("prepare to find start position just show master status");
                return findEndPositionWithMasterIdAndTimestamp(mysqlConnection); // 默认从当前最后一个位置进行消费
            }
        } else {
            if (entryPosition.getPosition() != null && entryPosition.getPosition() > 0L) {
                // 如果指定binlogName + offest，直接返回
                entryPosition = findPositionWithMasterIdAndTimestamp(mysqlConnection, entryPosition);
                logger.warn("prepare to find start position {}:{}:{}",
                    new Object[] { entryPosition.getJournalName(), entryPosition.getPosition(),
                            entryPosition.getTimestamp() });
                return entryPosition;
            } else {
                EntryPosition specificLogFilePosition = null;
                if (entryPosition.getTimestamp() != null && entryPosition.getTimestamp() > 0L) {
                    // 如果指定binlogName +
                    // timestamp，但没有指定对应的offest，尝试根据时间找一下offest
                    EntryPosition endPosition = findEndPosition(mysqlConnection);
                    if (endPosition != null) {
                        logger.warn("prepare to find start position {}:{}:{}",
                            new Object[] { entryPosition.getJournalName(), "", entryPosition.getTimestamp() });
                        specificLogFilePosition = findAsPerTimestampInSpecificLogFile(mysqlConnection,
                            entryPosition.getTimestamp(),
                            endPosition,
                            entryPosition.getJournalName(),
                            true);
                    }
                }

                if (specificLogFilePosition == null) {
                    // position不存在，从文件头开始
                    entryPosition.setPosition(BINLOG_START_OFFEST);
                    return entryPosition;
                } else {
                    return specificLogFilePosition;
                }
            }
        }
    } else {
        if (logPosition.getIdentity().getSourceAddress().equals(mysqlConnection.getConnector().getAddress())) {
            if (dumpErrorCountThreshold >= 0 && dumpErrorCount > dumpErrorCountThreshold) {
                // binlog定位位点失败,可能有两个原因:
                // 1. binlog位点被删除
                // 2.vip模式的mysql,发生了主备切换,判断一下serverId是否变化,针对这种模式可以发起一次基于时间戳查找合适的binlog位点
                boolean case2 = (standbyInfo == null || standbyInfo.getAddress() == null)
                                && logPosition.getPostion().getServerId() != null
                                && !logPosition.getPostion().getServerId().equals(findServerId(mysqlConnection));
                if (case2) {
                    EntryPosition findPosition = fallbackFindByStartTimestamp(logPosition, mysqlConnection);
                    dumpErrorCount = 0;
                    return findPosition;
                }

                Long timestamp = logPosition.getPostion().getTimestamp();
                if (isRdsOssMode() && (timestamp != null && timestamp > 0)) {
                    // 如果binlog位点不存在，并且属于timestamp不为空,可以返回null走到oss binlog处理
                    return null;
                }
            } else if (StringUtils.isBlank(logPosition.getPostion().getJournalName())
                    && logPosition.getPostion().getPosition() <= 0
                    && logPosition.getPostion().getTimestamp() > 0) {
                return fallbackFindByStartTimestamp(logPosition,mysqlConnection);
            }
            // 其余情况
            logger.warn("prepare to find start position just last position\n {}",
                JsonUtils.marshalToString(logPosition));
            return logPosition.getPostion();
        } else {
            // 针对切换的情况，考虑回退时间
            long newStartTimestamp = logPosition.getPostion().getTimestamp() - fallbackIntervalInSeconds * 1000;
            logger.warn("prepare to find start position by switch {}:{}:{}", new Object[] { "", "",
                    logPosition.getPostion().getTimestamp() });
            return findByStartTimeStamp(mysqlConnection, newStartTimestamp);
        }
    }
}

com.alibaba.otter.canal.common.utils.JsonUtils Java Examples