com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils Java Examples

public PositionRange getFirstBatch(ClientIdentity clientIdentity) {
    String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
        clientIdentity.getClientId());
    List<String> nodes = null;
    try {
        nodes = zkClientx.getChildren(path);
    } catch (ZkNoNodeException e) {
        // ignore
    }

    if (CollectionUtils.isEmpty(nodes)) {
        return null;
    }
    // 找到最小的Id
    ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
    for (String batchIdString : nodes) {
        batchIds.add(Long.valueOf(batchIdString));
    }
    Long minBatchId = Collections.min(batchIds);
    PositionRange result = getBatch(clientIdentity, minBatchId);
    if (result == null) { // 出现为null，说明zk节点有变化，重新获取
        return getFirstBatch(clientIdentity);
    } else {
        return result;
    }
}
 

public PositionRange getLastestBatch(ClientIdentity clientIdentity) {
    String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
        clientIdentity.getClientId());
    List<String> nodes = null;
    try {
        nodes = zkClientx.getChildren(path);
    } catch (ZkNoNodeException e) {
        // ignore
    }

    if (CollectionUtils.isEmpty(nodes)) {
        return null;
    }
    // 找到最大的Id
    ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
    for (String batchIdString : nodes) {
        batchIds.add(Long.valueOf(batchIdString));
    }
    Long maxBatchId = Collections.max(batchIds);
    PositionRange result = getBatch(clientIdentity, maxBatchId);
    if (result == null) { // 出现为null，说明zk节点有变化，重新获取
        return getLastestBatch(clientIdentity);
    } else {
        return result;
    }
}
 

public synchronized void start() {
    super.start();
    try {
        processStart();
        if (zkClient != null) {
            // 如果需要尽可能释放instance资源，不需要监听running节点，不然即使stop了这台机器，另一台机器立马会start
            String path = ZookeeperPathUtils.getDestinationServerRunning(destination);
            zkClient.subscribeDataChanges(path, dataListener);

            initRunning();
        } else {
            processActiveEnter();// 没有zk，直接启动
        }
    } catch (Exception e) {
        logger.error("start failed", e);
        // 没有正常启动，重置一下状态，避免干扰下一次start
        stop();
    }

}
 

public PositionRange getLastestBatch(ClientIdentity clientIdentity) {
    String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
        clientIdentity.getClientId());
    List<String> nodes = null;
    try {
        nodes = zkClientx.getChildren(path);
    } catch (ZkNoNodeException e) {
        // ignore
    }

    if (CollectionUtils.isEmpty(nodes)) {
        return null;
    }
    // 找到最大的Id
    ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
    for (String batchIdString : nodes) {
        batchIds.add(Long.valueOf(batchIdString));
    }
    Long maxBatchId = Collections.max(batchIds);
    PositionRange result = getBatch(clientIdentity, maxBatchId);
    if (result == null) { // 出现为null，说明zk节点有变化，重新获取
        return getLastestBatch(clientIdentity);
    } else {
        return result;
    }
}
 

public PositionRange getFirstBatch(ClientIdentity clientIdentity) {
    String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
        clientIdentity.getClientId());
    List<String> nodes = null;
    try {
        nodes = zkClientx.getChildren(path);
    } catch (ZkNoNodeException e) {
        // ignore
    }

    if (CollectionUtils.isEmpty(nodes)) {
        return null;
    }
    // 找到最小的Id
    ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
    for (String batchIdString : nodes) {
        batchIds.add(Long.valueOf(batchIdString));
    }
    Long minBatchId = Collections.min(batchIds);
    PositionRange result = getBatch(clientIdentity, minBatchId);
    if (result == null) { // 出现为null，说明zk节点有变化，重新获取
        return getFirstBatch(clientIdentity);
    } else {
        return result;
    }
}
 

public synchronized void stop() {
    super.stop();

    if (zkClient != null) {
        String path = ZookeeperPathUtils.getDestinationServerRunning(destination);
        zkClient.unsubscribeDataChanges(path, dataListener);

        releaseRunning(); // 尝试一下release
    } else {
        processActiveExit(); // 没有zk，直接启动
    }
    processStop();
}
 

public void start() {
    super.start();

    String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination, clientData.getClientId());
    zkClient.subscribeDataChanges(path, dataListener);
    initRunning();
}
 

public void stop() {
    super.stop();

    String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination, clientData.getClientId());
    zkClient.unsubscribeDataChanges(path, dataListener);
    releaseRunning(); // 尝试一下release
    // Fix issue #697
    if (delayExector != null) {
        delayExector.shutdown();
    }
}
 

public boolean releaseRunning() {
    if (check()) {
        String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination, clientData.getClientId());
        zkClient.delete(path);
        mutex.set(false);
        processActiveExit();
        return true;
    }

    return false;
}
 

private void processActiveEnter() {
    if (listener != null) {
        // 触发回调，建立与server的socket链接
        InetSocketAddress connectAddress = listener.processActiveEnter();
        String address = connectAddress.getAddress().getHostAddress() + ":" + connectAddress.getPort();
        this.clientData.setAddress(address);

        String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination,
            this.clientData.getClientId());
        // 序列化
        byte[] bytes = JsonUtils.marshalToByte(clientData);
        zkClient.writeData(path, bytes);
    }
}
 

@Before
public void setUp() {
    String path = ZookeeperPathUtils.getDestinationPath(destination);
    zkclientx.deleteRecursive(path);

    zkclientx.createPersistent(ZookeeperPathUtils.getClientIdNodePath(this.destination, clientId), true);
}
 

@Override
public LogPosition getLatestIndexBy(String destination) {
    String path = ZookeeperPathUtils.getParsePath(destination);
    byte[] data = zkClientx.readData(path, true);
    if (data == null || data.length == 0) {
        return null;
    }

    return JsonUtils.unmarshalFromByte(data, LogPosition.class);
}
 

@Override
public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
    String path = ZookeeperPathUtils.getParsePath(destination);
    byte[] data = JsonUtils.marshalToByte(logPosition);
    try {
        zkClientx.writeData(path, data);
    } catch (ZkNoNodeException e) {
        zkClientx.createPersistent(path, data, true);
    }
}
 

@Override
public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
    String path = ZookeeperPathUtils.getParsePath(destination);
    byte[] data = JsonUtils.marshalToByte(logPosition);
    try {
        zkClientx.writeData(path, data);
    } catch (ZkNoNodeException e) {
        zkClientx.createPersistent(path, data, true);
    }
}
 

public void start() {
        super.start();

//        获取正在运行的client节点
        String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination, clientData.getClientId());
//        订阅节点数据变化
        zkClient.subscribeDataChanges(path, dataListener);
        initRunning();
    }
 

public PositionRange removeBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException {
    String batchsPath = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
        clientIdentity.getClientId());
    List<String> nodes = zkClientx.getChildren(batchsPath);
    if (CollectionUtils.isEmpty(nodes)) {
        // 没有batch记录
        return null;
    }

    // 找到最小的Id
    ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
    for (String batchIdString : nodes) {
        batchIds.add(Long.valueOf(batchIdString));
    }
    Long minBatchId = Collections.min(batchIds);
    if (!minBatchId.equals(batchId)) {
        // 检查一下提交的ack/rollback，必须按batchId分出去的顺序提交，否则容易出现丢数据
        throw new CanalMetaManagerException(String.format("batchId:%d is not the firstly:%d", batchId, minBatchId));
    }

    if (!batchIds.contains(batchId)) {
        // 不存在对应的batchId
        return null;
    }
    PositionRange positionRange = getBatch(clientIdentity, batchId);
    if (positionRange != null) {
        String path = ZookeeperPathUtils
            .getBatchMarkWithIdPath(clientIdentity.getDestination(), clientIdentity.getClientId(), batchId);
        zkClientx.delete(path);
    }

    return positionRange;
}
 

public void addBatch(ClientIdentity clientIdentity, PositionRange positionRange,
                     Long batchId) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils
        .getBatchMarkWithIdPath(clientIdentity.getDestination(), clientIdentity.getClientId(), batchId);
    byte[] data = JsonUtils.marshalToByte(positionRange, SerializerFeature.WriteClassName);
    zkClientx.createPersistent(path, data, true);
}
 

public PositionRange getBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils
        .getBatchMarkWithIdPath(clientIdentity.getDestination(), clientIdentity.getClientId(), batchId);
    byte[] data = zkClientx.readData(path, true);
    if (data == null) {
        return null;
    }

    PositionRange positionRange = JsonUtils.unmarshalFromByte(data, PositionRange.class);
    return positionRange;
}
 

private boolean releaseRunning() {
    if (check()) {
        String path = ZookeeperPathUtils.getDestinationServerRunning(destination);
        zkClient.delete(path);
        mutex.set(false);
        processActiveExit();
        return true;
    }

    return false;
}
 

@Before
public void setUp() {
    String path = ZookeeperPathUtils.getDestinationPath(destination);
    zkclientx.deleteRecursive(path);

    zkclientx.createPersistent(ZookeeperPathUtils.getDestinationPath(destination), true);
}
 

public synchronized void stop() {
    super.stop();

    if (zkClient != null) {
        String path = ZookeeperPathUtils.getDestinationServerRunning(destination);
        zkClient.unsubscribeDataChanges(path, dataListener);

        releaseRunning(); // 尝试一下release
    } else {
        processActiveExit(); // 没有zk，直接启动
    }
    processStop();
}
 

@Override
public void onConstruct(EnhancedInstance objInst, Object[] allArguments) {

    String clusterPath = ZookeeperPathUtils.getDestinationClusterRoot(allArguments[0].toString());
    ZkClientx zkClientx = ((ClusterNodeAccessStrategy) objInst).getZkClient();
    ContextManager.getRuntimeContext().put("currentAddress", getCurrentAddress(zkClientx.getChildren(clusterPath)));

}
 

public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils.getCursorPath(clientIdentity.getDestination(), clientIdentity.getClientId());

    byte[] data = zkClientx.readData(path, true);
    if (data == null || data.length == 0) {
        return null;
    }

    return JsonUtils.unmarshalFromByte(data, Position.class);
}
 

public void updateCursor(ClientIdentity clientIdentity, Position position) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils.getCursorPath(clientIdentity.getDestination(), clientIdentity.getClientId());
    byte[] data = JsonUtils.marshalToByte(position, SerializerFeature.WriteClassName);
    try {
        zkClientx.writeData(path, data);
    } catch (ZkNoNodeException e) {
        zkClientx.createPersistent(path, data, true);// 第一次节点不存在，则尝试重建
    }
}
 

public Long addBatch(ClientIdentity clientIdentity, PositionRange positionRange) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
        clientIdentity.getClientId());
    byte[] data = JsonUtils.marshalToByte(positionRange, SerializerFeature.WriteClassName);
    String batchPath = zkClientx
        .createPersistentSequential(path + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR, data, true);
    String batchIdString = StringUtils.substringAfterLast(batchPath, ZookeeperPathUtils.ZOOKEEPER_SEPARATOR);
    return ZookeeperPathUtils.getBatchMarkId(batchIdString);
}
 

@Before
public void setUp() {
    String path = ZookeeperPathUtils.getDestinationPath(destination);
    zkclientx.deleteRecursive(path);

    zkclientx.createPersistent(ZookeeperPathUtils.getDestinationPath(destination), true);
}
 

public PositionRange removeBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException {
    String batchsPath = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
        clientIdentity.getClientId());
    List<String> nodes = zkClientx.getChildren(batchsPath);
    if (CollectionUtils.isEmpty(nodes)) {
        // 没有batch记录
        return null;
    }

    // 找到最小的Id
    ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
    for (String batchIdString : nodes) {
        batchIds.add(Long.valueOf(batchIdString));
    }
    Long minBatchId = Collections.min(batchIds);
    if (!minBatchId.equals(batchId)) {
        // 检查一下提交的ack/rollback，必须按batchId分出去的顺序提交，否则容易出现丢数据
        throw new CanalMetaManagerException(String.format("batchId:%d is not the firstly:%d", batchId, minBatchId));
    }

    if (!batchIds.contains(batchId)) {
        // 不存在对应的batchId
        return null;
    }
    PositionRange positionRange = getBatch(clientIdentity, batchId);
    if (positionRange != null) {
        String path = ZookeeperPathUtils
            .getBatchMarkWithIdPath(clientIdentity.getDestination(), clientIdentity.getClientId(), batchId);
        zkClientx.delete(path);
    }

    return positionRange;
}
 

public PositionRange getBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils
        .getBatchMarkWithIdPath(clientIdentity.getDestination(), clientIdentity.getClientId(), batchId);
    byte[] data = zkClientx.readData(path, true);
    if (data == null) {
        return null;
    }

    PositionRange positionRange = JsonUtils.unmarshalFromByte(data, PositionRange.class);
    return positionRange;
}
 

public void clearAllBatchs(ClientIdentity clientIdentity) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
        clientIdentity.getClientId());
    List<String> batchChilds = zkClientx.getChildren(path);

    for (String batchChild : batchChilds) {
        String batchPath = path + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR + batchChild;
        zkClientx.delete(batchPath);
    }
}
 

public Map<Long, PositionRange> listAllBatchs(ClientIdentity clientIdentity) {
    String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
        clientIdentity.getClientId());
    List<String> nodes = null;
    try {
        nodes = zkClientx.getChildren(path);
    } catch (ZkNoNodeException e) {
        // ignore
    }

    if (CollectionUtils.isEmpty(nodes)) {
        return Maps.newHashMap();
    }
    // 找到最大的Id
    ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
    for (String batchIdString : nodes) {
        batchIds.add(Long.valueOf(batchIdString));
    }

    Collections.sort(batchIds); // 从小到大排序
    Map<Long, PositionRange> positionRanges = Maps.newLinkedHashMap();
    for (Long batchId : batchIds) {
        PositionRange result = getBatch(clientIdentity, batchId);
        if (result == null) {// 出现为null，说明zk节点有变化，重新获取
            return listAllBatchs(clientIdentity);
        } else {
            positionRanges.put(batchId, result);
        }
    }

    return positionRanges;
}