com.alibaba.otter.canal.protocol.position.Position Java Examples

The following examples show how to use com.alibaba.otter.canal.protocol.position.Position. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: MemoryEventStoreMemBatchTest.java    From canal-1.1.3 with Apache License 2.0 6 votes vote down vote up 
@Test
public void testOnePutOneGet() {
    MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
    eventStore.setBatchMode(BatchMode.MEMSIZE);
    eventStore.start();

    boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L));
    Assert.assertTrue(result);

    Position position = eventStore.getFirstPosition();
    Events<Event> entrys = eventStore.tryGet(position, 1);
    Assert.assertTrue(entrys.getEvents().size() == 1);
    Assert.assertEquals(position, entrys.getPositionRange().getStart());
    Assert.assertEquals(position, entrys.getPositionRange().getEnd());

    eventStore.stop();
}
Example #2
Source File: MemoryEventStoreMemBatchTest.java    From canal with Apache License 2.0 6 votes vote down vote up 
@Test
public void testOnePutOneGet() {
    MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
    eventStore.setBatchMode(BatchMode.MEMSIZE);
    eventStore.start();

    boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L));
    Assert.assertTrue(result);

    Position position = eventStore.getFirstPosition();
    Events<Event> entrys = eventStore.tryGet(position, 1);
    Assert.assertTrue(entrys.getEvents().size() == 1);
    Assert.assertEquals(position, entrys.getPositionRange().getStart());
    Assert.assertEquals(position, entrys.getPositionRange().getEnd());

    eventStore.stop();
}
Example #3
Source File: MemoryEventStorePutAndGetTest.java    From canal with Apache License 2.0 6 votes vote down vote up 
@Test
public void testOnePutOneGet() {
    MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
    eventStore.start();

    boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L));
    Assert.assertTrue(result);

    Position position = eventStore.getFirstPosition();
    Events<Event> entrys = eventStore.tryGet(position, 1);
    Assert.assertTrue(entrys.getEvents().size() == 1);
    Assert.assertEquals(position, entrys.getPositionRange().getStart());
    Assert.assertEquals(position, entrys.getPositionRange().getEnd());

    eventStore.stop();
}
Example #4
Source File: FileMixedMetaManager.java    From canal-1.1.3 with Apache License 2.0 6 votes vote down vote up 
private Position loadCursor(String destination, ClientIdentity clientIdentity) {
    FileMetaInstanceData data = loadDataFromFile(dataFileCaches.get(destination));
    if (data == null) {
        return null;
    }

    List<FileMetaClientIdentityData> clientDatas = data.getClientDatas();
    if (clientDatas == null) {
        return null;
    }

    for (FileMetaClientIdentityData clientData : clientDatas) {
        if (clientData.getClientIdentity() != null && clientData.getClientIdentity().equals(clientIdentity)) {
            return clientData.getCursor();
        }
    }

    return null;
}
Example #5
Source File: FileMixedMetaManager.java    From canal with Apache License 2.0 6 votes vote down vote up 
private Position loadCursor(String destination, ClientIdentity clientIdentity) {
    FileMetaInstanceData data = loadDataFromFile(dataFileCaches.get(destination));
    if (data == null) {
        return null;
    }

    List<FileMetaClientIdentityData> clientDatas = data.getClientDatas();
    if (clientDatas == null) {
        return null;
    }

    for (FileMetaClientIdentityData clientData : clientDatas) {
        if (clientData.getClientIdentity() != null && clientData.getClientIdentity().equals(clientIdentity)) {
            return clientData.getCursor();
        }
    }

    return null;
}
Example #6
Source File: MemoryEventStoreWithBuffer.java    From canal-1.1.3 with Apache License 2.0 6 votes vote down vote up 
public Events<Event> get(Position start, int batchSize) throws InterruptedException, CanalStoreException {
    final ReentrantLock lock = this.lock;
    lock.lockInterruptibly();
    try {
        try {
            while (!checkUnGetSlotAt((LogPosition) start, batchSize))
                notEmpty.await();
        } catch (InterruptedException ie) {
            notEmpty.signal(); // propagate to non-interrupted thread
            throw ie;
        }

        return doGet(start, batchSize);
    } finally {
        lock.unlock();
    }
}
Example #7
Source File: AbstractMetaManagerTest.java    From canal-1.1.3 with Apache License 2.0 6 votes vote down vote up 
public Position doCursorTest(CanalMetaManager metaManager) {
    metaManager.subscribe(clientIdentity);

    Position position1 = metaManager.getCursor(clientIdentity);
    Assert.assertNull(position1);

    PositionRange range = buildRange(1);

    metaManager.updateCursor(clientIdentity, range.getStart());
    Position position2 = metaManager.getCursor(clientIdentity);
    Assert.assertEquals(range.getStart(), position2);

    metaManager.updateCursor(clientIdentity, range.getEnd());
    Position position3 = metaManager.getCursor(clientIdentity);
    Assert.assertEquals(range.getEnd(), position3);

    return position3;
}
Example #8
Source File: PeriodMixedMetaManagerTest.java    From canal-1.1.3 with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCursorAll() {
    PeriodMixedMetaManager metaManager = new PeriodMixedMetaManager();

    ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
    zooKeeperMetaManager.setZkClientx(zkclientx);

    metaManager.setZooKeeperMetaManager(zooKeeperMetaManager);
    metaManager.start();
    Position lastPosition = doCursorTest(metaManager);

    sleep(1000L);
    // 重新构建一次，能获得上一次zk上的记录
    PeriodMixedMetaManager metaManager2 = new PeriodMixedMetaManager();
    metaManager2.setZooKeeperMetaManager(zooKeeperMetaManager);
    metaManager2.start();

    Position position = metaManager2.getCursor(clientIdentity);
    Assert.assertEquals(position, lastPosition);
    metaManager.stop();
}
Example #9
Source File: FileMixedMetaManagerTest.java    From canal-1.1.3 with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCursorAll() {
    FileMixedMetaManager metaManager = new FileMixedMetaManager();
    metaManager.setDataDirByFile(dataDir);
    metaManager.setPeriod(100);
    metaManager.start();

    Position lastPosition = doCursorTest(metaManager);

    sleep(1000L);
    // 重新构建一次，能获得上一次zk上的记录
    FileMixedMetaManager metaManager2 = new FileMixedMetaManager();
    metaManager2.setDataDirByFile(dataDir);
    metaManager2.setPeriod(100);
    metaManager2.start();

    Position position = metaManager2.getCursor(clientIdentity);
    Assert.assertEquals(position, lastPosition);
    metaManager.stop();
}
Example #10
Source File: MixedMetaManagerTest.java    From canal with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCursorAll() {
    MixedMetaManager metaManager = new MixedMetaManager();

    ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
    zooKeeperMetaManager.setZkClientx(zkclientx);

    metaManager.setZooKeeperMetaManager(zooKeeperMetaManager);
    metaManager.start();
    Position lastPosition = doCursorTest(metaManager);

    sleep(1000L);
    // 重新构建一次，能获得上一次zk上的记录
    MixedMetaManager metaManager2 = new MixedMetaManager();
    metaManager2.setZooKeeperMetaManager(zooKeeperMetaManager);
    metaManager2.start();

    Position position = metaManager2.getCursor(clientIdentity);
    Assert.assertEquals(position, lastPosition);
    metaManager.stop();
}
Example #11
Source File: AbstractCanalStoreScavenge.java    From canal with Apache License 2.0 6 votes vote down vote up 
/**
 * 找出该destination中可被清理掉的position位置
 * 
 * @param destination
 */
private Position getLatestAckPosition(String destination) {
    List<ClientIdentity> clientIdentitys = canalMetaManager.listAllSubscribeInfo(destination);
    LogPosition result = null;
    if (!CollectionUtils.isEmpty(clientIdentitys)) {
        // 尝试找到一个最小的logPosition
        for (ClientIdentity clientIdentity : clientIdentitys) {
            LogPosition position = (LogPosition) canalMetaManager.getCursor(clientIdentity);
            if (position == null) {
                continue;
            }

            if (result == null) {
                result = position;
            } else {
                result = min(result, position);
            }
        }
    }

    return result;
}
Example #12
Source File: CanalServerWithEmbedded.java    From canal-1.1.3 with Apache License 2.0 6 votes vote down vote up 
/**
 * 根据不同的参数，选择不同的方式获取数据
 */
private Events<Event> getEvents(CanalEventStore eventStore, Position start, int batchSize, Long timeout,
                                TimeUnit unit) {
    if (timeout == null) {
        return eventStore.tryGet(start, batchSize);
    } else {
        try {
            if (timeout <= 0) {
                return eventStore.get(start, batchSize);
            } else {
                return eventStore.get(start, batchSize, timeout, unit);
            }
        } catch (Exception e) {
            throw new CanalServerException(e);
        }
    }
}
Example #13
Source File: FileMixedMetaManager.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
private void flushDataToFile(String destination, File dataFile) {
    FileMetaInstanceData data = new FileMetaInstanceData();
    if (destinations.containsKey(destination)) {
        synchronized (destination.intern()) { // 基于destination控制一下并发更新
            data.setDestination(destination);

            List<FileMetaClientIdentityData> clientDatas = Lists.newArrayList();
            List<ClientIdentity> clientIdentitys = destinations.get(destination);
            for (ClientIdentity clientIdentity : clientIdentitys) {
                FileMetaClientIdentityData clientData = new FileMetaClientIdentityData();
                clientData.setClientIdentity(clientIdentity);
                Position position = cursors.get(clientIdentity);
                if (position != null && position != nullCursor) {
                    clientData.setCursor((LogPosition) position);
                }

                clientDatas.add(clientData);
            }

            data.setClientDatas(clientDatas);
        }

        String json = JsonUtils.marshalToString(data);
        try {
            FileUtils.writeStringToFile(dataFile, json);
        } catch (IOException e) {
            throw new CanalMetaManagerException(e);
        }
    }
}
Example #14
Source File: FileMixedMetaManager.java    From canal with Apache License 2.0 5 votes vote down vote up 
private void flushDataToFile(String destination, File dataFile) {
    FileMetaInstanceData data = new FileMetaInstanceData();
    if (destinations.containsKey(destination)) {
        synchronized (destination.intern()) { // 基于destination控制一下并发更新
            data.setDestination(destination);

            List<FileMetaClientIdentityData> clientDatas = Lists.newArrayList();
            List<ClientIdentity> clientIdentitys = destinations.get(destination);
            for (ClientIdentity clientIdentity : clientIdentitys) {
                FileMetaClientIdentityData clientData = new FileMetaClientIdentityData();
                clientData.setClientIdentity(clientIdentity);
                Position position = cursors.get(clientIdentity);
                if (position != null && position != nullCursor) {
                    clientData.setCursor((LogPosition) position);
                }

                clientDatas.add(clientData);
            }

            data.setClientDatas(clientDatas);
        }

        String json = JsonUtils.marshalToString(data);
        try {
            FileUtils.writeStringToFile(dataFile, json);
        } catch (IOException e) {
            throw new CanalMetaManagerException(e);
        }
    }
}
Example #15
Source File: MemoryEventStoreMemBatchTest.java    From canal with Apache License 2.0 5 votes vote down vote up 
@Test
public void testFullPutBatchGet() {
    int bufferSize = 16;
    MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
    eventStore.setBufferSize(bufferSize);
    eventStore.setBatchMode(BatchMode.MEMSIZE);
    eventStore.start();

    for (int i = 0; i < bufferSize; i++) {
        boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + i));
        sleep(100L);
        Assert.assertTrue(result);
    }

    Position first = eventStore.getFirstPosition();
    Position lastest = eventStore.getLatestPosition();
    Assert.assertEquals(first, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L)));
    Assert.assertEquals(lastest, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L + bufferSize - 1)));

    System.out.println("start get");
    Events<Event> entrys1 = eventStore.tryGet(first, bufferSize);
    System.out.println("first get size : " + entrys1.getEvents().size());

    Assert.assertTrue(entrys1.getEvents().size() == bufferSize);
    Assert.assertEquals(first, entrys1.getPositionRange().getStart());
    Assert.assertEquals(lastest, entrys1.getPositionRange().getEnd());

    Assert.assertEquals(first, CanalEventUtils.createPosition(entrys1.getEvents().get(0)));
    Assert.assertEquals(lastest, CanalEventUtils.createPosition(entrys1.getEvents().get(bufferSize - 1)));
    eventStore.stop();
}
Example #16
Source File: PeriodMixedMetaManager.java    From canal with Apache License 2.0 5 votes vote down vote up 
public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
    Position position = super.getCursor(clientIdentity);
    if (position == nullCursor) {
        return null;
    } else {
        return position;
    }
}
Example #17
Source File: PeriodMixedMetaManager.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
    Position position = super.getCursor(clientIdentity);
    if (position == nullCursor) {
        return null;
    } else {
        return position;
    }
}
Example #18
Source File: MemoryEventStoreWithBuffer.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
public Events<Event> tryGet(Position start, int batchSize) throws CanalStoreException {
    final ReentrantLock lock = this.lock;
    lock.lock();
    try {
        return doGet(start, batchSize);
    } finally {
        lock.unlock();
    }
}
Example #19
Source File: MemoryEventStoreWithBuffer.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
public Events<Event> get(Position start, int batchSize, long timeout, TimeUnit unit) throws InterruptedException,
                                                                                    CanalStoreException {
    long nanos = unit.toNanos(timeout);
    final ReentrantLock lock = this.lock;
    lock.lockInterruptibly();
    try {
        for (;;) {
            if (checkUnGetSlotAt((LogPosition) start, batchSize)) {
                return doGet(start, batchSize);
            }

            if (nanos <= 0) {
                // 如果时间到了，有多少取多少
                return doGet(start, batchSize);
            }

            try {
                nanos = notEmpty.awaitNanos(nanos);
            } catch (InterruptedException ie) {
                notEmpty.signal(); // propagate to non-interrupted thread
                throw ie;
            }

        }
    } finally {
        lock.unlock();
    }
}
Example #20
Source File: MemoryEventStorePutAndGetTest.java    From canal with Apache License 2.0 5 votes vote down vote up 
@Test
public void testFullPutBatchGet() {
    int bufferSize = 16;
    MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
    eventStore.setBufferSize(bufferSize);
    eventStore.start();

    for (int i = 0; i < bufferSize; i++) {
        boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + i));
        sleep(100L);
        Assert.assertTrue(result);
    }

    Position first = eventStore.getFirstPosition();
    Position lastest = eventStore.getLatestPosition();
    Assert.assertEquals(first, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L)));
    Assert.assertEquals(lastest, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L + bufferSize - 1)));

    System.out.println("start get");
    Events<Event> entrys1 = eventStore.tryGet(first, bufferSize);
    System.out.println("first get size : " + entrys1.getEvents().size());

    Assert.assertTrue(entrys1.getEvents().size() == bufferSize);
    Assert.assertEquals(first, entrys1.getPositionRange().getStart());
    Assert.assertEquals(lastest, entrys1.getPositionRange().getEnd());

    Assert.assertEquals(first, CanalEventUtils.createPosition(entrys1.getEvents().get(0)));
    Assert.assertEquals(lastest, CanalEventUtils.createPosition(entrys1.getEvents().get(bufferSize - 1)));
    eventStore.stop();
}
Example #21
Source File: MemoryEventStoreWithBuffer.java    From canal with Apache License 2.0 5 votes vote down vote up 
public Events<Event> get(Position start, int batchSize, long timeout, TimeUnit unit) throws InterruptedException,
                                                                                    CanalStoreException {
    long nanos = unit.toNanos(timeout);
    final ReentrantLock lock = this.lock;
    lock.lockInterruptibly();
    try {
        for (;;) {
            if (checkUnGetSlotAt((LogPosition) start, batchSize)) {
                return doGet(start, batchSize);
            }

            if (nanos <= 0) {
                // 如果时间到了，有多少取多少
                return doGet(start, batchSize);
            }

            try {
                nanos = notEmpty.awaitNanos(nanos);
            } catch (InterruptedException ie) {
                notEmpty.signal(); // propagate to non-interrupted thread
                throw ie;
            }

        }
    } finally {
        lock.unlock();
    }
}
Example #22
Source File: MixedMetaManager.java    From canal with Apache License 2.0 5 votes vote down vote up 
@Override
public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
    Position position = super.getCursor(clientIdentity);
    if (position == nullCursor) {
        return null;
    } else {
        return position;
    }
}
Example #23
Source File: MemoryEventStorePutAndGetTest.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
@Test
public void testFullPutBatchGet() {
    int bufferSize = 16;
    MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
    eventStore.setBufferSize(bufferSize);
    eventStore.start();

    for (int i = 0; i < bufferSize; i++) {
        boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + i));
        sleep(100L);
        Assert.assertTrue(result);
    }

    Position first = eventStore.getFirstPosition();
    Position lastest = eventStore.getLatestPosition();
    Assert.assertEquals(first, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L)));
    Assert.assertEquals(lastest, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L + bufferSize - 1)));

    System.out.println("start get");
    Events<Event> entrys1 = eventStore.tryGet(first, bufferSize);
    System.out.println("first get size : " + entrys1.getEvents().size());

    Assert.assertTrue(entrys1.getEvents().size() == bufferSize);
    Assert.assertEquals(first, entrys1.getPositionRange().getStart());
    Assert.assertEquals(lastest, entrys1.getPositionRange().getEnd());

    Assert.assertEquals(first, CanalEventUtils.createPosition(entrys1.getEvents().get(0)));
    Assert.assertEquals(lastest, CanalEventUtils.createPosition(entrys1.getEvents().get(bufferSize - 1)));
    eventStore.stop();
}
Example #24
Source File: FileMixedMetaManager.java    From canal with Apache License 2.0 5 votes vote down vote up 
public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
    Position position = super.getCursor(clientIdentity);
    if (position == nullCursor) {
        return null;
    } else {
        return position;
    }
}
Example #25
Source File: MemoryEventStoreMemBatchTest.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
@Test
public void testFullPutBatchGet() {
    int bufferSize = 16;
    MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
    eventStore.setBufferSize(bufferSize);
    eventStore.setBatchMode(BatchMode.MEMSIZE);
    eventStore.start();

    for (int i = 0; i < bufferSize; i++) {
        boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + i));
        sleep(100L);
        Assert.assertTrue(result);
    }

    Position first = eventStore.getFirstPosition();
    Position lastest = eventStore.getLatestPosition();
    Assert.assertEquals(first, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L)));
    Assert.assertEquals(lastest, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L + bufferSize - 1)));

    System.out.println("start get");
    Events<Event> entrys1 = eventStore.tryGet(first, bufferSize);
    System.out.println("first get size : " + entrys1.getEvents().size());

    Assert.assertTrue(entrys1.getEvents().size() == bufferSize);
    Assert.assertEquals(first, entrys1.getPositionRange().getStart());
    Assert.assertEquals(lastest, entrys1.getPositionRange().getEnd());

    Assert.assertEquals(first, CanalEventUtils.createPosition(entrys1.getEvents().get(0)));
    Assert.assertEquals(lastest, CanalEventUtils.createPosition(entrys1.getEvents().get(bufferSize - 1)));
    eventStore.stop();
}
Example #26
Source File: MixedMetaManager.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
public void updateCursor(final ClientIdentity clientIdentity, final Position position)
                                                                                      throws CanalMetaManagerException {
    super.updateCursor(clientIdentity, position);

    // 异步刷新
    executor.submit(new Runnable() {

        public void run() {
            zooKeeperMetaManager.updateCursor(clientIdentity, position);
        }
    });
}
Example #27
Source File: ZooKeeperMetaManager.java    From canal with Apache License 2.0 5 votes vote down vote up 
public void updateCursor(ClientIdentity clientIdentity, Position position) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils.getCursorPath(clientIdentity.getDestination(), clientIdentity.getClientId());
    byte[] data = JsonUtils.marshalToByte(position, SerializerFeature.WriteClassName);
    try {
        zkClientx.writeData(path, data);
    } catch (ZkNoNodeException e) {
        zkClientx.createPersistent(path, data, true);// 第一次节点不存在，则尝试重建
    }
}
Example #28
Source File: MixedMetaManager.java    From canal with Apache License 2.0 5 votes vote down vote up 
public void updateCursor(final ClientIdentity clientIdentity, final Position position)
                                                                                      throws CanalMetaManagerException {
    super.updateCursor(clientIdentity, position);

    // 异步刷新
    executor.submit(new Runnable() {

        public void run() {
            zooKeeperMetaManager.updateCursor(clientIdentity, position);
        }
    });
}
Example #29
Source File: MemoryEventStoreWithBuffer.java    From canal with Apache License 2.0 5 votes vote down vote up 
public Events<Event> tryGet(Position start, int batchSize) throws CanalStoreException {
    final ReentrantLock lock = this.lock;
    lock.lock();
    try {
        return doGet(start, batchSize);
    } finally {
        lock.unlock();
    }
}
Example #30
Source File: CanalReaderMetaManager.java    From DataLink with Apache License 2.0 5 votes vote down vote up 
@Override
public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
    Position position = super.getCursor(clientIdentity);
    if (position == nullCursor) {
        return null;
    } else {
        return position;
    }
}