com.alibaba.otter.canal.store.model.Event Java Exaples

Source File: CanalEventUtils.java From canal-1.1.3 with Apache License 2.0

6 votes

/**
 * 根据entry创建对应的Position对象
 */
public static LogPosition createPosition(Event event) {
    EntryPosition position = new EntryPosition();
    position.setJournalName(event.getJournalName());
    position.setPosition(event.getPosition());
    position.setTimestamp(event.getExecuteTime());
    // add serverId at 2016-06-28
    position.setServerId(event.getServerId());
    // add gtid
    position.setGtid(event.getGtid());

    LogPosition logPosition = new LogPosition();
    logPosition.setPostion(position);
    logPosition.setIdentity(event.getLogIdentity());
    return logPosition;
}

Source File: MemoryEventStorePutAndGetTest.java From canal with Apache License 2.0

6 votes

@Test
public void testOnePutOneGet() {
    MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
    eventStore.start();

    boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L));
    Assert.assertTrue(result);

    Position position = eventStore.getFirstPosition();
    Events<Event> entrys = eventStore.tryGet(position, 1);
    Assert.assertTrue(entrys.getEvents().size() == 1);
    Assert.assertEquals(position, entrys.getPositionRange().getStart());
    Assert.assertEquals(position, entrys.getPositionRange().getEnd());

    eventStore.stop();
}

Source File: GroupEventSink.java From canal with Apache License 2.0

6 votes

protected boolean doSink(List<Event> events) {
    int size = events.size();
    for (int i = 0; i < events.size(); i++) {
        Event event = events.get(i);
        try {
            barrier.await(event);// 进行timeline的归并调度处理
            if (filterTransactionEntry) {
                super.doSink(Arrays.asList(event));
            } else if (i == size - 1) {
                // 针对事务数据，只有到最后一条数据都通过后，才进行sink操作，保证原子性
                // 同时批量sink，也要保证在最后一条数据释放状态之前写出数据，否则就有并发问题
                return super.doSink(events);
            }
        } catch (InterruptedException e) {
            return false;
        } finally {
            barrier.clear(event);
        }
    }

    return false;
}

Source File: DummyEventStore.java From canal with Apache License 2.0

6 votes

public boolean tryPut(List<Event> datas) throws CanalStoreException {
    System.out.println("\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
    for (Event data : datas) {

        Date date = new Date(data.getExecuteTime());
        SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
        if (data.getEntryType() == EntryType.TRANSACTIONBEGIN || data.getEntryType() == EntryType.TRANSACTIONEND) {
            // System.out.println(MessageFormat.format(messgae, new Object[]
            // { Thread.currentThread().getName(),
            // header.getLogfilename(), header.getLogfileoffset(),
            // format.format(date),
            // data.getEntry().getEntryType(), "" }));
            System.out.println(data.getEntryType());

        } else {
            System.out.println(MessageFormat.format(messgae,
                new Object[] { Thread.currentThread().getName(), data.getJournalName(),
                        String.valueOf(data.getPosition()), format.format(date) }));
        }

    }
    System.out.println("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n");
    return true;
}

Source File: MemoryEventStoreMemBatchTest.java From canal with Apache License 2.0

6 votes

@Test
public void testOnePutOneGet() {
    MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
    eventStore.setBatchMode(BatchMode.MEMSIZE);
    eventStore.start();

    boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L));
    Assert.assertTrue(result);

    Position position = eventStore.getFirstPosition();
    Events<Event> entrys = eventStore.tryGet(position, 1);
    Assert.assertTrue(entrys.getEvents().size() == 1);
    Assert.assertEquals(position, entrys.getPositionRange().getStart());
    Assert.assertEquals(position, entrys.getPositionRange().getEnd());

    eventStore.stop();
}

Source File: CanalServerWithEmbedded.java From canal with Apache License 2.0

6 votes

/**
 * 根据不同的参数，选择不同的方式获取数据
 */
private Events<Event> getEvents(CanalEventStore eventStore, Position start, int batchSize, Long timeout,
                                TimeUnit unit) {
    if (timeout == null) {
        return eventStore.tryGet(start, batchSize);
    } else {
        try {
            if (timeout <= 0) {
                return eventStore.get(start, batchSize);
            } else {
                return eventStore.get(start, batchSize, timeout, unit);
            }
        } catch (Exception e) {
            throw new CanalServerException(e);
        }
    }
}

Source File: MemoryEventStoreWithBuffer.java From canal with Apache License 2.0

6 votes

public void put(List<Event> data) throws InterruptedException, CanalStoreException {
    if (data == null || data.isEmpty()) {
        return;
    }

    final ReentrantLock lock = this.lock;
    lock.lockInterruptibly();
    try {
        try {
            while (!checkFreeSlotAt(putSequence.get() + data.size())) { // 检查是否有空位
                notFull.await(); // wait until not full
            }
        } catch (InterruptedException ie) {
            notFull.signal(); // propagate to non-interrupted thread
            throw ie;
        }
        doPut(data);
        if (Thread.interrupted()) {
            throw new InterruptedException();
        }
    } finally {
        lock.unlock();
    }
}

Source File: TimelineTransactionBarrier.java From canal-1.1.3 with Apache License 2.0

6 votes

protected boolean isPermit(Event event, long state) {
    if (txState.intValue() == 1 && inTransaction.get()) { // 如果处于事务中，直接允许通过。因为事务头已经做过判断
        return true;
    } else if (txState.intValue() == 0) {
        boolean result = super.isPermit(event, state);
        if (result) {
            // 可能第一条送过来的数据不为Begin，需要做判断处理，如果非事务，允许直接通过，比如DDL语句
            if (isTransactionBegin(event)) {
                if (txState.compareAndSet(0, 1)) {
                    inTransaction.set(true);
                    return true; // 事务允许通过
                }
            } else if (txState.compareAndSet(0, 2)) { // 非事务保护中
                // 当基于zk-cursor启动的时候，拿到的第一个Event是TransactionEnd
                return true; // DDL/DCL/TransactionEnd允许通过
            }
        }
    }

    return false;
}

Source File: DummyEventStore.java From canal with Apache License 2.0

6 votes

public boolean put(List<Event> datas, long timeout, TimeUnit unit) throws InterruptedException, CanalStoreException {
    for (Event data : datas) {
        Date date = new Date(data.getExecuteTime());
        SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
        if (data.getEntryType() == EntryType.TRANSACTIONBEGIN || data.getEntryType() == EntryType.TRANSACTIONEND) {
            // System.out.println(MessageFormat.format(messgae, new Object[]
            // { Thread.currentThread().getName(),
            // header.getLogfilename(), header.getLogfileoffset(),
            // format.format(date),
            // data.getEntry().getEntryType(), "" }));
            System.out.println(data.getEntryType());

        } else {
            System.out.println(MessageFormat.format(messgae,
                new Object[] { Thread.currentThread().getName(), data.getJournalName(),
                        String.valueOf(data.getPosition()), format.format(date) }));
        }
    }
    return true;
}

Source File: MemoryEventStoreWithBuffer.java From canal with Apache License 2.0

6 votes

public Events<Event> get(Position start, int batchSize) throws InterruptedException, CanalStoreException {
    final ReentrantLock lock = this.lock;
    lock.lockInterruptibly();
    try {
        try {
            while (!checkUnGetSlotAt((LogPosition) start, batchSize))
                notEmpty.await();
        } catch (InterruptedException ie) {
            notEmpty.signal(); // propagate to non-interrupted thread
            throw ie;
        }

        return doGet(start, batchSize);
    } finally {
        lock.unlock();
    }
}

Source File: TimelineTransactionBarrier.java From canal with Apache License 2.0

6 votes

protected boolean isPermit(Event event, long state) {
    if (txState.intValue() == 1 && inTransaction.get()) { // 如果处于事务中，直接允许通过。因为事务头已经做过判断
        return true;
    } else if (txState.intValue() == 0) {
        boolean result = super.isPermit(event, state);
        if (result) {
            // 可能第一条送过来的数据不为Begin，需要做判断处理，如果非事务，允许直接通过，比如DDL语句
            if (isTransactionBegin(event)) {
                if (txState.compareAndSet(0, 1)) {
                    inTransaction.set(true);
                    return true; // 事务允许通过
                }
            } else if (txState.compareAndSet(0, 2)) { // 非事务保护中
                // 当基于zk-cursor启动的时候，拿到的第一个Event是TransactionEnd
                return true; // DDL/DCL/TransactionEnd允许通过
            }
        }
    }

    return false;
}

Source File: FixedGroupEventSink.java From DataLink with Apache License 2.0

6 votes

protected boolean doSink(List<Event> events) {
    int size = events.size();
    for (int i = 0; i < events.size(); i++) {
        Event event = events.get(i);
        try {
            barrier.await(event);// 进行timeline的归并调度处理
            if (filterTransactionEntry) {
                super.doSink(Arrays.asList(event));
            } else if (i == size - 1) {
                // 针对事务数据，只有到最后一条数据都通过后，才进行sink操作，保证原子性
                // 同时批量sink，也要保证在最后一条数据释放状态之前写出数据，否则就有并发问题
                return super.doSink(events);
            }
        } catch (InterruptedException e) {
            return false;
        } finally {
            barrier.clear(event);
        }
    }

    return false;
}

Source File: DummyEventStore.java From canal-1.1.3 with Apache License 2.0

6 votes

public void put(List<Event> datas) throws InterruptedException, CanalStoreException {
    for (Event data : datas) {
        Date date = new Date(data.getExecuteTime());
        SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
        if (data.getEntryType() == EntryType.TRANSACTIONBEGIN || data.getEntryType() == EntryType.TRANSACTIONEND) {
            // System.out.println(MessageFormat.format(messgae, new Object[]
            // { Thread.currentThread().getName(),
            // header.getLogfilename(), header.getLogfileoffset(),
            // format.format(date),
            // data.getEntry().getEntryType(), "" }));
            System.out.println(data.getEntryType());

        } else {
            System.out.println(MessageFormat.format(messgae,
                new Object[] { Thread.currentThread().getName(), data.getJournalName(),
                        String.valueOf(data.getPosition()), format.format(date) }));
        }
    }
}

Source File: CanalServerWithEmbedded.java From canal-1.1.3 with Apache License 2.0

6 votes

/**
 * 根据不同的参数，选择不同的方式获取数据
 */
private Events<Event> getEvents(CanalEventStore eventStore, Position start, int batchSize, Long timeout,
                                TimeUnit unit) {
    if (timeout == null) {
        return eventStore.tryGet(start, batchSize);
    } else {
        try {
            if (timeout <= 0) {
                return eventStore.get(start, batchSize);
            } else {
                return eventStore.get(start, batchSize, timeout, unit);
            }
        } catch (Exception e) {
            throw new CanalServerException(e);
        }
    }
}

Source File: MemoryEventStoreMemBatchTest.java From canal-1.1.3 with Apache License 2.0

6 votes

@Test
public void testOnePutOneGet() {
    MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
    eventStore.setBatchMode(BatchMode.MEMSIZE);
    eventStore.start();

    boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L));
    Assert.assertTrue(result);

    Position position = eventStore.getFirstPosition();
    Events<Event> entrys = eventStore.tryGet(position, 1);
    Assert.assertTrue(entrys.getEvents().size() == 1);
    Assert.assertEquals(position, entrys.getPositionRange().getStart());
    Assert.assertEquals(position, entrys.getPositionRange().getEnd());

    eventStore.stop();
}

Source File: MemoryEventStorePutAndGetTest.java From canal-1.1.3 with Apache License 2.0

6 votes

@Test
public void testOnePutOneGet() {
    MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
    eventStore.start();

    boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L));
    Assert.assertTrue(result);

    Position position = eventStore.getFirstPosition();
    Events<Event> entrys = eventStore.tryGet(position, 1);
    Assert.assertTrue(entrys.getEvents().size() == 1);
    Assert.assertEquals(position, entrys.getPositionRange().getStart());
    Assert.assertEquals(position, entrys.getPositionRange().getEnd());

    eventStore.stop();
}

Source File: MemoryEventStoreWithBuffer.java From canal-1.1.3 with Apache License 2.0

6 votes

public Events<Event> get(Position start, int batchSize) throws InterruptedException, CanalStoreException {
    final ReentrantLock lock = this.lock;
    lock.lockInterruptibly();
    try {
        try {
            while (!checkUnGetSlotAt((LogPosition) start, batchSize))
                notEmpty.await();
        } catch (InterruptedException ie) {
            notEmpty.signal(); // propagate to non-interrupted thread
            throw ie;
        }

        return doGet(start, batchSize);
    } finally {
        lock.unlock();
    }
}

Source File: MemoryEventStoreWithBuffer.java From canal-1.1.3 with Apache License 2.0

6 votes

public boolean tryPut(List<Event> data) throws CanalStoreException {
    if (data == null || data.isEmpty()) {
        return true;
    }

    final ReentrantLock lock = this.lock;
    lock.lock();
    try {
        if (!checkFreeSlotAt(putSequence.get() + data.size())) {
            return false;
        } else {
            doPut(data);
            return true;
        }
    } finally {
        lock.unlock();
    }
}

Source File: MemoryEventStoreWithBuffer.java From canal-1.1.3 with Apache License 2.0

6 votes

public void put(List<Event> data) throws InterruptedException, CanalStoreException {
    if (data == null || data.isEmpty()) {
        return;
    }

    final ReentrantLock lock = this.lock;
    lock.lockInterruptibly();
    try {
        try {
            while (!checkFreeSlotAt(putSequence.get() + data.size())) { // 检查是否有空位
                notFull.await(); // wait until not full
            }
        } catch (InterruptedException ie) {
            notFull.signal(); // propagate to non-interrupted thread
            throw ie;
        }
        doPut(data);
        if (Thread.interrupted()) {
            throw new InterruptedException();
        }
    } finally {
        lock.unlock();
    }
}

Source File: CanalEventUtils.java From canal with Apache License 2.0

6 votes

/**
 * 根据entry创建对应的Position对象
 */
public static LogPosition createPosition(Event event) {
    EntryPosition position = new EntryPosition();
    position.setJournalName(event.getJournalName());
    position.setPosition(event.getPosition());
    position.setTimestamp(event.getExecuteTime());
    // add serverId at 2016-06-28
    position.setServerId(event.getServerId());
    // add gtid
    position.setGtid(event.getGtid());

    LogPosition logPosition = new LogPosition();
    logPosition.setPostion(position);
    logPosition.setIdentity(event.getLogIdentity());
    return logPosition;
}

Source File: DummyEventStore.java From canal-1.1.3 with Apache License 2.0

6 votes

public boolean tryPut(List<Event> datas) throws CanalStoreException {
    System.out.println("\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
    for (Event data : datas) {

        Date date = new Date(data.getExecuteTime());
        SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
        if (data.getEntryType() == EntryType.TRANSACTIONBEGIN || data.getEntryType() == EntryType.TRANSACTIONEND) {
            // System.out.println(MessageFormat.format(messgae, new Object[]
            // { Thread.currentThread().getName(),
            // header.getLogfilename(), header.getLogfileoffset(),
            // format.format(date),
            // data.getEntry().getEntryType(), "" }));
            System.out.println(data.getEntryType());

        } else {
            System.out.println(MessageFormat.format(messgae,
                new Object[] { Thread.currentThread().getName(), data.getJournalName(),
                        String.valueOf(data.getPosition()), format.format(date) }));
        }

    }
    System.out.println("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n");
    return true;
}

Source File: DummyEventStore.java From canal-1.1.3 with Apache License 2.0

6 votes

public boolean put(List<Event> datas, long timeout, TimeUnit unit) throws InterruptedException, CanalStoreException {
    for (Event data : datas) {
        Date date = new Date(data.getExecuteTime());
        SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
        if (data.getEntryType() == EntryType.TRANSACTIONBEGIN || data.getEntryType() == EntryType.TRANSACTIONEND) {
            // System.out.println(MessageFormat.format(messgae, new Object[]
            // { Thread.currentThread().getName(),
            // header.getLogfilename(), header.getLogfileoffset(),
            // format.format(date),
            // data.getEntry().getEntryType(), "" }));
            System.out.println(data.getEntryType());

        } else {
            System.out.println(MessageFormat.format(messgae,
                new Object[] { Thread.currentThread().getName(), data.getJournalName(),
                        String.valueOf(data.getPosition()), format.format(date) }));
        }
    }
    return true;
}

Source File: FixedTimelineTransactionBarrier.java From DataLink with Apache License 2.0

6 votes

protected boolean isPermit(Event event, long state) {
    if (txState.intValue() == 1 && inTransaction.get()) { // 如果处于事务中，直接允许通过。因为事务头已经做过判断
        return true;
    } else if (txState.intValue() == 0) {
        boolean result = super.isPermit(event, state);
        if (result) {
            // 可能第一条送过来的数据不为Begin，需要做判断处理，如果非事务，允许直接通过，比如DDL语句
            if (isTransactionBegin(event)) {
                if (txState.compareAndSet(0, 1)) {
                    inTransaction.set(true);
                    return true; // 事务允许通过
                }
            } else if (txState.compareAndSet(0, 2)) { // 非事务保护中
                return true; // DDL/DCL允许通过
            }
        }
    }

    return false;
}

Source File: MemoryEventStoreBase.java From canal with Apache License 2.0

5 votes

protected Event buildEvent(String binlogFile, long offset, long timestamp, long eventLenght) {
    Header.Builder headerBuilder = Header.newBuilder();
    headerBuilder.setLogfileName(binlogFile);
    headerBuilder.setLogfileOffset(offset);
    headerBuilder.setExecuteTime(timestamp);
    headerBuilder.setEventLength(eventLenght);
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(headerBuilder.build());
    Entry entry = entryBuilder.build();

    return new Event(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L), entry);
}

Source File: MemoryEventStorePutAndGetTest.java From canal-1.1.3 with Apache License 2.0

5 votes

@Test
public void testFullPutBatchGet() {
    int bufferSize = 16;
    MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
    eventStore.setBufferSize(bufferSize);
    eventStore.start();

    for (int i = 0; i < bufferSize; i++) {
        boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + i));
        sleep(100L);
        Assert.assertTrue(result);
    }

    Position first = eventStore.getFirstPosition();
    Position lastest = eventStore.getLatestPosition();
    Assert.assertEquals(first, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L)));
    Assert.assertEquals(lastest, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L + bufferSize - 1)));

    System.out.println("start get");
    Events<Event> entrys1 = eventStore.tryGet(first, bufferSize);
    System.out.println("first get size : " + entrys1.getEvents().size());

    Assert.assertTrue(entrys1.getEvents().size() == bufferSize);
    Assert.assertEquals(first, entrys1.getPositionRange().getStart());
    Assert.assertEquals(lastest, entrys1.getPositionRange().getEnd());

    Assert.assertEquals(first, CanalEventUtils.createPosition(entrys1.getEvents().get(0)));
    Assert.assertEquals(lastest, CanalEventUtils.createPosition(entrys1.getEvents().get(bufferSize - 1)));
    eventStore.stop();
}

Source File: CanalReaderDownStreamHandler.java From DataLink with Apache License 2.0

5 votes

@Override
public List<Event> before(List<Event> events) {
    if (logger.isDebugEnabled()) {
        events.stream().forEach(e -> {
            logger.debug("Event Dump in Handler : EntryType is {}, EventType is {},SchemaName is {},TableName is {}.",
                    e.getEntry().getEntryType(),
                    e.getEntry().getHeader().getEventType(),
                    e.getEntry().getHeader().getSchemaName(),
                    e.getEntry().getHeader().getTableName());
        });
    }
    return super.before(events);
}

Source File: TimelineTransactionBarrier.java From canal with Apache License 2.0

5 votes

public void await(Event event) throws InterruptedException {
    try {
        super.await(event);
    } catch (InterruptedException e) {
        // 出现线程中断，可能是因为关闭或者主备切换
        // 主备切换对应的事务尾会未正常发送，需要强制设置为事务结束，允许其他队列通过
        reset();
        throw e;
    }
}

Source File: MemoryEventStoreWithBuffer.java From canal with Apache License 2.0

5 votes

public Events<Event> tryGet(Position start, int batchSize) throws CanalStoreException {
    final ReentrantLock lock = this.lock;
    lock.lock();
    try {
        return doGet(start, batchSize);
    } finally {
        lock.unlock();
    }
}

Source File: TimelineTransactionBarrier.java From canal with Apache License 2.0

5 votes

public void await(Event event, long timeout, TimeUnit unit) throws InterruptedException, TimeoutException {
    try {
        super.await(event, timeout, unit);
    } catch (InterruptedException e) {
        // 出现线程中断，可能是因为关闭或者主备切换
        // 主备切换对应的事务尾会未正常发送，需要强制设置为事务结束，允许其他队列通过
        reset();
        throw e;
    }
}

Source File: MemoryEventStoreMemBatchTest.java From canal-1.1.3 with Apache License 2.0

5 votes

@Test
public void testFullPutBatchGet() {
    int bufferSize = 16;
    MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
    eventStore.setBufferSize(bufferSize);
    eventStore.setBatchMode(BatchMode.MEMSIZE);
    eventStore.start();

    for (int i = 0; i < bufferSize; i++) {
        boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + i));
        sleep(100L);
        Assert.assertTrue(result);
    }

    Position first = eventStore.getFirstPosition();
    Position lastest = eventStore.getLatestPosition();
    Assert.assertEquals(first, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L)));
    Assert.assertEquals(lastest, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L + bufferSize - 1)));

    System.out.println("start get");
    Events<Event> entrys1 = eventStore.tryGet(first, bufferSize);
    System.out.println("first get size : " + entrys1.getEvents().size());

    Assert.assertTrue(entrys1.getEvents().size() == bufferSize);
    Assert.assertEquals(first, entrys1.getPositionRange().getStart());
    Assert.assertEquals(lastest, entrys1.getPositionRange().getEnd());

    Assert.assertEquals(first, CanalEventUtils.createPosition(entrys1.getEvents().get(0)));
    Assert.assertEquals(lastest, CanalEventUtils.createPosition(entrys1.getEvents().get(bufferSize - 1)));
    eventStore.stop();
}

com.alibaba.otter.canal.store.model.Event Java Examples