com.alibaba.otter.canal.protocol.CanalEntry.EntryType Java Examples

The following examples show how to use com.alibaba.otter.canal.protocol.CanalEntry.EntryType. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: EntryEventSink.java    From canal-1.1.3 with Apache License 2.0 6 votes vote down vote up 
protected boolean doFilter(CanalEntry.Entry entry) {
    if (filter != null && entry.getEntryType() == EntryType.ROWDATA) {
        String name = getSchemaNameAndTableName(entry);
        boolean need = filter.filter(name);
        if (!need) {
            logger.debug("filter name[{}] entry : {}:{}",
                name,
                entry.getHeader().getLogfileName(),
                entry.getHeader().getLogfileOffset());
        }

        return need;
    } else {
        return true;
    }
}
Example #2
Source File: LogEventConvert.java    From canal with Apache License 2.0 6 votes vote down vote up 
private Entry parseGTIDLogEvent(GtidLogEvent logEvent) {
    LogHeader logHeader = logEvent.getHeader();
    Pair.Builder builder = Pair.newBuilder();
    builder.setKey("gtid");
    builder.setValue(logEvent.getGtidStr());

    if (logEvent.getLastCommitted() != null) {
        builder.setKey("lastCommitted");
        builder.setValue(String.valueOf(logEvent.getLastCommitted()));
        builder.setKey("sequenceNumber");
        builder.setValue(String.valueOf(logEvent.getSequenceNumber()));
    }

    Header header = createHeader(logHeader, "", "", EventType.GTID);
    return createEntry(header, EntryType.GTIDLOG, builder.build().toByteString());
}
Example #3
Source File: EntryEventSink.java    From canal with Apache License 2.0 6 votes vote down vote up 
protected boolean doFilter(CanalEntry.Entry entry) {
    if (filter != null && entry.getEntryType() == EntryType.ROWDATA) {
        String name = getSchemaNameAndTableName(entry);
        boolean need = filter.filter(name);
        if (!need) {
            logger.debug("filter name[{}] entry : {}:{}",
                name,
                entry.getHeader().getLogfileName(),
                entry.getHeader().getLogfileOffset());
        }

        return need;
    } else {
        return true;
    }
}
Example #4
Source File: LogEventConvert.java    From canal-1.1.3 with Apache License 2.0 6 votes vote down vote up 
private Entry parseGTIDLogEvent(GtidLogEvent logEvent) {
    LogHeader logHeader = logEvent.getHeader();
    Pair.Builder builder = Pair.newBuilder();
    builder.setKey("gtid");
    builder.setValue(logEvent.getGtidStr());

    if (logEvent.getLastCommitted() != null) {
        builder.setKey("lastCommitted");
        builder.setValue(String.valueOf(logEvent.getLastCommitted()));
        builder.setKey("sequenceNumber");
        builder.setValue(String.valueOf(logEvent.getSequenceNumber()));
    }

    Header header = createHeader(logHeader, "", "", EventType.GTID);
    return createEntry(header, EntryType.GTIDLOG, builder.build().toByteString());
}
Example #5
Source File: DummyEventStore.java    From canal with Apache License 2.0 6 votes vote down vote up 
public boolean tryPut(List<Event> datas) throws CanalStoreException {
    System.out.println("\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
    for (Event data : datas) {

        Date date = new Date(data.getExecuteTime());
        SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
        if (data.getEntryType() == EntryType.TRANSACTIONBEGIN || data.getEntryType() == EntryType.TRANSACTIONEND) {
            // System.out.println(MessageFormat.format(messgae, new Object[]
            // { Thread.currentThread().getName(),
            // header.getLogfilename(), header.getLogfileoffset(),
            // format.format(date),
            // data.getEntry().getEntryType(), "" }));
            System.out.println(data.getEntryType());

        } else {
            System.out.println(MessageFormat.format(messgae,
                new Object[] { Thread.currentThread().getName(), data.getJournalName(),
                        String.valueOf(data.getPosition()), format.format(date) }));
        }

    }
    System.out.println("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n");
    return true;
}
Example #6
Source File: DummyEventStore.java    From canal with Apache License 2.0 6 votes vote down vote up 
public boolean put(List<Event> datas, long timeout, TimeUnit unit) throws InterruptedException, CanalStoreException {
    for (Event data : datas) {
        Date date = new Date(data.getExecuteTime());
        SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
        if (data.getEntryType() == EntryType.TRANSACTIONBEGIN || data.getEntryType() == EntryType.TRANSACTIONEND) {
            // System.out.println(MessageFormat.format(messgae, new Object[]
            // { Thread.currentThread().getName(),
            // header.getLogfilename(), header.getLogfileoffset(),
            // format.format(date),
            // data.getEntry().getEntryType(), "" }));
            System.out.println(data.getEntryType());

        } else {
            System.out.println(MessageFormat.format(messgae,
                new Object[] { Thread.currentThread().getName(), data.getJournalName(),
                        String.valueOf(data.getPosition()), format.format(date) }));
        }
    }
    return true;
}
Example #7
Source File: DummyEventStore.java    From canal with Apache License 2.0 6 votes vote down vote up 
public void put(List<Event> datas) throws InterruptedException, CanalStoreException {
    for (Event data : datas) {
        Date date = new Date(data.getExecuteTime());
        SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
        if (data.getEntryType() == EntryType.TRANSACTIONBEGIN || data.getEntryType() == EntryType.TRANSACTIONEND) {
            // System.out.println(MessageFormat.format(messgae, new Object[]
            // { Thread.currentThread().getName(),
            // header.getLogfilename(), header.getLogfileoffset(),
            // format.format(date),
            // data.getEntry().getEntryType(), "" }));
            System.out.println(data.getEntryType());

        } else {
            System.out.println(MessageFormat.format(messgae,
                new Object[] { Thread.currentThread().getName(), data.getJournalName(),
                        String.valueOf(data.getPosition()), format.format(date) }));
        }
    }
}
Example #8
Source File: DummyEventStore.java    From canal-1.1.3 with Apache License 2.0 6 votes vote down vote up 
public void put(List<Event> datas) throws InterruptedException, CanalStoreException {
    for (Event data : datas) {
        Date date = new Date(data.getExecuteTime());
        SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
        if (data.getEntryType() == EntryType.TRANSACTIONBEGIN || data.getEntryType() == EntryType.TRANSACTIONEND) {
            // System.out.println(MessageFormat.format(messgae, new Object[]
            // { Thread.currentThread().getName(),
            // header.getLogfilename(), header.getLogfileoffset(),
            // format.format(date),
            // data.getEntry().getEntryType(), "" }));
            System.out.println(data.getEntryType());

        } else {
            System.out.println(MessageFormat.format(messgae,
                new Object[] { Thread.currentThread().getName(), data.getJournalName(),
                        String.valueOf(data.getPosition()), format.format(date) }));
        }
    }
}
Example #9
Source File: DummyEventStore.java    From canal-1.1.3 with Apache License 2.0 6 votes vote down vote up 
public boolean put(List<Event> datas, long timeout, TimeUnit unit) throws InterruptedException, CanalStoreException {
    for (Event data : datas) {
        Date date = new Date(data.getExecuteTime());
        SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
        if (data.getEntryType() == EntryType.TRANSACTIONBEGIN || data.getEntryType() == EntryType.TRANSACTIONEND) {
            // System.out.println(MessageFormat.format(messgae, new Object[]
            // { Thread.currentThread().getName(),
            // header.getLogfilename(), header.getLogfileoffset(),
            // format.format(date),
            // data.getEntry().getEntryType(), "" }));
            System.out.println(data.getEntryType());

        } else {
            System.out.println(MessageFormat.format(messgae,
                new Object[] { Thread.currentThread().getName(), data.getJournalName(),
                        String.valueOf(data.getPosition()), format.format(date) }));
        }
    }
    return true;
}
Example #10
Source File: DummyEventStore.java    From canal-1.1.3 with Apache License 2.0 6 votes vote down vote up 
public boolean tryPut(List<Event> datas) throws CanalStoreException {
    System.out.println("\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
    for (Event data : datas) {

        Date date = new Date(data.getExecuteTime());
        SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
        if (data.getEntryType() == EntryType.TRANSACTIONBEGIN || data.getEntryType() == EntryType.TRANSACTIONEND) {
            // System.out.println(MessageFormat.format(messgae, new Object[]
            // { Thread.currentThread().getName(),
            // header.getLogfilename(), header.getLogfileoffset(),
            // format.format(date),
            // data.getEntry().getEntryType(), "" }));
            System.out.println(data.getEntryType());

        } else {
            System.out.println(MessageFormat.format(messgae,
                new Object[] { Thread.currentThread().getName(), data.getJournalName(),
                        String.valueOf(data.getPosition()), format.format(date) }));
        }

    }
    System.out.println("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n");
    return true;
}
Example #11
Source File: LogEventConvert_old.java    From DBus with Apache License 2.0 5 votes vote down vote up 
public static Entry createEntry(Header header, EntryType entryType, ByteString storeValue) {
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(header);
    entryBuilder.setEntryType(entryType);
    entryBuilder.setStoreValue(storeValue);
    return entryBuilder.build();
}
Example #12
Source File: AbstractEventParser.java    From DBus with Apache License 2.0 5 votes vote down vote up 
protected TimerTask buildHeartBeatTimeTask(ErosaConnection connection) {
    return new TimerTask() {

        public void run() {
            try {
                if (exception == null || lastEntryTime > 0) {
                    // 如果未出现异常，或者有第一条正常数据
                    long now = System.currentTimeMillis();
                    long inteval = (now - lastEntryTime) / 1000;
                    if (inteval >= detectingIntervalInSeconds) {
                        Header.Builder headerBuilder = Header.newBuilder();
                        headerBuilder.setExecuteTime(now);
                        Entry.Builder entryBuilder = Entry.newBuilder();
                        entryBuilder.setHeader(headerBuilder.build());
                        entryBuilder.setEntryType(EntryType.HEARTBEAT);
                        Entry entry = entryBuilder.build();
                        // 提交到sink中，目前不会提交到store中，会在sink中进行忽略
                        consumeTheEventAndProfilingIfNecessary(Arrays.asList(entry));
                    }
                }

            } catch (Throwable e) {
                logger.warn("heartBeat run failed " + ExceptionUtils.getStackTrace(e));
            }
        }

    };
}
Example #13
Source File: LogEventConvert.java    From DBus with Apache License 2.0 5 votes vote down vote up 
private Entry buildQueryEntry(String queryString, LogHeader logHeader) {
    Header header = createHeader(binlogFileName, logHeader, "", "", EventType.QUERY);
    RowChange.Builder rowChangeBuider = RowChange.newBuilder();
    rowChangeBuider.setSql(queryString);
    rowChangeBuider.setEventType(EventType.QUERY);
    return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
}
Example #14
Source File: LogEventConvert.java    From DBus with Apache License 2.0 5 votes vote down vote up 
public static Entry createEntry(Header header, EntryType entryType, ByteString storeValue) {
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(header);
    entryBuilder.setEntryType(entryType);
    entryBuilder.setStoreValue(storeValue);
    return entryBuilder.build();
}
Example #15
Source File: LogEventConvert_old.java    From DBus with Apache License 2.0 5 votes vote down vote up 
private Entry buildQueryEntry(String queryString, LogHeader logHeader) {
    Header header = createHeader(binlogFileName, logHeader, "", "", EventType.QUERY);
    RowChange.Builder rowChangeBuider = RowChange.newBuilder();
    rowChangeBuider.setSql(queryString);
    rowChangeBuider.setEventType(EventType.QUERY);
    return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
}
Example #16
Source File: Event.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
public Event(LogIdentity logIdentity, CanalEntry.Entry entry, boolean raw){
    this.logIdentity = logIdentity;
    this.entryType = entry.getEntryType();
    this.executeTime = entry.getHeader().getExecuteTime();
    this.journalName = entry.getHeader().getLogfileName();
    this.position = entry.getHeader().getLogfileOffset();
    this.serverId = entry.getHeader().getServerId();
    this.gtid = entry.getHeader().getGtid();
    this.eventType = entry.getHeader().getEventType();
    if (entryType == EntryType.ROWDATA) {
        List<CanalEntry.Pair> props = entry.getHeader().getPropsList();
        if (props != null) {
            for (CanalEntry.Pair p : props) {
                if ("rowsCount".equals(p.getKey())) {
                    rowsCount = Integer.parseInt(p.getValue());
                    break;
                }
            }
        }
    }

    if (raw) {
        // build raw
        this.rawEntry = entry.toByteString();
        this.rawLength = rawEntry.size();
    } else {
        this.entry = entry;
        // 按照6倍的event length预估
        this.rawLength = entry.getHeader().getEventLength() * 6;
    }
}
Example #17
Source File: LogEventConvert.java    From DBus with Apache License 2.0 5 votes vote down vote up 
private Entry buildQueryEntry(String queryString, LogHeader logHeader) {
    Header header = createHeader(binlogFileName, logHeader, "", "", EventType.QUERY);
    RowChange.Builder rowChangeBuider = RowChange.newBuilder();
    rowChangeBuider.setSql(queryString);
    rowChangeBuider.setEventType(EventType.QUERY);
    return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
}
Example #18
Source File: LogEventConvert.java    From DBus with Apache License 2.0 5 votes vote down vote up 
public static Entry createEntry(Header header, EntryType entryType, ByteString storeValue) {
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(header);
    entryBuilder.setEntryType(entryType);
    entryBuilder.setStoreValue(storeValue);
    return entryBuilder.build();
}
Example #19
Source File: AbstractEventParser.java    From canal with Apache License 2.0 5 votes vote down vote up 
protected TimerTask buildHeartBeatTimeTask(ErosaConnection connection) {
    return new TimerTask() {

        public void run() {
            try {
                if (exception == null || lastEntryTime > 0) {
                    // 如果未出现异常，或者有第一条正常数据
                    long now = System.currentTimeMillis();
                    long inteval = (now - lastEntryTime) / 1000;
                    if (inteval >= detectingIntervalInSeconds) {
                        Header.Builder headerBuilder = Header.newBuilder();
                        headerBuilder.setExecuteTime(now);
                        Entry.Builder entryBuilder = Entry.newBuilder();
                        entryBuilder.setHeader(headerBuilder.build());
                        entryBuilder.setEntryType(EntryType.HEARTBEAT);
                        Entry entry = entryBuilder.build();
                        // 提交到sink中，目前不会提交到store中，会在sink中进行忽略
                        consumeTheEventAndProfilingIfNecessary(Arrays.asList(entry));
                    }
                }

            } catch (Throwable e) {
                logger.warn("heartBeat run failed ", e);
            }
        }

    };
}
Example #20
Source File: AbstractEventParser.java    From canal with Apache License 2.0 5 votes vote down vote up 
protected LogPosition buildLastTransactionPosition(List<CanalEntry.Entry> entries) { // 初始化一下
    for (int i = entries.size() - 1; i > 0; i--) {
        CanalEntry.Entry entry = entries.get(i);
        if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {// 尽量记录一个事务做为position
            return buildLastPosition(entry);
        }
    }

    return null;
}
Example #21
Source File: ProtocolTest.java    From canal with Apache License 2.0 5 votes vote down vote up 
@Test(expected = CanalClientException.class)
public void testSimple() throws IOException {
    Header.Builder headerBuilder = Header.newBuilder();
    headerBuilder.setLogfileName("mysql-bin.000001");
    headerBuilder.setLogfileOffset(1024);
    headerBuilder.setExecuteTime(1024);
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(headerBuilder.build());
    entryBuilder.setEntryType(EntryType.ROWDATA);
    Entry entry = entryBuilder.build();
    Message message = new Message(3, true, Arrays.asList(entry.toByteString()));

    byte[] body = buildData(message);
    Packet packet = Packet.parseFrom(body);
    switch (packet.getType()) {
        case MESSAGES: {
            if (!packet.getCompression().equals(Compression.NONE)) {
                throw new CanalClientException("compression is not supported in this connector");
            }

            Messages messages = Messages.parseFrom(packet.getBody());
            Message result = new Message(messages.getBatchId());
            for (ByteString byteString : messages.getMessagesList()) {
                result.addEntry(Entry.parseFrom(byteString));
            }

            System.out.println(result);
            break;
        }
        default: {
            throw new CanalClientException("unexpected packet type: " + packet.getType());
        }
    }
}
Example #22
Source File: LogEventConvert.java    From canal with Apache License 2.0 5 votes vote down vote up 
private Entry parseHeartbeatLogEvent(HeartbeatLogEvent logEvent) {
    Header.Builder headerBuilder = Header.newBuilder();
    headerBuilder.setEventType(EventType.MHEARTBEAT);
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(headerBuilder.build());
    entryBuilder.setEntryType(EntryType.HEARTBEAT);
    return entryBuilder.build();
}
Example #23
Source File: AbstractEventParser.java    From DBus with Apache License 2.0 5 votes vote down vote up 
protected LogPosition buildLastTransactionPosition(List<CanalEntry.Entry> entries) { // 初始化一下
    for (int i = entries.size() - 1; i > 0; i--) {
        CanalEntry.Entry entry = entries.get(i);
        if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {// 尽量记录一个事务做为position
            return buildLastPosition(entry);
        }
    }

    return null;
}
Example #24
Source File: LogEventConvert.java    From canal with Apache License 2.0 5 votes vote down vote up 
public static Entry createEntry(Header header, EntryType entryType, ByteString storeValue) {
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(header);
    entryBuilder.setEntryType(entryType);
    entryBuilder.setStoreValue(storeValue);
    return entryBuilder.build();
}
Example #25
Source File: EventTransactionBufferTest.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
private static Entry buildEntry(String binlogFile, long offset, long timestamp, EntryType type) {
    Header.Builder headerBuilder = Header.newBuilder();
    headerBuilder.setLogfileName(binlogFile);
    headerBuilder.setLogfileOffset(offset);
    headerBuilder.setExecuteTime(timestamp);
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(headerBuilder.build());
    entryBuilder.setEntryType(type);
    return entryBuilder.build();
}
Example #26
Source File: EventTransactionBufferTest.java    From canal with Apache License 2.0 5 votes vote down vote up 
private static Entry buildEntry(String binlogFile, long offset, long timestamp, EntryType type) {
    Header.Builder headerBuilder = Header.newBuilder();
    headerBuilder.setLogfileName(binlogFile);
    headerBuilder.setLogfileOffset(offset);
    headerBuilder.setExecuteTime(timestamp);
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(headerBuilder.build());
    entryBuilder.setEntryType(type);
    return entryBuilder.build();
}
Example #27
Source File: Event.java    From canal with Apache License 2.0 5 votes vote down vote up 
public Event(LogIdentity logIdentity, CanalEntry.Entry entry, boolean raw){
    this.logIdentity = logIdentity;
    this.entryType = entry.getEntryType();
    this.executeTime = entry.getHeader().getExecuteTime();
    this.journalName = entry.getHeader().getLogfileName();
    this.position = entry.getHeader().getLogfileOffset();
    this.serverId = entry.getHeader().getServerId();
    this.gtid = entry.getHeader().getGtid();
    this.eventType = entry.getHeader().getEventType();
    if (entryType == EntryType.ROWDATA) {
        List<CanalEntry.Pair> props = entry.getHeader().getPropsList();
        if (props != null) {
            for (CanalEntry.Pair p : props) {
                if ("rowsCount".equals(p.getKey())) {
                    rowsCount = Integer.parseInt(p.getValue());
                    break;
                }
            }
        }
    }

    if (raw) {
        // build raw
        this.rawEntry = entry.toByteString();
        this.rawLength = rawEntry.size();
    } else {
        this.entry = entry;
        // 按照6倍的event length预估
        this.rawLength = entry.getHeader().getEventLength() * 6;
    }
}
Example #28
Source File: AbstractEventParser.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
protected TimerTask buildHeartBeatTimeTask(ErosaConnection connection) {
    return new TimerTask() {

        public void run() {
            try {
                if (exception == null || lastEntryTime > 0) {
                    // 如果未出现异常，或者有第一条正常数据
                    long now = System.currentTimeMillis();
                    long inteval = (now - lastEntryTime) / 1000;
                    if (inteval >= detectingIntervalInSeconds) {
                        Header.Builder headerBuilder = Header.newBuilder();
                        headerBuilder.setExecuteTime(now);
                        Entry.Builder entryBuilder = Entry.newBuilder();
                        entryBuilder.setHeader(headerBuilder.build());
                        entryBuilder.setEntryType(EntryType.HEARTBEAT);
                        Entry entry = entryBuilder.build();
                        // 提交到sink中，目前不会提交到store中，会在sink中进行忽略
                        consumeTheEventAndProfilingIfNecessary(Arrays.asList(entry));
                    }
                }

            } catch (Throwable e) {
                logger.warn("heartBeat run failed ", e);
            }
        }

    };
}
Example #29
Source File: AbstractEventParser.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
protected LogPosition buildLastTransactionPosition(List<CanalEntry.Entry> entries) { // 初始化一下
    for (int i = entries.size() - 1; i > 0; i--) {
        CanalEntry.Entry entry = entries.get(i);
        if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {// 尽量记录一个事务做为position
            return buildLastPosition(entry);
        }
    }

    return null;
}
Example #30
Source File: LogEventConvert.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
public static Entry createEntry(Header header, EntryType entryType, ByteString storeValue) {
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(header);
    entryBuilder.setEntryType(entryType);
    entryBuilder.setStoreValue(storeValue);
    return entryBuilder.build();
}