com.alibaba.otter.canal.protocol.CanalEntry.Header Java Examples

The following examples show how to use com.alibaba.otter.canal.protocol.CanalEntry.Header. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: LogEventConvert.java    From DBus with Apache License 2.0 6 votes vote down vote up 
private Header.Builder createHeaderBuilder(String binlogFile, LogHeader logHeader, String schemaName, String tableName,
                                           EventType eventType) {
    // header会做信息冗余,方便以后做检索或者过滤
    Header.Builder headerBuilder = Header.newBuilder();
    headerBuilder.setVersion(version);
    headerBuilder.setLogfileName(binlogFile);
    headerBuilder.setLogfileOffset(logHeader.getLogPos() - logHeader.getEventLen());
    headerBuilder.setServerId(logHeader.getServerId());
    headerBuilder.setServerenCode(UTF_8);// 经过java输出后所有的编码为unicode
    headerBuilder.setExecuteTime(logHeader.getWhen() * 1000L);
    headerBuilder.setSourceType(Type.MYSQL);
    if (eventType != null) {
        headerBuilder.setEventType(eventType);
    }
    if (schemaName != null) {
        headerBuilder.setSchemaName(schemaName);
    }
    if (tableName != null) {
        headerBuilder.setTableName(tableName);
    }
    headerBuilder.setEventLength(logHeader.getEventLen());
    return headerBuilder;
}
Example #2
Source File: LogEventConvert.java    From canal-1.1.3 with Apache License 2.0 6 votes vote down vote up 
private Entry parseGTIDLogEvent(GtidLogEvent logEvent) {
    LogHeader logHeader = logEvent.getHeader();
    Pair.Builder builder = Pair.newBuilder();
    builder.setKey("gtid");
    builder.setValue(logEvent.getGtidStr());

    if (logEvent.getLastCommitted() != null) {
        builder.setKey("lastCommitted");
        builder.setValue(String.valueOf(logEvent.getLastCommitted()));
        builder.setKey("sequenceNumber");
        builder.setValue(String.valueOf(logEvent.getSequenceNumber()));
    }

    Header header = createHeader(logHeader, "", "", EventType.GTID);
    return createEntry(header, EntryType.GTIDLOG, builder.build().toByteString());
}
Example #3
Source File: LogEventConvert_old.java    From DBus with Apache License 2.0 6 votes vote down vote up 
private Header createHeader(String binlogFile, LogHeader logHeader, String schemaName, String tableName,
                            EventType eventType) {
    // header������Ϣ����,�����Ժ����������߹���
    Header.Builder headerBuilder = Header.newBuilder();
    headerBuilder.setVersion(version);
    headerBuilder.setLogfileName(binlogFile);
    headerBuilder.setLogfileOffset(logHeader.getLogPos() - logHeader.getEventLen());
    headerBuilder.setServerId(logHeader.getServerId());
    headerBuilder.setServerenCode(UTF_8);// ����java��������еı���Ϊunicode
    headerBuilder.setExecuteTime(logHeader.getWhen() * 1000L);
    headerBuilder.setSourceType(Type.MYSQL);
    if (eventType != null) {
        headerBuilder.setEventType(eventType);
    }
    if (schemaName != null) {
        headerBuilder.setSchemaName(schemaName);
    }
    if (tableName != null) {
        headerBuilder.setTableName(tableName);
    }
    headerBuilder.setEventLength(logHeader.getEventLen());
    return headerBuilder.build();
}
Example #4
Source File: LogEventConvert.java    From DBus with Apache License 2.0 6 votes vote down vote up 
private Header.Builder createHeaderBuilder(String binlogFile, LogHeader logHeader, String schemaName, String tableName,
                                           EventType eventType) {
     // header会做信息冗余,方便以后做检索或者过滤
    Header.Builder headerBuilder = Header.newBuilder();
    headerBuilder.setVersion(version);
    headerBuilder.setLogfileName(binlogFile);
    headerBuilder.setLogfileOffset(logHeader.getLogPos() - logHeader.getEventLen());
    headerBuilder.setServerId(logHeader.getServerId());
    headerBuilder.setServerenCode(UTF_8);// 经过java输出后所有的编码为unicode
    headerBuilder.setExecuteTime(logHeader.getWhen() * 1000L);
    headerBuilder.setSourceType(Type.MYSQL);
    if (eventType != null) {
        headerBuilder.setEventType(eventType);
    }
    if (schemaName != null) {
        headerBuilder.setSchemaName(schemaName);
    }
    if (tableName != null) {
        headerBuilder.setTableName(tableName);
    }
    headerBuilder.setEventLength(logHeader.getEventLen());

    return headerBuilder;
}
Example #5
Source File: LogEventConvert.java    From canal with Apache License 2.0 6 votes vote down vote up 
private Entry parseGTIDLogEvent(GtidLogEvent logEvent) {
    LogHeader logHeader = logEvent.getHeader();
    Pair.Builder builder = Pair.newBuilder();
    builder.setKey("gtid");
    builder.setValue(logEvent.getGtidStr());

    if (logEvent.getLastCommitted() != null) {
        builder.setKey("lastCommitted");
        builder.setValue(String.valueOf(logEvent.getLastCommitted()));
        builder.setKey("sequenceNumber");
        builder.setValue(String.valueOf(logEvent.getSequenceNumber()));
    }

    Header header = createHeader(logHeader, "", "", EventType.GTID);
    return createEntry(header, EntryType.GTIDLOG, builder.build().toByteString());
}
Example #6
Source File: LogEventConvert.java    From canal with Apache License 2.0 5 votes vote down vote up 
private Entry parseHeartbeatLogEvent(HeartbeatLogEvent logEvent) {
    Header.Builder headerBuilder = Header.newBuilder();
    headerBuilder.setEventType(EventType.MHEARTBEAT);
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(headerBuilder.build());
    entryBuilder.setEntryType(EntryType.HEARTBEAT);
    return entryBuilder.build();
}
Example #7
Source File: LogEventConvert.java    From DBus with Apache License 2.0 5 votes vote down vote up 
public static Entry createEntry(Header header, EntryType entryType, ByteString storeValue) {
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(header);
    entryBuilder.setEntryType(entryType);
    entryBuilder.setStoreValue(storeValue);
    return entryBuilder.build();
}
Example #8
Source File: LogEventConvert_old.java    From DBus with Apache License 2.0 5 votes vote down vote up 
private Entry buildQueryEntry(String queryString, LogHeader logHeader) {
    Header header = createHeader(binlogFileName, logHeader, "", "", EventType.QUERY);
    RowChange.Builder rowChangeBuider = RowChange.newBuilder();
    rowChangeBuider.setSql(queryString);
    rowChangeBuider.setEventType(EventType.QUERY);
    return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
}
Example #9
Source File: LogEventConvert_old.java    From DBus with Apache License 2.0 5 votes vote down vote up 
public static Entry createEntry(Header header, EntryType entryType, ByteString storeValue) {
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(header);
    entryBuilder.setEntryType(entryType);
    entryBuilder.setStoreValue(storeValue);
    return entryBuilder.build();
}
Example #10
Source File: LogEventConvert.java    From DBus with Apache License 2.0 5 votes vote down vote up 
private Entry buildQueryEntry(String queryString, LogHeader logHeader) {
    Header header = createHeader(binlogFileName, logHeader, "", "", EventType.QUERY);
    RowChange.Builder rowChangeBuider = RowChange.newBuilder();
    rowChangeBuider.setSql(queryString);
    rowChangeBuider.setEventType(EventType.QUERY);
    return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
}
Example #11
Source File: LogEventConvert.java    From DBus with Apache License 2.0 5 votes vote down vote up 
public static Entry createEntry(Header header, EntryType entryType, ByteString storeValue) {
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(header);
    entryBuilder.setEntryType(entryType);
    entryBuilder.setStoreValue(storeValue);
    return entryBuilder.build();
}
Example #12
Source File: GroupEventSinkTest.java    From canal with Apache License 2.0 5 votes vote down vote up 
private static Entry buildEntry(String binlogFile, long offset, long timestamp) {
    Header.Builder headerBuilder = Header.newBuilder();
    headerBuilder.setLogfileName(binlogFile);
    headerBuilder.setLogfileOffset(offset);
    headerBuilder.setExecuteTime(timestamp);
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(headerBuilder.build());
    return entryBuilder.build();
}
Example #13
Source File: ProtocolTest.java    From canal with Apache License 2.0 5 votes vote down vote up 
@Test(expected = CanalClientException.class)
public void testSimple() throws IOException {
    Header.Builder headerBuilder = Header.newBuilder();
    headerBuilder.setLogfileName("mysql-bin.000001");
    headerBuilder.setLogfileOffset(1024);
    headerBuilder.setExecuteTime(1024);
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(headerBuilder.build());
    entryBuilder.setEntryType(EntryType.ROWDATA);
    Entry entry = entryBuilder.build();
    Message message = new Message(3, true, Arrays.asList(entry.toByteString()));

    byte[] body = buildData(message);
    Packet packet = Packet.parseFrom(body);
    switch (packet.getType()) {
        case MESSAGES: {
            if (!packet.getCompression().equals(Compression.NONE)) {
                throw new CanalClientException("compression is not supported in this connector");
            }

            Messages messages = Messages.parseFrom(packet.getBody());
            Message result = new Message(messages.getBatchId());
            for (ByteString byteString : messages.getMessagesList()) {
                result.addEntry(Entry.parseFrom(byteString));
            }

            System.out.println(result);
            break;
        }
        default: {
            throw new CanalClientException("unexpected packet type: " + packet.getType());
        }
    }
}
Example #14
Source File: ProtocolTest.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSimple() throws IOException {
    Header.Builder headerBuilder = Header.newBuilder();
    headerBuilder.setLogfileName("mysql-bin.000001");
    headerBuilder.setLogfileOffset(1024);
    headerBuilder.setExecuteTime(1024);
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(headerBuilder.build());
    entryBuilder.setEntryType(EntryType.ROWDATA);
    Entry entry = entryBuilder.build();
    Message message = new Message(3, true, Arrays.asList(entry.toByteString()));

    byte[] body = buildData(message);
    Packet packet = Packet.parseFrom(body);
    switch (packet.getType()) {
        case MESSAGES: {
            if (!packet.getCompression().equals(Compression.NONE)) {
                throw new CanalClientException("compression is not supported in this connector");
            }

            Messages messages = Messages.parseFrom(packet.getBody());
            Message result = new Message(messages.getBatchId());
            for (ByteString byteString : messages.getMessagesList()) {
                result.addEntry(Entry.parseFrom(byteString));
            }

            System.out.println(result);
            break;
        }
        default: {
            throw new CanalClientException("unexpected packet type: " + packet.getType());
        }
    }
}
Example #15
Source File: LogEventConvert.java    From canal with Apache License 2.0 5 votes vote down vote up 
private Entry buildQueryEntry(String queryString, LogHeader logHeader, String tableName) {
    Header header = createHeader(logHeader, "", tableName, EventType.QUERY);
    RowChange.Builder rowChangeBuider = RowChange.newBuilder();
    rowChangeBuider.setSql(queryString);
    rowChangeBuider.setEventType(EventType.QUERY);
    return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
}
Example #16
Source File: LogEventConvert.java    From canal with Apache License 2.0 5 votes vote down vote up 
private Entry buildQueryEntry(String queryString, LogHeader logHeader) {
    Header header = createHeader(logHeader, "", "", EventType.QUERY);
    RowChange.Builder rowChangeBuider = RowChange.newBuilder();
    rowChangeBuider.setSql(queryString);
    rowChangeBuider.setEventType(EventType.QUERY);
    return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
}
Example #17
Source File: LogEventConvert.java    From canal with Apache License 2.0 5 votes vote down vote up 
public static Entry createEntry(Header header, EntryType entryType, ByteString storeValue) {
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(header);
    entryBuilder.setEntryType(entryType);
    entryBuilder.setStoreValue(storeValue);
    return entryBuilder.build();
}
Example #18
Source File: AbstractEventParser.java    From canal with Apache License 2.0 5 votes vote down vote up 
protected TimerTask buildHeartBeatTimeTask(ErosaConnection connection) {
    return new TimerTask() {

        public void run() {
            try {
                if (exception == null || lastEntryTime > 0) {
                    // 如果未出现异常，或者有第一条正常数据
                    long now = System.currentTimeMillis();
                    long inteval = (now - lastEntryTime) / 1000;
                    if (inteval >= detectingIntervalInSeconds) {
                        Header.Builder headerBuilder = Header.newBuilder();
                        headerBuilder.setExecuteTime(now);
                        Entry.Builder entryBuilder = Entry.newBuilder();
                        entryBuilder.setHeader(headerBuilder.build());
                        entryBuilder.setEntryType(EntryType.HEARTBEAT);
                        Entry entry = entryBuilder.build();
                        // 提交到sink中，目前不会提交到store中，会在sink中进行忽略
                        consumeTheEventAndProfilingIfNecessary(Arrays.asList(entry));
                    }
                }

            } catch (Throwable e) {
                logger.warn("heartBeat run failed ", e);
            }
        }

    };
}
Example #19
Source File: EventTransactionBufferTest.java    From canal with Apache License 2.0 5 votes vote down vote up 
private static Entry buildEntry(String binlogFile, long offset, long timestamp) {
    Header.Builder headerBuilder = Header.newBuilder();
    headerBuilder.setLogfileName(binlogFile);
    headerBuilder.setLogfileOffset(offset);
    headerBuilder.setExecuteTime(timestamp);
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(headerBuilder.build());
    return entryBuilder.build();
}
Example #20
Source File: EventTransactionBufferTest.java    From canal with Apache License 2.0 5 votes vote down vote up 
private static Entry buildEntry(String binlogFile, long offset, long timestamp, EntryType type) {
    Header.Builder headerBuilder = Header.newBuilder();
    headerBuilder.setLogfileName(binlogFile);
    headerBuilder.setLogfileOffset(offset);
    headerBuilder.setExecuteTime(timestamp);
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(headerBuilder.build());
    entryBuilder.setEntryType(type);
    return entryBuilder.build();
}
Example #21
Source File: MemoryEventStoreBase.java    From canal with Apache License 2.0 5 votes vote down vote up 
protected Event buildEvent(String binlogFile, long offset, long timestamp) {
    Header.Builder headerBuilder = Header.newBuilder();
    headerBuilder.setLogfileName(binlogFile);
    headerBuilder.setLogfileOffset(offset);
    headerBuilder.setExecuteTime(timestamp);
    headerBuilder.setEventLength(1024);
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(headerBuilder.build());
    Entry entry = entryBuilder.build();

    return new Event(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L), entry);
}
Example #22
Source File: MemoryEventStoreBase.java    From canal with Apache License 2.0 5 votes vote down vote up 
protected Event buildEvent(String binlogFile, long offset, long timestamp, long eventLenght) {
    Header.Builder headerBuilder = Header.newBuilder();
    headerBuilder.setLogfileName(binlogFile);
    headerBuilder.setLogfileOffset(offset);
    headerBuilder.setExecuteTime(timestamp);
    headerBuilder.setEventLength(eventLenght);
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(headerBuilder.build());
    Entry entry = entryBuilder.build();

    return new Event(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L), entry);
}
Example #23
Source File: LogEventConvert.java    From DBus with Apache License 2.0 5 votes vote down vote up 
private Entry buildQueryEntry(String queryString, LogHeader logHeader) {
    Header header = createHeader(binlogFileName, logHeader, "", "", EventType.QUERY);
    RowChange.Builder rowChangeBuider = RowChange.newBuilder();
    rowChangeBuider.setSql(queryString);
    rowChangeBuider.setEventType(EventType.QUERY);
    return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
}
Example #24
Source File: GroupEventSinkTest.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
private static Entry buildEntry(String binlogFile, long offset, long timestamp) {
    Header.Builder headerBuilder = Header.newBuilder();
    headerBuilder.setLogfileName(binlogFile);
    headerBuilder.setLogfileOffset(offset);
    headerBuilder.setExecuteTime(timestamp);
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(headerBuilder.build());
    return entryBuilder.build();
}
Example #25
Source File: LogEventConvert.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
private Entry parseHeartbeatLogEvent(HeartbeatLogEvent logEvent) {
    Header.Builder headerBuilder = Header.newBuilder();
    headerBuilder.setEventType(EventType.MHEARTBEAT);
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(headerBuilder.build());
    entryBuilder.setEntryType(EntryType.HEARTBEAT);
    return entryBuilder.build();
}
Example #26
Source File: LogEventConvert.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
private Entry buildQueryEntry(String queryString, LogHeader logHeader, String tableName) {
    Header header = createHeader(logHeader, "", tableName, EventType.QUERY);
    RowChange.Builder rowChangeBuider = RowChange.newBuilder();
    rowChangeBuider.setSql(queryString);
    rowChangeBuider.setEventType(EventType.QUERY);
    return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
}
Example #27
Source File: LogEventConvert.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
private Entry buildQueryEntry(String queryString, LogHeader logHeader) {
    Header header = createHeader(logHeader, "", "", EventType.QUERY);
    RowChange.Builder rowChangeBuider = RowChange.newBuilder();
    rowChangeBuider.setSql(queryString);
    rowChangeBuider.setEventType(EventType.QUERY);
    return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
}
Example #28
Source File: LogEventConvert.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
public static Entry createEntry(Header header, EntryType entryType, ByteString storeValue) {
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(header);
    entryBuilder.setEntryType(entryType);
    entryBuilder.setStoreValue(storeValue);
    return entryBuilder.build();
}
Example #29
Source File: AbstractEventParser.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
protected TimerTask buildHeartBeatTimeTask(ErosaConnection connection) {
    return new TimerTask() {

        public void run() {
            try {
                if (exception == null || lastEntryTime > 0) {
                    // 如果未出现异常，或者有第一条正常数据
                    long now = System.currentTimeMillis();
                    long inteval = (now - lastEntryTime) / 1000;
                    if (inteval >= detectingIntervalInSeconds) {
                        Header.Builder headerBuilder = Header.newBuilder();
                        headerBuilder.setExecuteTime(now);
                        Entry.Builder entryBuilder = Entry.newBuilder();
                        entryBuilder.setHeader(headerBuilder.build());
                        entryBuilder.setEntryType(EntryType.HEARTBEAT);
                        Entry entry = entryBuilder.build();
                        // 提交到sink中，目前不会提交到store中，会在sink中进行忽略
                        consumeTheEventAndProfilingIfNecessary(Arrays.asList(entry));
                    }
                }

            } catch (Throwable e) {
                logger.warn("heartBeat run failed ", e);
            }
        }

    };
}
Example #30
Source File: EventTransactionBufferTest.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
private static Entry buildEntry(String binlogFile, long offset, long timestamp) {
    Header.Builder headerBuilder = Header.newBuilder();
    headerBuilder.setLogfileName(binlogFile);
    headerBuilder.setLogfileOffset(offset);
    headerBuilder.setExecuteTime(timestamp);
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(headerBuilder.build());
    return entryBuilder.build();
}