Major refactoring

The thing still works somehow
This commit is contained in:
2018-11-06 22:51:56 +01:00
parent c4bbbc439f
commit 5ef0642499
21 changed files with 1236 additions and 1042 deletions
+137
View File
@@ -0,0 +1,137 @@
package binlog
import (
"fmt"
"strconv"
"strings"
"github.com/localhots/blt/tools"
)
// FormatDescription is a description of binary log format.
type FormatDescription struct {
Version uint16
ServerVersion string
CreateTimestamp uint32
EventHeaderLength uint8
EventTypeHeaderLengths []uint8
ServerDetails ServerDetails
}
// ServerDetails contains server feature details.
type ServerDetails struct {
Flavor Flavor
Version int
ChecksumAlgorithm ChecksumAlgorithm
}
// FormatDescriptionEvent contains server details and binary log format
// description. It is usually the first event in a log file.
type FormatDescriptionEvent struct {
FormatDescription
}
// Flavor defines the specific kind of MySQL-like database.
type Flavor string
// ChecksumAlgorithm is a checksum algorithm is the one used by the server.
type ChecksumAlgorithm byte
const (
// FlavorMySQL is the MySQL db flavor.
FlavorMySQL = "MySQL"
// ChecksumAlgorithmNone means no checksum appened.
ChecksumAlgorithmNone ChecksumAlgorithm = 0x00
// ChecksumAlgorithmCRC32 used to append a 4 byte checksum at the end.
ChecksumAlgorithmCRC32 ChecksumAlgorithm = 0x01
// ChecksumAlgorithmUndefined is used when checksum algorithm is not known.
ChecksumAlgorithmUndefined ChecksumAlgorithm = 0xFF
)
// Decode decodes given buffer into a format description event.
// Spec: https://dev.mysql.com/doc/internals/en/format-description-event.html
func (e *FormatDescriptionEvent) Decode(data []byte) error {
buf := tools.NewBuffer(data)
e.Version = buf.ReadUint16()
e.ServerVersion = trimString(buf.ReadStringVarLen(50))
e.CreateTimestamp = buf.ReadUint32()
e.EventHeaderLength = buf.ReadUint8()
e.EventTypeHeaderLengths = buf.ReadStringEOF()
e.ServerDetails = ServerDetails{
Flavor: FlavorMySQL,
Version: parseVersionNumber(e.ServerVersion),
ChecksumAlgorithm: ChecksumAlgorithmUndefined,
}
if e.ServerDetails.Version > 50601 {
// Last 5 bytes are:
// [1] Checksum algorithm
// [4] Checksum
e.ServerDetails.ChecksumAlgorithm = ChecksumAlgorithm(data[len(data)-5])
e.EventTypeHeaderLengths = e.EventTypeHeaderLengths[:len(e.EventTypeHeaderLengths)-5]
}
return nil
}
// HeaderLen returns length of event header.
func (fd FormatDescription) HeaderLen() int {
const defaultHeaderLength = 19
if fd.EventHeaderLength > 0 {
return int(fd.EventHeaderLength)
}
return defaultHeaderLength
}
// PostHeaderLen returns length of a post-header for a given event type.
func (fd FormatDescription) PostHeaderLen(et EventType) int {
return int(fd.EventTypeHeaderLengths[et-1])
}
// TableIDSize returns table ID size for a given event type.
func (fd FormatDescription) TableIDSize(et EventType) int {
if fd.PostHeaderLen(et) == 6 {
return 4
}
return 6
}
func (ca ChecksumAlgorithm) String() string {
switch ca {
case ChecksumAlgorithmNone:
return "None"
case ChecksumAlgorithmCRC32:
return "CRC32"
case ChecksumAlgorithmUndefined:
return "Undefined"
default:
return fmt.Sprintf("Unknown(%d)", ca)
}
}
// parseVersionNumber turns string version into a number just like the library
// mysql_get_server_version function does.
// Example: 5.7.19-log gets represented as 50719
// Spec: https://dev.mysql.com/doc/refman/8.0/en/mysql-get-server-version.html
func parseVersionNumber(v string) int {
tokens := strings.Split(v, ".")
major, _ := strconv.Atoi(tokens[0])
minor, _ := strconv.Atoi(tokens[1])
var patch int
for i, c := range tokens[2] {
if c < '0' || c > '9' {
patch, _ = strconv.Atoi(tokens[2][:i])
break
}
}
return major*10000 + minor*100 + patch
}
func trimString(str []byte) string {
for i, c := range str {
if c == 0x00 {
return string(str[:i])
}
}
return string(str)
}
+48
View File
@@ -0,0 +1,48 @@
package binlog
import (
"errors"
"github.com/localhots/blt/tools"
)
var (
// ErrInvalidHeader is returned when event header cannot be parsed.
ErrInvalidHeader = errors.New("Header is invalid")
)
// EventHeader represents binlog event header.
type EventHeader struct {
Timestamp uint32
Type EventType
ServerID uint32
EventLen uint32
NextOffset uint32
Flags uint16
ExtraHeaders []byte
}
// Decode decodes given buffer into event header.
// Spec: https://dev.mysql.com/doc/internals/en/event-header-fields.html
func (h *EventHeader) Decode(connBuff []byte, fd FormatDescription) error {
headerLen := fd.HeaderLen()
if len(connBuff) < headerLen {
return ErrInvalidHeader
}
buf := tools.NewBuffer(connBuff)
h.Timestamp = buf.ReadUint32()
h.Type = EventType(buf.ReadUint8())
h.ServerID = buf.ReadUint32()
h.EventLen = buf.ReadUint32()
if fd.Version == 0 || fd.Version >= 3 {
h.NextOffset = buf.ReadUint32()
h.Flags = buf.ReadUint16()
}
if fd.Version >= 4 {
h.ExtraHeaders = buf.ReadStringVarLen(headerLen - 19)
}
return nil
}
+28
View File
@@ -0,0 +1,28 @@
package binlog
import "github.com/localhots/blt/tools"
// Position ...
type Position struct {
File string
Offset uint64
}
// RotateEvent is written at the end of the file that points to the next file in
// the squence. It is written when a binary log file exceeds a size limit.
type RotateEvent struct {
NextFile Position
}
// Decode decodes given buffer into a rotate event.
// Spec: https://dev.mysql.com/doc/internals/en/rotate-event.html
func (e *RotateEvent) Decode(connBuff []byte, fd FormatDescription) error {
buf := tools.NewBuffer(connBuff)
if fd.Version > 1 {
e.NextFile.Offset = buf.ReadUint64()
} else {
e.NextFile.Offset = 4
}
e.NextFile.File = string(buf.ReadStringEOF())
return nil
}
+203
View File
@@ -0,0 +1,203 @@
package binlog
import (
"github.com/localhots/blt/mysql"
"github.com/localhots/blt/tools"
"github.com/localhots/pretty"
)
// RowsEvent contains a Rows Event.
type RowsEvent struct {
Type EventType
TableID uint64
Flags uint16
ExtraData []byte
ColumnCount uint64
ColumnBitmap1 []byte
ColumnBitmap2 []byte
Rows [][]interface{}
}
type rowsFlag uint16
const (
rowsFlagEndOfStatement rowsFlag = 0x0001
rowsFlagNoForeignKeyChecks rowsFlag = 0x0002
rowsFlagNoUniqueKeyChecks rowsFlag = 0x0004
rowsFlagRowHasColumns rowsFlag = 0x0008
freeTableMapID = 0x00FFFFFF
)
// PeekTableID returns table ID without decoding whole event.
func (e *RowsEvent) PeekTableID(connBuff []byte, fd FormatDescription) uint64 {
if fd.TableIDSize(e.Type) == 6 {
return mysql.DecodeUint48(connBuff)
}
return uint64(mysql.DecodeUint32(connBuff))
}
// Decode decodes given buffer into a rows event event.
func (e *RowsEvent) Decode(connBuff []byte, fd FormatDescription, td TableDescription) error {
// pretty.Println(data)
buf := tools.NewBuffer(connBuff)
idSize := fd.TableIDSize(e.Type)
if idSize == 6 {
e.TableID = buf.ReadUint48()
} else {
e.TableID = uint64(buf.ReadUint32())
}
e.Flags = buf.ReadUint16()
if RowsEventHasExtraData(e.Type) {
// Extra data length is part of extra data, deduct 2 bytes as they
// already store its length
extraLen := buf.ReadUint16() - 2
e.ExtraData = buf.ReadStringVarLen(int(extraLen))
}
e.ColumnCount, _, _ = buf.ReadUintLenEnc()
e.ColumnBitmap1 = buf.ReadStringVarLen(int(e.ColumnCount+7) / 8)
if RowsEventHasSecondBitmap(e.Type) {
e.ColumnBitmap2 = buf.ReadStringVarLen(int(e.ColumnCount+7) / 8)
}
pretty.Println(e.Type.String(), e, td, buf.Cur())
e.decodeRows(buf, td, e.ColumnBitmap1)
return nil
}
func (e *RowsEvent) decodeRows(buf *tools.Buffer, td TableDescription, bm []byte) {
count := 0
for i := 0; i < int(e.ColumnCount); i++ {
if isBitSet(bm, i) {
count++
}
}
count = (count + 7) / 8
nullBM := buf.ReadStringVarLen(count)
nullCnt := 0
row := make([]interface{}, e.ColumnCount)
pretty.Println(count, nullBM)
var err error
for i := 0; i < int(e.ColumnCount); i++ {
if !isBitSet(bm, i) {
continue
}
isNull := (uint32(nullBM[nullCnt/8]) >> uint32(nullCnt%8)) & 0x01
nullCnt++
if isNull > 0 {
row[i] = nil
continue
}
row[i], err = e.decodeValue(buf, mysql.ColumnType(td.ColumnTypes[i]), td.ColumnMeta[i])
if err != nil {
panic(err)
}
}
}
func (e *RowsEvent) decodeValue(buf *tools.Buffer, ct mysql.ColumnType, meta uint16) (interface{}, error) {
switch ct {
case mysql.ColumnTypeDecimal:
pretty.Println("Type", ct.String())
case mysql.ColumnTypeTiny:
pretty.Println("Type", ct.String())
case mysql.ColumnTypeShort:
pretty.Println("Type", ct.String())
case mysql.ColumnTypeLong:
pretty.Println("Type", ct.String())
case mysql.ColumnTypeFloat:
pretty.Println("Type", ct.String())
case mysql.ColumnTypeDouble:
pretty.Println("Type", ct.String())
case mysql.ColumnTypeNull:
pretty.Println("Type", ct.String())
case mysql.ColumnTypeTimestamp:
pretty.Println("Type", ct.String())
case mysql.ColumnTypeLonglong:
pretty.Println("Type", ct.String())
case mysql.ColumnTypeInt24:
pretty.Println("Type", ct.String())
case mysql.ColumnTypeDate:
pretty.Println("Type", ct.String())
case mysql.ColumnTypeTime:
pretty.Println("Type", ct.String())
case mysql.ColumnTypeDatetime:
pretty.Println("Type", ct.String())
case mysql.ColumnTypeYear:
pretty.Println("Type", ct.String())
case mysql.ColumnTypeVarchar:
pretty.Println("Type", ct.String())
case mysql.ColumnTypeBit:
pretty.Println("Type", ct.String())
case mysql.ColumnTypeJSON:
pretty.Println("Type", ct.String())
case mysql.ColumnTypeNewDecimal:
pretty.Println("Type", ct.String())
case mysql.ColumnTypeEnum:
pretty.Println("Type", ct.String())
case mysql.ColumnTypeSet:
pretty.Println("Type", ct.String())
case mysql.ColumnTypeTinyblob:
pretty.Println("Type", ct.String())
case mysql.ColumnTypeMediumblob:
pretty.Println("Type", ct.String())
case mysql.ColumnTypeLongblob:
pretty.Println("Type", ct.String())
case mysql.ColumnTypeBlob:
pretty.Println("Type", ct.String())
case mysql.ColumnTypeVarstring:
pretty.Println("Type", ct.String())
case mysql.ColumnTypeString:
pretty.Println("Type", ct.String())
case mysql.ColumnTypeGeometry:
pretty.Println("Type", ct.String())
}
return nil, nil
}
func isBitSet(bm []byte, i int) bool {
return bm[i>>3]&(1<<(uint(i)&7)) > 0
}
// RowsEventVersion returns rows event versions. If event is not a rows type -1
// is returned.
func RowsEventVersion(et EventType) int {
switch et {
case EventTypeWriteRowsV0, EventTypeUpdateRowsV0, EventTypeDeleteRowsV0:
return 0
case EventTypeWriteRowsV1, EventTypeUpdateRowsV1, EventTypeDeleteRowsV1:
return 1
case EventTypeWriteRowsV2, EventTypeUpdateRowsV2, EventTypeDeleteRowsV2:
return 2
default:
return -1
}
}
// RowsEventHasExtraData returns true if given event is of rows type and
// contains extra data.
func RowsEventHasExtraData(et EventType) bool {
return RowsEventVersion(et) == 2
}
// RowsEventHasSecondBitmap returns true if given event is of rows type and
// contains a second bitmap.
func RowsEventHasSecondBitmap(et EventType) bool {
switch et {
case EventTypeUpdateRowsV1, EventTypeUpdateRowsV2:
return true
default:
return false
}
}
+85
View File
@@ -0,0 +1,85 @@
package binlog
import (
"github.com/localhots/blt/mysql"
"github.com/localhots/blt/tools"
)
// TableDescription contains table details required to process rows events.
type TableDescription struct {
Flags uint16
SchemaName string
TableName string
ColumnCount uint64
ColumnTypes []byte
ColumnMeta []uint16
NullBitmask []byte
}
// TableMapEvent contains table description alongside an ID that would be used
// to reference the table in the following rows events.
type TableMapEvent struct {
TableID uint64
TableDescription
}
// Decode decodes given buffer into a table map event.
// Spec: https://dev.mysql.com/doc/internals/en/table-map-event.html
func (e *TableMapEvent) Decode(connBuff []byte, fd FormatDescription) error {
buf := tools.NewBuffer(connBuff)
idSize := fd.TableIDSize(EventTypeTableMap)
if idSize == 6 {
e.TableID = buf.ReadUint48()
} else {
e.TableID = uint64(buf.ReadUint32())
}
e.Flags = buf.ReadUint16()
schemaName, _ := buf.ReadStringLenEnc()
e.SchemaName = string(schemaName)
buf.Skip(1) // Always 0x00
tableName, _ := buf.ReadStringLenEnc()
e.TableName = string(tableName)
buf.Skip(1) // Always 0x00
e.ColumnCount, _, _ = buf.ReadUintLenEnc()
e.ColumnTypes = buf.ReadStringVarLen(int(e.ColumnCount))
colMeta, _ := buf.ReadStringLenEnc()
e.ColumnMeta = decodeColumnMeta(colMeta, e.ColumnTypes)
e.NullBitmask = buf.ReadStringVarLen(int(e.ColumnCount+8) / 7)
return nil
}
func decodeColumnMeta(data []byte, cols []byte) []uint16 {
pos := 0
meta := make([]uint16, len(cols))
for i, typ := range cols {
switch mysql.ColumnType(typ) {
case mysql.ColumnTypeString,
mysql.ColumnTypeNewDecimal:
// TODO: Is that correct?
meta[i] = uint16(data[pos])<<8 | uint16(data[pos+1])
pos += 2
case mysql.ColumnTypeVarchar,
mysql.ColumnTypeVarstring,
mysql.ColumnTypeBit:
// TODO: Is that correct?
meta[i] = mysql.DecodeUint16(data[pos:])
pos += 2
case mysql.ColumnTypeFloat,
mysql.ColumnTypeDouble,
mysql.ColumnTypeBlob,
mysql.ColumnTypeGeometry,
mysql.ColumnTypeJSON,
mysql.ColumnTypeTime2,
mysql.ColumnTypeDatetime2,
mysql.ColumnTypeTimestamp2:
meta[i] = uint16(data[pos])
pos++
}
}
return meta
}
+209
View File
@@ -0,0 +1,209 @@
package binlog
import (
"fmt"
)
// EventType defines a binary log event type.
type EventType byte
// Spec: https://dev.mysql.com/doc/internals/en/event-classes-and-types.html
const (
// EventTypeUnknown is an event that should never occur.
EventTypeUnknown EventType = 0
// EventTypeStartV3 is the Start_event of binlog format 3.
EventTypeStartV3 EventType = 1
// EventTypeQuery is created for each query that modifies the database,
// unless the query is logged row-based.
EventTypeQuery EventType = 2
// EventTypeStop is written to the log files under these circumstances:
// A master writes the event to the binary log when it shuts down.
// A slave writes the event to the relay log when it shuts down or when a
// RESET SLAVE statement is executed.
EventTypeStop EventType = 3
// EventTypeRotate is written at the end of the file that points to the next
// file in the squence. It is written when a binary log file exceeds a size
// limit.
EventTypeRotate EventType = 4
// EventTypeIntvar will be created just before a Query_event, if the query
// uses one of the variables LAST_INSERT_ID or INSERT_ID.
EventTypeIntvar EventType = 5
// EventTypeLoad ...
EventTypeLoad EventType = 6
// EventTypeSlave ...
EventTypeSlave EventType = 7
// EventTypeCreateFile ...
EventTypeCreateFile EventType = 8
// EventTypeAppendBlock is created to contain the file data.
EventTypeAppendBlock EventType = 9
// EventTypeExecLoad ...
EventTypeExecLoad EventType = 10
// EventTypeDeleteFile occurs when the LOAD DATA failed on the master.
// This event notifies the slave not to do the load and to delete the
// temporary file.
EventTypeDeleteFile EventType = 11
// EventTypeNewLoad ...
EventTypeNewLoad EventType = 12
// EventTypeRand logs random seed used by the next RAND(), and by PASSWORD()
// in 4.1.0.
EventTypeRand EventType = 13
// EventTypeUserVar is written every time a statement uses a user variable;
// precedes other events for the statement. Indicates the value to use for
// the user variable in the next statement. This is written only before a
// QUERY_EVENT and is not used with row-based logging.
EventTypeUserVar EventType = 14
// EventTypeFormatDescription is saved by threads which read it, as they
// need it for future use (to decode the ordinary events).
EventTypeFormatDescription EventType = 15
// EventTypeXID is generated for a commit of a transaction that modifies one
// or more tables of an XA-capable storage engine.
EventTypeXID EventType = 16
// EventTypeBeginLoadQuery is for the first block of file to be loaded, its
// only difference from Append_block event is that this event creates or
// truncates existing file before writing data.
EventTypeBeginLoadQuery EventType = 17
// EventTypeExecuteLoadQuery is responsible for LOAD DATA execution, it
// similar to Query_event but before executing the query it substitutes
// original filename in LOAD DATA query with name of temporary file.
EventTypeExecuteLoadQuery EventType = 18
// EventTypeTableMap is used in row-based mode where it preceeds every row
// operation event and maps a table definition to a number. The table
// definition consists of database name, table name, and column definitions.
EventTypeTableMap EventType = 19
// EventTypeWriteRowsV0 represents inserted rows. Used in MySQL 5.1.0 to
// 5.1.15.
EventTypeWriteRowsV0 EventType = 20
// EventTypeUpdateRowsV0 represents updated rows. It contains both old and
// new versions. Used in MySQL 5.1.0 to 5.1.15.
EventTypeUpdateRowsV0 EventType = 21
// EventTypeDeleteRowsV0 represents deleted rows. Used in MySQL 5.1.0 to
// 5.1.15.
EventTypeDeleteRowsV0 EventType = 22
// EventTypeWriteRowsV1 represents inserted rows. Used in MySQL 5.1.15 to
// 5.6.
EventTypeWriteRowsV1 EventType = 23
// EventTypeUpdateRowsV1 represents updated rows. It contains both old and
// new versions. Used in MySQL 5.1.15 to 5.6.
EventTypeUpdateRowsV1 EventType = 24
// EventTypeDeleteRowsV1 represents deleted rows. Used in MySQL 5.1.15 to
// 5.6.
EventTypeDeleteRowsV1 EventType = 25
// EventTypeIncident represents an incident, an occurance out of the
// ordinary, that happened on the master. The event is used to inform the
// slave that something out of the ordinary happened on the master that
// might cause the database to be in an inconsistent state.
EventTypeIncident EventType = 26
// EventTypeHeartbeet is a replication event used to ensure to slave that
// master is alive. The event is originated by master's dump thread and sent
// straight to slave without being logged. Slave itself does not store it in
// relay log but rather uses a data for immediate checks and throws away the
// event.
EventTypeHeartbeet EventType = 27
// EventTypeIgnorable is a kind of event that could be ignored.
EventTypeIgnorable EventType = 28
// EventTypeRowsQuery is a subclass of the IgnorableEvent, to record the
// original query for the rows events in RBR.
EventTypeRowsQuery EventType = 29
// EventTypeWriteRowsV2 represents inserted rows. Used starting from MySQL
// 5.6.
EventTypeWriteRowsV2 EventType = 30
// EventTypeUpdateRowsV2 represents updated rows. It contains both old and
// new versions. Used starting from MySQL 5.6.
EventTypeUpdateRowsV2 EventType = 31
// EventTypeDeleteRowsV2 represents deleted rows. Used starting from MySQL
// 5.6.
EventTypeDeleteRowsV2 EventType = 32
// EventTypeGTID is an event that contains latest GTID.
// GTID stands for Global Transaction IDentifier It is composed of two
// parts:
// * SID for Source Identifier, and
// * GNO for Group Number. The basic idea is to associate an identifier, the
// Global Transaction IDentifier or GTID, to every transaction. When a
// transaction is copied to a slave, re-executed on the slave, and written
// to the slave's binary log, the GTID is preserved. When a slave connects
// to a master, the slave uses GTIDs instead of (file, offset).
EventTypeGTID EventType = 33
// EventTypeAnonymousGTID is a subclass of GTIDEvent.
EventTypeAnonymousGTID EventType = 34
// EventTypePreviousGTIDs is a subclass of GTIDEvent.
EventTypePreviousGTIDs EventType = 35
)
func (et EventType) String() string {
switch et {
case EventTypeUnknown:
return "UnknownEvent"
case EventTypeStartV3:
return "StartEventV3"
case EventTypeQuery:
return "QueryEvent"
case EventTypeStop:
return "StopEvent"
case EventTypeRotate:
return "RotateEvent"
case EventTypeIntvar:
return "IntvarEvent"
case EventTypeLoad:
return "LoadEvent"
case EventTypeSlave:
return "SlaveEvent"
case EventTypeCreateFile:
return "CreateFileEvent"
case EventTypeAppendBlock:
return "AppendBlockEvent"
case EventTypeExecLoad:
return "ExecLoadEvent"
case EventTypeDeleteFile:
return "DeleteFileEvent"
case EventTypeNewLoad:
return "NewLoadEvent"
case EventTypeRand:
return "RandEvent"
case EventTypeUserVar:
return "UserVarEvent"
case EventTypeFormatDescription:
return "FormatDescriptionEvent"
case EventTypeXID:
return "XIDEvent"
case EventTypeBeginLoadQuery:
return "BeginLoadQueryEvent"
case EventTypeExecuteLoadQuery:
return "ExecuteLoadQueryEvent"
case EventTypeTableMap:
return "TableMapEvent"
case EventTypeWriteRowsV0:
return "WriteRowsEventV0"
case EventTypeUpdateRowsV0:
return "UpdateRowsEventV0"
case EventTypeDeleteRowsV0:
return "DeleteRowsEventV0"
case EventTypeWriteRowsV1:
return "WriteRowsEventV1"
case EventTypeUpdateRowsV1:
return "UpdateRowsEventV1"
case EventTypeDeleteRowsV1:
return "DeleteRowsEventV1"
case EventTypeIncident:
return "IncidentEvent"
case EventTypeHeartbeet:
return "HeartbeetEvent"
case EventTypeIgnorable:
return "IgnorableEvent"
case EventTypeRowsQuery:
return "RowsQueryEvent"
case EventTypeWriteRowsV2:
return "WriteRowsEventV2"
case EventTypeUpdateRowsV2:
return "UpdateRowsEventV2"
case EventTypeDeleteRowsV2:
return "DeleteRowsEventV2"
case EventTypeGTID:
return "GTIDEvent"
case EventTypeAnonymousGTID:
return "AnonymousGTIDEvent"
case EventTypePreviousGTIDs:
return "PreviousGTIDsEvent"
default:
return fmt.Sprintf("Unknown(%d)", et)
}
}