From 9927217e74e69817633f121271f6462643cc9051 Mon Sep 17 00:00:00 2001 From: meiji163 Date: Mon, 22 Sep 2025 11:29:15 -0700 Subject: [PATCH 01/56] WIP: add GTID support --- go/base/context.go | 1 + go/binlog/binlog_entry.go | 15 +++++++-------- go/binlog/gomysql_reader.go | 29 ++++++++++++++++++++--------- go/cmd/gh-ost/main.go | 1 + go/logic/streamer.go | 8 ++++++++ go/mysql/binlog.go | 9 ++++++--- go/mysql/utils.go | 7 +++++++ 7 files changed, 50 insertions(+), 20 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index e4008cdd6..2f208050a 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -108,6 +108,7 @@ type MigrationContext struct { // This is useful when connecting to a MySQL instance where the external port // may not match the internal port. SkipPortValidation bool + UseGTID bool config ContextConfig configMutex *sync.Mutex diff --git a/go/binlog/binlog_entry.go b/go/binlog/binlog_entry.go index 5650accdd..18f6ff038 100644 --- a/go/binlog/binlog_entry.go +++ b/go/binlog/binlog_entry.go @@ -7,21 +7,22 @@ package binlog import ( "fmt" + "github.com/github/gh-ost/go/mysql" + + gomysql "github.com/go-mysql-org/go-mysql/mysql" ) // BinlogEntry describes an entry in the binary log type BinlogEntry struct { Coordinates mysql.BinlogCoordinates - EndLogPos uint64 - - DmlEvent *BinlogDMLEvent + DmlEvent *BinlogDMLEvent } // NewBinlogEntry creates an empty, ready to go BinlogEntry object -func NewBinlogEntry(logFile string, logPos uint64) *BinlogEntry { +func NewBinlogEntry(logFile string, logPos uint64, gtidSet gomysql.GTIDSet) *BinlogEntry { binlogEntry := &BinlogEntry{ - Coordinates: mysql.BinlogCoordinates{LogFile: logFile, LogPos: int64(logPos)}, + Coordinates: mysql.BinlogCoordinates{LogFile: logFile, LogPos: int64(logPos), ExecutedGTIDSet: gtidSet}, } return binlogEntry } @@ -36,9 +37,7 @@ func NewBinlogEntryAt(coordinates mysql.BinlogCoordinates) *BinlogEntry { // Duplicate creates and returns a new binlog entry, with some of the attributes pre-assigned func (this *BinlogEntry) Duplicate() *BinlogEntry { - binlogEntry := NewBinlogEntry(this.Coordinates.LogFile, uint64(this.Coordinates.LogPos)) - binlogEntry.EndLogPos = this.EndLogPos - return binlogEntry + return NewBinlogEntry(this.Coordinates.LogFile, uint64(this.Coordinates.LogPos), this.Coordinates.ExecutedGTIDSet) } // String() returns a string representation of this binlog entry diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index d42ba1f30..305429069 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -27,6 +27,7 @@ type GoMySQLReader struct { binlogStreamer *replication.BinlogStreamer currentCoordinates mysql.BinlogCoordinates currentCoordinatesMutex *sync.Mutex + useGTID bool LastAppliedRowsEventHint mysql.BinlogCoordinates } @@ -60,11 +61,16 @@ func (this *GoMySQLReader) ConnectBinlogStreamer(coordinates mysql.BinlogCoordin this.currentCoordinates = coordinates this.migrationContext.Log.Infof("Connecting binlog streamer at %+v", this.currentCoordinates) - // Start sync with specified binlog file and position - this.binlogStreamer, err = this.binlogSyncer.StartSync(gomysql.Position{ - Name: this.currentCoordinates.LogFile, - Pos: uint32(this.currentCoordinates.LogPos), - }) + + // Start sync with specified GTID set or binlog file and position + if this.migrationContext.UseGTID { + this.binlogStreamer, err = this.binlogSyncer.StartSyncGTID(this.currentCoordinates.ExecutedGTIDSet) + } else { + this.binlogStreamer, err = this.binlogSyncer.StartSync(gomysql.Position{ + Name: this.currentCoordinates.LogFile, + Pos: uint32(this.currentCoordinates.LogPos)}, + ) + } return err } @@ -148,16 +154,21 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha this.currentCoordinates.EventSize = int64(ev.Header.EventSize) }() - switch binlogEvent := ev.Event.(type) { + switch event := ev.Event.(type) { + case *replication.GTIDEvent: + // TODO: convert *replication.GTIDEvent -> mysql.GTIDSet + if this.migrationContext.UseGTID { + this.migrationContext.Log.Info("TODO: handle GTID event in binlog stream!") + } case *replication.RotateEvent: func() { this.currentCoordinatesMutex.Lock() defer this.currentCoordinatesMutex.Unlock() - this.currentCoordinates.LogFile = string(binlogEvent.NextLogName) + this.currentCoordinates.LogFile = string(event.NextLogName) }() - this.migrationContext.Log.Infof("rotate to next log from %s:%d to %s", this.currentCoordinates.LogFile, int64(ev.Header.LogPos), binlogEvent.NextLogName) + this.migrationContext.Log.Infof("rotate to next log from %s:%d to %s", this.currentCoordinates.LogFile, int64(ev.Header.LogPos), event.NextLogName) case *replication.RowsEvent: - if err := this.handleRowsEvent(ev, binlogEvent, entriesChannel); err != nil { + if err := this.handleRowsEvent(ev, event, entriesChannel); err != nil { return err } } diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 027bfc86e..bae4b5a41 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -87,6 +87,7 @@ func main() { flag.BoolVar(&migrationContext.AliyunRDS, "aliyun-rds", false, "set to 'true' when you execute on Aliyun RDS.") flag.BoolVar(&migrationContext.GoogleCloudPlatform, "gcp", false, "set to 'true' when you execute on a 1st generation Google Cloud Platform (GCP).") flag.BoolVar(&migrationContext.AzureMySQL, "azure", false, "set to 'true' when you execute on Azure Database on MySQL.") + flag.BoolVar(&migrationContext.UseGTID, "gtid", false, "set to 'true' to use MySQL GTIDs for binlog positioning") executeFlag := flag.Bool("execute", false, "actually execute the alter & migrate the table. Default is noop: do some tests and exit") flag.BoolVar(&migrationContext.TestOnReplica, "test-on-replica", false, "Have the migration run on a replica, not on the master. At the end of migration replication is stopped, and tables are swapped and immediately swap-revert. Replication remains stopped and you can compare the two tables for building trust") diff --git a/go/logic/streamer.go b/go/logic/streamer.go index 20bcf4275..2fc3d1f07 100644 --- a/go/logic/streamer.go +++ b/go/logic/streamer.go @@ -16,6 +16,7 @@ import ( "github.com/github/gh-ost/go/binlog" "github.com/github/gh-ost/go/mysql" + gomysql "github.com/go-mysql-org/go-mysql/mysql" "github.com/openark/golib/sqlutils" ) @@ -151,6 +152,13 @@ func (this *EventsStreamer) readCurrentBinlogCoordinates() error { LogFile: m.GetString("File"), LogPos: m.GetInt64("Position"), } + if execGtidSet := m.GetString("Executed_Gtid_Set"); execGtidSet != "" { + var err error + this.initialBinlogCoordinates.ExecutedGTIDSet, err = gomysql.ParseMysqlGTIDSet(execGtidSet) + if err != nil { + return err + } + } foundMasterStatus = true return nil diff --git a/go/mysql/binlog.go b/go/mysql/binlog.go index 5f52f1d4e..f24311735 100644 --- a/go/mysql/binlog.go +++ b/go/mysql/binlog.go @@ -10,13 +10,16 @@ import ( "fmt" "strconv" "strings" + + gomysql "github.com/go-mysql-org/go-mysql/mysql" ) // BinlogCoordinates described binary log coordinates in the form of log file & log position. type BinlogCoordinates struct { - LogFile string - LogPos int64 - EventSize int64 + ExecutedGTIDSet gomysql.GTIDSet + LogFile string + LogPos int64 + EventSize int64 } // ParseBinlogCoordinates will parse an InstanceKey from a string representation such as 127.0.0.1:3306 diff --git a/go/mysql/utils.go b/go/mysql/utils.go index 71146d70a..458f37ccc 100644 --- a/go/mysql/utils.go +++ b/go/mysql/utils.go @@ -14,6 +14,7 @@ import ( "github.com/github/gh-ost/go/sql" + gomysql "github.com/go-mysql-org/go-mysql/mysql" "github.com/openark/golib/log" "github.com/openark/golib/sqlutils" ) @@ -182,6 +183,12 @@ func GetSelfBinlogCoordinates(dbVersion string, db *gosql.DB) (selfBinlogCoordin LogFile: m.GetString("File"), LogPos: m.GetInt64("Position"), } + if execGtidSet := m.GetString("Executed_Gtid_Set"); execGtidSet != "" { + selfBinlogCoordinates.ExecutedGTIDSet, err = gomysql.ParseMysqlGTIDSet(execGtidSet) + if err != nil { + return err + } + } return nil }) return selfBinlogCoordinates, err From 4851e63c5655b3d7302a7e97880f1bbcab0b4dad Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Fri, 26 Mar 2021 03:24:16 +0100 Subject: [PATCH 02/56] Cleanp --- go/binlog/gomysql_reader.go | 1 - go/cmd/gh-ost/main.go | 2 +- go/mysql/binlog.go | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index 305429069..bb356808b 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -27,7 +27,6 @@ type GoMySQLReader struct { binlogStreamer *replication.BinlogStreamer currentCoordinates mysql.BinlogCoordinates currentCoordinatesMutex *sync.Mutex - useGTID bool LastAppliedRowsEventHint mysql.BinlogCoordinates } diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index bae4b5a41..49348706a 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -87,7 +87,7 @@ func main() { flag.BoolVar(&migrationContext.AliyunRDS, "aliyun-rds", false, "set to 'true' when you execute on Aliyun RDS.") flag.BoolVar(&migrationContext.GoogleCloudPlatform, "gcp", false, "set to 'true' when you execute on a 1st generation Google Cloud Platform (GCP).") flag.BoolVar(&migrationContext.AzureMySQL, "azure", false, "set to 'true' when you execute on Azure Database on MySQL.") - flag.BoolVar(&migrationContext.UseGTID, "gtid", false, "set to 'true' to use MySQL GTIDs for binlog positioning") + flag.BoolVar(&migrationContext.UseGTID, "gtid", false, "set to 'true' to enable MySQL GTIDs for replication binlog positioning.") executeFlag := flag.Bool("execute", false, "actually execute the alter & migrate the table. Default is noop: do some tests and exit") flag.BoolVar(&migrationContext.TestOnReplica, "test-on-replica", false, "Have the migration run on a replica, not on the master. At the end of migration replication is stopped, and tables are swapped and immediately swap-revert. Replication remains stopped and you can compare the two tables for building trust") diff --git a/go/mysql/binlog.go b/go/mysql/binlog.go index f24311735..c4c1eb8c0 100644 --- a/go/mysql/binlog.go +++ b/go/mysql/binlog.go @@ -14,7 +14,7 @@ import ( gomysql "github.com/go-mysql-org/go-mysql/mysql" ) -// BinlogCoordinates described binary log coordinates in the form of log file & log position. +// BinlogCoordinates described binary log coordinates in the form of log file & log position or GTID set. type BinlogCoordinates struct { ExecutedGTIDSet gomysql.GTIDSet LogFile string From 4c70abeda191ec5d38a4b265caed0af87ea60421 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Fri, 26 Mar 2021 03:27:43 +0100 Subject: [PATCH 03/56] Add doc for flag --- doc/command-line-flags.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/command-line-flags.md b/doc/command-line-flags.md index b12e8a7ce..99fa240e5 100644 --- a/doc/command-line-flags.md +++ b/doc/command-line-flags.md @@ -160,6 +160,10 @@ Table name prefix to be used on the temporary tables. Add this flag when executing on a 1st generation Google Cloud Platform (GCP). +### gtid + +Add this flag to enable support for [MySQL GTIDs](https://dev.mysql.com/doc/refman/5.7/en/replication-gtids-concepts.html) for replication binlog positioning. + ### heartbeat-interval-millis Default 100. See [`subsecond-lag`](subsecond-lag.md) for details. From 89a40dbb8aa9ef8a713861e1a2f103a46b085fa5 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sat, 27 Mar 2021 00:37:19 +0100 Subject: [PATCH 04/56] Rename GTIDSet var --- go/binlog/binlog_entry.go | 6 +++--- go/binlog/gomysql_reader.go | 2 +- go/logic/streamer.go | 4 ++-- go/mysql/binlog.go | 14 +++++++++----- go/mysql/utils.go | 4 ++-- 5 files changed, 17 insertions(+), 13 deletions(-) diff --git a/go/binlog/binlog_entry.go b/go/binlog/binlog_entry.go index 18f6ff038..5cdc51f0c 100644 --- a/go/binlog/binlog_entry.go +++ b/go/binlog/binlog_entry.go @@ -1,5 +1,5 @@ /* - Copyright 2016 GitHub Inc. + Copyright 2021 GitHub Inc. See https://github.com/github/gh-ost/blob/master/LICENSE */ @@ -22,7 +22,7 @@ type BinlogEntry struct { // NewBinlogEntry creates an empty, ready to go BinlogEntry object func NewBinlogEntry(logFile string, logPos uint64, gtidSet gomysql.GTIDSet) *BinlogEntry { binlogEntry := &BinlogEntry{ - Coordinates: mysql.BinlogCoordinates{LogFile: logFile, LogPos: int64(logPos), ExecutedGTIDSet: gtidSet}, + Coordinates: mysql.BinlogCoordinates{LogFile: logFile, LogPos: int64(logPos), GTIDSet: gtidSet}, } return binlogEntry } @@ -37,7 +37,7 @@ func NewBinlogEntryAt(coordinates mysql.BinlogCoordinates) *BinlogEntry { // Duplicate creates and returns a new binlog entry, with some of the attributes pre-assigned func (this *BinlogEntry) Duplicate() *BinlogEntry { - return NewBinlogEntry(this.Coordinates.LogFile, uint64(this.Coordinates.LogPos), this.Coordinates.ExecutedGTIDSet) + return NewBinlogEntry(this.Coordinates.LogFile, uint64(this.Coordinates.LogPos), this.Coordinates.GTIDSet) } // String() returns a string representation of this binlog entry diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index bb356808b..4c4f67203 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -63,7 +63,7 @@ func (this *GoMySQLReader) ConnectBinlogStreamer(coordinates mysql.BinlogCoordin // Start sync with specified GTID set or binlog file and position if this.migrationContext.UseGTID { - this.binlogStreamer, err = this.binlogSyncer.StartSyncGTID(this.currentCoordinates.ExecutedGTIDSet) + this.binlogStreamer, err = this.binlogSyncer.StartSyncGTID(this.currentCoordinates.GTIDSet) } else { this.binlogStreamer, err = this.binlogSyncer.StartSync(gomysql.Position{ Name: this.currentCoordinates.LogFile, diff --git a/go/logic/streamer.go b/go/logic/streamer.go index 2fc3d1f07..8b274e5d9 100644 --- a/go/logic/streamer.go +++ b/go/logic/streamer.go @@ -152,9 +152,9 @@ func (this *EventsStreamer) readCurrentBinlogCoordinates() error { LogFile: m.GetString("File"), LogPos: m.GetInt64("Position"), } - if execGtidSet := m.GetString("Executed_Gtid_Set"); execGtidSet != "" { + if execGtidSet := m.GetString("Executed_Gtid_Set"); execGtidSet != "" && this.migrationContext.UseGTID { var err error - this.initialBinlogCoordinates.ExecutedGTIDSet, err = gomysql.ParseMysqlGTIDSet(execGtidSet) + this.initialBinlogCoordinates.GTIDSet, err = gomysql.ParseMysqlGTIDSet(execGtidSet) if err != nil { return err } diff --git a/go/mysql/binlog.go b/go/mysql/binlog.go index c4c1eb8c0..c843e852c 100644 --- a/go/mysql/binlog.go +++ b/go/mysql/binlog.go @@ -14,12 +14,12 @@ import ( gomysql "github.com/go-mysql-org/go-mysql/mysql" ) -// BinlogCoordinates described binary log coordinates in the form of log file & log position or GTID set. +// BinlogCoordinates described binary log coordinates in the form of a GTID set and/or log file & log position. type BinlogCoordinates struct { - ExecutedGTIDSet gomysql.GTIDSet - LogFile string - LogPos int64 - EventSize int64 + GTIDSet gomysql.GTIDSet + LogFile string + LogPos int64 + EventSize int64 } // ParseBinlogCoordinates will parse an InstanceKey from a string representation such as 127.0.0.1:3306 @@ -38,6 +38,10 @@ func ParseBinlogCoordinates(logFileLogPos string) (*BinlogCoordinates, error) { // DisplayString returns a user-friendly string representation of these coordinates func (this *BinlogCoordinates) DisplayString() string { + if this.GTIDSet != nil { + return this.GTIDSet.String() + + } return fmt.Sprintf("%s:%d", this.LogFile, this.LogPos) } diff --git a/go/mysql/utils.go b/go/mysql/utils.go index 458f37ccc..907295aef 100644 --- a/go/mysql/utils.go +++ b/go/mysql/utils.go @@ -1,5 +1,5 @@ /* - Copyright 2016 GitHub Inc. + Copyright 2021 GitHub Inc. See https://github.com/github/gh-ost/blob/master/LICENSE */ @@ -184,7 +184,7 @@ func GetSelfBinlogCoordinates(dbVersion string, db *gosql.DB) (selfBinlogCoordin LogPos: m.GetInt64("Position"), } if execGtidSet := m.GetString("Executed_Gtid_Set"); execGtidSet != "" { - selfBinlogCoordinates.ExecutedGTIDSet, err = gomysql.ParseMysqlGTIDSet(execGtidSet) + selfBinlogCoordinates.GTIDSet, err = gomysql.ParseMysqlGTIDSet(execGtidSet) if err != nil { return err } From 2901ac2f120d2362f7c11ba43cef53ec2a04bd97 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sat, 27 Mar 2021 05:44:12 +0100 Subject: [PATCH 05/56] Fix GTID SID parsing --- go.mod | 1 + go.sum | 2 ++ go/base/context.go | 2 +- go/binlog/gomysql_reader.go | 24 ++++++++++++++++++++---- go/cmd/gh-ost/main.go | 2 +- go/logic/inspect.go | 16 ++++++++++++++-- go/logic/streamer.go | 2 +- vendor/modules.txt | 3 +++ 8 files changed, 43 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 0cca873cf..b6b30774f 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/google/uuid v1.6.0 github.com/hashicorp/go-version v1.7.0 github.com/openark/golib v0.0.0-20210531070646-355f37940af8 + github.com/satori/go.uuid v1.2.0 github.com/stretchr/testify v1.10.0 github.com/testcontainers/testcontainers-go v0.37.0 github.com/testcontainers/testcontainers-go/modules/mysql v0.37.0 diff --git a/go.sum b/go.sum index 1a21c591a..60e9146f2 100644 --- a/go.sum +++ b/go.sum @@ -114,6 +114,8 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/shirou/gopsutil/v4 v4.25.1 h1:QSWkTc+fu9LTAWfkZwZ6j8MSUk4A2LV7rbH0ZqmLjXs= github.com/shirou/gopsutil/v4 v4.25.1/go.mod h1:RoUCUpndaJFtT+2zsZzzmhvbfGoDCJ7nFXKJf8GqJbI= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= diff --git a/go/base/context.go b/go/base/context.go index 2f208050a..0a1cae739 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -108,7 +108,7 @@ type MigrationContext struct { // This is useful when connecting to a MySQL instance where the external port // may not match the internal port. SkipPortValidation bool - UseGTID bool + UseGTIDs bool config ContextConfig configMutex *sync.Mutex diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index 4c4f67203..cb96ed241 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -17,6 +17,7 @@ import ( gomysql "github.com/go-mysql-org/go-mysql/mysql" "github.com/go-mysql-org/go-mysql/replication" + uuid "github.com/satori/go.uuid" "golang.org/x/net/context" ) @@ -62,7 +63,7 @@ func (this *GoMySQLReader) ConnectBinlogStreamer(coordinates mysql.BinlogCoordin this.migrationContext.Log.Infof("Connecting binlog streamer at %+v", this.currentCoordinates) // Start sync with specified GTID set or binlog file and position - if this.migrationContext.UseGTID { + if this.migrationContext.UseGTIDs { this.binlogStreamer, err = this.binlogSyncer.StartSyncGTID(this.currentCoordinates.GTIDSet) } else { this.binlogStreamer, err = this.binlogSyncer.StartSync(gomysql.Position{ @@ -155,11 +156,26 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha switch event := ev.Event.(type) { case *replication.GTIDEvent: - // TODO: convert *replication.GTIDEvent -> mysql.GTIDSet - if this.migrationContext.UseGTID { - this.migrationContext.Log.Info("TODO: handle GTID event in binlog stream!") + if !this.migrationContext.UseGTIDs { + continue } + sid, err := uuid.FromBytes(event.SID) + if err != nil { + return err + } + gtidSet, err := gomysql.ParseMysqlGTIDSet(fmt.Sprintf("%s:%d", sid, event.GNO)) + if err != nil { + return err + } + func() { + this.currentCoordinatesMutex.Lock() + defer this.currentCoordinatesMutex.Unlock() + this.currentCoordinates.GTIDSet = gtidSet + }() case *replication.RotateEvent: + if this.migrationContext.UseGTIDs { + continue + } func() { this.currentCoordinatesMutex.Lock() defer this.currentCoordinatesMutex.Unlock() diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 49348706a..940cb1737 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -87,7 +87,7 @@ func main() { flag.BoolVar(&migrationContext.AliyunRDS, "aliyun-rds", false, "set to 'true' when you execute on Aliyun RDS.") flag.BoolVar(&migrationContext.GoogleCloudPlatform, "gcp", false, "set to 'true' when you execute on a 1st generation Google Cloud Platform (GCP).") flag.BoolVar(&migrationContext.AzureMySQL, "azure", false, "set to 'true' when you execute on Azure Database on MySQL.") - flag.BoolVar(&migrationContext.UseGTID, "gtid", false, "set to 'true' to enable MySQL GTIDs for replication binlog positioning.") + flag.BoolVar(&migrationContext.UseGTIDs, "gtid", false, "set to 'true' to enable MySQL GTIDs for replication binlog positioning.") executeFlag := flag.Bool("execute", false, "actually execute the alter & migrate the table. Default is noop: do some tests and exit") flag.BoolVar(&migrationContext.TestOnReplica, "test-on-replica", false, "Have the migration run on a replica, not on the master. At the end of migration replication is stopped, and tables are swapped and immediately swap-revert. Replication remains stopped and you can compare the two tables for building trust") diff --git a/go/logic/inspect.go b/go/logic/inspect.go index b6d80fda7..e7767be89 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -380,9 +380,21 @@ func (this *Inspector) applyBinlogFormat() error { // validateBinlogs checks that binary log configuration is good to go func (this *Inspector) validateBinlogs() error { query := `select /* gh-ost */ @@global.log_bin, @@global.binlog_format` + var gtidMode string var hasBinaryLogs bool - if err := this.db.QueryRow(query).Scan(&hasBinaryLogs, &this.migrationContext.OriginalBinlogFormat); err != nil { - return err + if this.migrationContext.UseGTIDs { + query := `select @@global.log_bin, @@global.binlog_format, @@global.gtid_mode` + if err := this.db.QueryRow(query).Scan(&hasBinaryLogs, &this.migrationContext.OriginalBinlogFormat, >idMode); err != nil { + return err + } + if gtidMode != "ON" { + return fmt.Errorf("%s:%d must have gtid_mode=ON to use GTID support", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) + } + } else { + query := `select @@global.log_bin, @@global.binlog_format` + if err := this.db.QueryRow(query).Scan(&hasBinaryLogs, &this.migrationContext.OriginalBinlogFormat); err != nil { + return err + } } if !hasBinaryLogs { return fmt.Errorf("%s must have binary logs enabled", this.connectionConfig.Key.String()) diff --git a/go/logic/streamer.go b/go/logic/streamer.go index 8b274e5d9..889ec0909 100644 --- a/go/logic/streamer.go +++ b/go/logic/streamer.go @@ -152,7 +152,7 @@ func (this *EventsStreamer) readCurrentBinlogCoordinates() error { LogFile: m.GetString("File"), LogPos: m.GetInt64("Position"), } - if execGtidSet := m.GetString("Executed_Gtid_Set"); execGtidSet != "" && this.migrationContext.UseGTID { + if execGtidSet := m.GetString("Executed_Gtid_Set"); execGtidSet != "" && this.migrationContext.UseGTIDs { var err error this.initialBinlogCoordinates.GTIDSet, err = gomysql.ParseMysqlGTIDSet(execGtidSet) if err != nil { diff --git a/vendor/modules.txt b/vendor/modules.txt index 5cc85376c..450f1ec8a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -203,6 +203,9 @@ github.com/pmezard/go-difflib/difflib # github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c ## explicit; go 1.14 github.com/power-devops/perfstat +# github.com/satori/go.uuid v1.2.0 +## explicit +github.com/satori/go.uuid # github.com/shirou/gopsutil/v4 v4.25.1 ## explicit; go 1.18 github.com/shirou/gopsutil/v4/common From 27160e3cbc98cdce05e1e456756c5d62d0ac2c3a Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sat, 27 Mar 2021 05:47:58 +0100 Subject: [PATCH 06/56] Cleanup --- go/logic/inspect.go | 5 ++--- go/mysql/binlog.go | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index e7767be89..03df2ee90 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -379,10 +379,9 @@ func (this *Inspector) applyBinlogFormat() error { // validateBinlogs checks that binary log configuration is good to go func (this *Inspector) validateBinlogs() error { - query := `select /* gh-ost */ @@global.log_bin, @@global.binlog_format` - var gtidMode string var hasBinaryLogs bool if this.migrationContext.UseGTIDs { + var gtidMode string query := `select @@global.log_bin, @@global.binlog_format, @@global.gtid_mode` if err := this.db.QueryRow(query).Scan(&hasBinaryLogs, &this.migrationContext.OriginalBinlogFormat, >idMode); err != nil { return err @@ -417,7 +416,7 @@ func (this *Inspector) validateBinlogs() error { } this.migrationContext.Log.Infof("%s has %s binlog_format. I will change it to ROW, and will NOT change it back, even in the event of failure.", this.connectionConfig.Key.String(), this.migrationContext.OriginalBinlogFormat) } - query = `select /* gh-ost */ @@global.binlog_row_image` + query := `select /* gh-ost */ @@global.binlog_row_image` if err := this.db.QueryRow(query).Scan(&this.migrationContext.OriginalBinlogRowImage); err != nil { return err } diff --git a/go/mysql/binlog.go b/go/mysql/binlog.go index c843e852c..d5e7e508f 100644 --- a/go/mysql/binlog.go +++ b/go/mysql/binlog.go @@ -40,7 +40,6 @@ func ParseBinlogCoordinates(logFileLogPos string) (*BinlogCoordinates, error) { func (this *BinlogCoordinates) DisplayString() string { if this.GTIDSet != nil { return this.GTIDSet.String() - } return fmt.Sprintf("%s:%d", this.LogFile, this.LogPos) } From 1cf3f4cae8e234dbdebea04fe7ef055bbb2c14e5 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sat, 27 Mar 2021 06:01:37 +0100 Subject: [PATCH 07/56] Add to docs --- doc/command-line-flags.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/command-line-flags.md b/doc/command-line-flags.md index 99fa240e5..c4c10695a 100644 --- a/doc/command-line-flags.md +++ b/doc/command-line-flags.md @@ -162,7 +162,7 @@ Add this flag when executing on a 1st generation Google Cloud Platform (GCP). ### gtid -Add this flag to enable support for [MySQL GTIDs](https://dev.mysql.com/doc/refman/5.7/en/replication-gtids-concepts.html) for replication binlog positioning. +Add this flag to enable support for [MySQL GTIDs](https://dev.mysql.com/doc/refman/5.7/en/replication-gtids-concepts.html) for replication binlog positioning. This requires `gtid_mode` to be set to `ON`. ### heartbeat-interval-millis From 7e546070f11c49c63b9abaf5d2269c6aa845a16f Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sat, 27 Mar 2021 06:11:50 +0100 Subject: [PATCH 08/56] Require enforce_gtid_consistency=ON --- doc/command-line-flags.md | 2 +- go/logic/inspect.go | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/doc/command-line-flags.md b/doc/command-line-flags.md index c4c10695a..153c5265a 100644 --- a/doc/command-line-flags.md +++ b/doc/command-line-flags.md @@ -162,7 +162,7 @@ Add this flag when executing on a 1st generation Google Cloud Platform (GCP). ### gtid -Add this flag to enable support for [MySQL GTIDs](https://dev.mysql.com/doc/refman/5.7/en/replication-gtids-concepts.html) for replication binlog positioning. This requires `gtid_mode` to be set to `ON`. +Add this flag to enable support for [MySQL GTIDs](https://dev.mysql.com/doc/refman/5.7/en/replication-gtids-concepts.html) for replication binlog positioning. This requires `gtid_mode` and `enforce_gtid_consistency` to be set to `ON`. ### heartbeat-interval-millis diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 03df2ee90..efcbf512d 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -381,14 +381,17 @@ func (this *Inspector) applyBinlogFormat() error { func (this *Inspector) validateBinlogs() error { var hasBinaryLogs bool if this.migrationContext.UseGTIDs { - var gtidMode string - query := `select @@global.log_bin, @@global.binlog_format, @@global.gtid_mode` - if err := this.db.QueryRow(query).Scan(&hasBinaryLogs, &this.migrationContext.OriginalBinlogFormat, >idMode); err != nil { + var gtidMode, enforceGtidConsistency string + query := `select @@global.log_bin, @@global.binlog_format, @@global.gtid_mode, @@global.enforce_gtid_consistency` + if err := this.db.QueryRow(query).Scan(&hasBinaryLogs, &this.migrationContext.OriginalBinlogFormat, >idMode, &enforceGtidConsistency); err != nil { return err } if gtidMode != "ON" { return fmt.Errorf("%s:%d must have gtid_mode=ON to use GTID support", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) } + if enforceGtidConsistency != "ON" { + return fmt.Errorf("%s:%d must have enforce_gtid_consistency=ON to use GTID support", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) + } } else { query := `select @@global.log_bin, @@global.binlog_format` if err := this.db.QueryRow(query).Scan(&hasBinaryLogs, &this.migrationContext.OriginalBinlogFormat); err != nil { From 6de0e2ed7764457ab716f6e7228ca540fec84b1a Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sat, 27 Mar 2021 06:16:44 +0100 Subject: [PATCH 09/56] Rename validator func --- go/logic/inspect.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index efcbf512d..bc62742ea 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -70,7 +70,7 @@ func (this *Inspector) InitDBConnections() (err error) { if err := this.validateGrants(); err != nil { return err } - if err := this.validateBinlogs(); err != nil { + if err := this.validateBinlogsAndGTID(); err != nil { return err } if err := this.applyBinlogFormat(); err != nil { @@ -377,8 +377,8 @@ func (this *Inspector) applyBinlogFormat() error { return nil } -// validateBinlogs checks that binary log configuration is good to go -func (this *Inspector) validateBinlogs() error { +// validateBinlogsAndGTID checks that binary log and optional GTID configuration is good to go +func (this *Inspector) validateBinlogsAndGTID() error { var hasBinaryLogs bool if this.migrationContext.UseGTIDs { var gtidMode, enforceGtidConsistency string From 70fc75c1b1baa96d3f2591f760d29706b60cfec7 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sat, 27 Mar 2021 06:55:00 +0100 Subject: [PATCH 10/56] simplify check in validateBinlogsAndGTID() --- go/logic/inspect.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index bc62742ea..cb8c1f62e 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -386,11 +386,8 @@ func (this *Inspector) validateBinlogsAndGTID() error { if err := this.db.QueryRow(query).Scan(&hasBinaryLogs, &this.migrationContext.OriginalBinlogFormat, >idMode, &enforceGtidConsistency); err != nil { return err } - if gtidMode != "ON" { - return fmt.Errorf("%s:%d must have gtid_mode=ON to use GTID support", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) - } - if enforceGtidConsistency != "ON" { - return fmt.Errorf("%s:%d must have enforce_gtid_consistency=ON to use GTID support", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) + if gtidMode != "ON" || enforceGtidConsistency != "ON" { + return fmt.Errorf("%s:%d must have gtid_mode=ON and enforce_gtid_consistency=ON to use GTID support", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) } } else { query := `select @@global.log_bin, @@global.binlog_format` From 977ee3d1559136316c17d0c002c04157139191fe Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sat, 27 Mar 2021 19:16:33 +0100 Subject: [PATCH 11/56] Only update GTIDSet if there was no err --- go/logic/streamer.go | 4 ++-- go/mysql/utils.go | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/go/logic/streamer.go b/go/logic/streamer.go index 889ec0909..83fbfa467 100644 --- a/go/logic/streamer.go +++ b/go/logic/streamer.go @@ -153,11 +153,11 @@ func (this *EventsStreamer) readCurrentBinlogCoordinates() error { LogPos: m.GetInt64("Position"), } if execGtidSet := m.GetString("Executed_Gtid_Set"); execGtidSet != "" && this.migrationContext.UseGTIDs { - var err error - this.initialBinlogCoordinates.GTIDSet, err = gomysql.ParseMysqlGTIDSet(execGtidSet) + gtidSet, err := gomysql.ParseMysqlGTIDSet(execGtidSet) if err != nil { return err } + this.initialBinlogCoordinates.GTIDSet = gtidSet } foundMasterStatus = true diff --git a/go/mysql/utils.go b/go/mysql/utils.go index 907295aef..e70ce18e6 100644 --- a/go/mysql/utils.go +++ b/go/mysql/utils.go @@ -184,10 +184,11 @@ func GetSelfBinlogCoordinates(dbVersion string, db *gosql.DB) (selfBinlogCoordin LogPos: m.GetInt64("Position"), } if execGtidSet := m.GetString("Executed_Gtid_Set"); execGtidSet != "" { - selfBinlogCoordinates.GTIDSet, err = gomysql.ParseMysqlGTIDSet(execGtidSet) + gtidSet, err := gomysql.ParseMysqlGTIDSet(execGtidSet) if err != nil { return err } + selfBinlogCoordinates.GTIDSet = gtidSet } return nil }) From eafe840fbcd9d667e9f2716bddbe105553c7c4ae Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sat, 27 Mar 2021 22:30:50 +0100 Subject: [PATCH 12/56] Simplify GTIDEvent -> GTIDSet --- go/binlog/gomysql_reader.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index cb96ed241..2be10485d 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -163,14 +163,18 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha if err != nil { return err } - gtidSet, err := gomysql.ParseMysqlGTIDSet(fmt.Sprintf("%s:%d", sid, event.GNO)) + uuidSet, err := gomysql.ParseUUIDSet(fmt.Sprintf("%s:%d", sid, event.GNO)) if err != nil { return err } func() { this.currentCoordinatesMutex.Lock() defer this.currentCoordinatesMutex.Unlock() - this.currentCoordinates.GTIDSet = gtidSet + this.currentCoordinates.GTIDSet = &gomysql.MysqlGTIDSet{ + Sets: map[string]*gomysql.UUIDSet{ + sid.String(): uuidSet, + }, + } }() case *replication.RotateEvent: if this.migrationContext.UseGTIDs { From 3130fd49a8f001a775c61438f3d4001c3e7b3778 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sat, 27 Mar 2021 22:58:34 +0100 Subject: [PATCH 13/56] Simplify GTIDEvent -> GTIDSet further, resolve go.uuid dep issue --- go/binlog/gomysql_reader.go | 6 +- vendor/github.com/satori/go.uuid/.travis.yml | 23 ++ vendor/github.com/satori/go.uuid/LICENSE | 20 ++ vendor/github.com/satori/go.uuid/README.md | 65 +++++ vendor/github.com/satori/go.uuid/codec.go | 206 +++++++++++++++ vendor/github.com/satori/go.uuid/generator.go | 239 ++++++++++++++++++ vendor/github.com/satori/go.uuid/sql.go | 78 ++++++ vendor/github.com/satori/go.uuid/uuid.go | 161 ++++++++++++ 8 files changed, 793 insertions(+), 5 deletions(-) create mode 100644 vendor/github.com/satori/go.uuid/.travis.yml create mode 100644 vendor/github.com/satori/go.uuid/LICENSE create mode 100644 vendor/github.com/satori/go.uuid/README.md create mode 100644 vendor/github.com/satori/go.uuid/codec.go create mode 100644 vendor/github.com/satori/go.uuid/generator.go create mode 100644 vendor/github.com/satori/go.uuid/sql.go create mode 100644 vendor/github.com/satori/go.uuid/uuid.go diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index 2be10485d..83e580f97 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -163,16 +163,12 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha if err != nil { return err } - uuidSet, err := gomysql.ParseUUIDSet(fmt.Sprintf("%s:%d", sid, event.GNO)) - if err != nil { - return err - } func() { this.currentCoordinatesMutex.Lock() defer this.currentCoordinatesMutex.Unlock() this.currentCoordinates.GTIDSet = &gomysql.MysqlGTIDSet{ Sets: map[string]*gomysql.UUIDSet{ - sid.String(): uuidSet, + sid.String(): gomysql.NewUUIDSet(sid, gomysql.Interval{event.GNO, 0}), }, } }() diff --git a/vendor/github.com/satori/go.uuid/.travis.yml b/vendor/github.com/satori/go.uuid/.travis.yml new file mode 100644 index 000000000..20dd53b8d --- /dev/null +++ b/vendor/github.com/satori/go.uuid/.travis.yml @@ -0,0 +1,23 @@ +language: go +sudo: false +go: + - 1.2 + - 1.3 + - 1.4 + - 1.5 + - 1.6 + - 1.7 + - 1.8 + - 1.9 + - tip +matrix: + allow_failures: + - go: tip + fast_finish: true +before_install: + - go get github.com/mattn/goveralls + - go get golang.org/x/tools/cmd/cover +script: + - $HOME/gopath/bin/goveralls -service=travis-ci +notifications: + email: false diff --git a/vendor/github.com/satori/go.uuid/LICENSE b/vendor/github.com/satori/go.uuid/LICENSE new file mode 100644 index 000000000..926d54987 --- /dev/null +++ b/vendor/github.com/satori/go.uuid/LICENSE @@ -0,0 +1,20 @@ +Copyright (C) 2013-2018 by Maxim Bublis + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/satori/go.uuid/README.md b/vendor/github.com/satori/go.uuid/README.md new file mode 100644 index 000000000..7b1a722df --- /dev/null +++ b/vendor/github.com/satori/go.uuid/README.md @@ -0,0 +1,65 @@ +# UUID package for Go language + +[![Build Status](https://travis-ci.org/satori/go.uuid.png?branch=master)](https://travis-ci.org/satori/go.uuid) +[![Coverage Status](https://coveralls.io/repos/github/satori/go.uuid/badge.svg?branch=master)](https://coveralls.io/github/satori/go.uuid) +[![GoDoc](http://godoc.org/github.com/satori/go.uuid?status.png)](http://godoc.org/github.com/satori/go.uuid) + +This package provides pure Go implementation of Universally Unique Identifier (UUID). Supported both creation and parsing of UUIDs. + +With 100% test coverage and benchmarks out of box. + +Supported versions: +* Version 1, based on timestamp and MAC address (RFC 4122) +* Version 2, based on timestamp, MAC address and POSIX UID/GID (DCE 1.1) +* Version 3, based on MD5 hashing (RFC 4122) +* Version 4, based on random numbers (RFC 4122) +* Version 5, based on SHA-1 hashing (RFC 4122) + +## Installation + +Use the `go` command: + + $ go get github.com/satori/go.uuid + +## Requirements + +UUID package requires Go >= 1.2. + +## Example + +```go +package main + +import ( + "fmt" + "github.com/satori/go.uuid" +) + +func main() { + // Creating UUID Version 4 + u1 := uuid.NewV4() + fmt.Printf("UUIDv4: %s\n", u1) + + // Parsing UUID from string input + u2, err := uuid.FromString("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + if err != nil { + fmt.Printf("Something gone wrong: %s", err) + } + fmt.Printf("Successfully parsed: %s", u2) +} +``` + +## Documentation + +[Documentation](http://godoc.org/github.com/satori/go.uuid) is hosted at GoDoc project. + +## Links +* [RFC 4122](http://tools.ietf.org/html/rfc4122) +* [DCE 1.1: Authentication and Security Services](http://pubs.opengroup.org/onlinepubs/9696989899/chap5.htm#tagcjh_08_02_01_01) + +## Copyright + +Copyright (C) 2013-2018 by Maxim Bublis . + +UUID package released under MIT License. +See [LICENSE](https://github.com/satori/go.uuid/blob/master/LICENSE) for details. diff --git a/vendor/github.com/satori/go.uuid/codec.go b/vendor/github.com/satori/go.uuid/codec.go new file mode 100644 index 000000000..656892c53 --- /dev/null +++ b/vendor/github.com/satori/go.uuid/codec.go @@ -0,0 +1,206 @@ +// Copyright (C) 2013-2018 by Maxim Bublis +// +// Permission is hereby granted, free of charge, to any person obtaining +// a copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to +// permit persons to whom the Software is furnished to do so, subject to +// the following conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +package uuid + +import ( + "bytes" + "encoding/hex" + "fmt" +) + +// FromBytes returns UUID converted from raw byte slice input. +// It will return error if the slice isn't 16 bytes long. +func FromBytes(input []byte) (u UUID, err error) { + err = u.UnmarshalBinary(input) + return +} + +// FromBytesOrNil returns UUID converted from raw byte slice input. +// Same behavior as FromBytes, but returns a Nil UUID on error. +func FromBytesOrNil(input []byte) UUID { + uuid, err := FromBytes(input) + if err != nil { + return Nil + } + return uuid +} + +// FromString returns UUID parsed from string input. +// Input is expected in a form accepted by UnmarshalText. +func FromString(input string) (u UUID, err error) { + err = u.UnmarshalText([]byte(input)) + return +} + +// FromStringOrNil returns UUID parsed from string input. +// Same behavior as FromString, but returns a Nil UUID on error. +func FromStringOrNil(input string) UUID { + uuid, err := FromString(input) + if err != nil { + return Nil + } + return uuid +} + +// MarshalText implements the encoding.TextMarshaler interface. +// The encoding is the same as returned by String. +func (u UUID) MarshalText() (text []byte, err error) { + text = []byte(u.String()) + return +} + +// UnmarshalText implements the encoding.TextUnmarshaler interface. +// Following formats are supported: +// "6ba7b810-9dad-11d1-80b4-00c04fd430c8", +// "{6ba7b810-9dad-11d1-80b4-00c04fd430c8}", +// "urn:uuid:6ba7b810-9dad-11d1-80b4-00c04fd430c8" +// "6ba7b8109dad11d180b400c04fd430c8" +// ABNF for supported UUID text representation follows: +// uuid := canonical | hashlike | braced | urn +// plain := canonical | hashlike +// canonical := 4hexoct '-' 2hexoct '-' 2hexoct '-' 6hexoct +// hashlike := 12hexoct +// braced := '{' plain '}' +// urn := URN ':' UUID-NID ':' plain +// URN := 'urn' +// UUID-NID := 'uuid' +// 12hexoct := 6hexoct 6hexoct +// 6hexoct := 4hexoct 2hexoct +// 4hexoct := 2hexoct 2hexoct +// 2hexoct := hexoct hexoct +// hexoct := hexdig hexdig +// hexdig := '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | +// 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | +// 'A' | 'B' | 'C' | 'D' | 'E' | 'F' +func (u *UUID) UnmarshalText(text []byte) (err error) { + switch len(text) { + case 32: + return u.decodeHashLike(text) + case 36: + return u.decodeCanonical(text) + case 38: + return u.decodeBraced(text) + case 41: + fallthrough + case 45: + return u.decodeURN(text) + default: + return fmt.Errorf("uuid: incorrect UUID length: %s", text) + } +} + +// decodeCanonical decodes UUID string in format +// "6ba7b810-9dad-11d1-80b4-00c04fd430c8". +func (u *UUID) decodeCanonical(t []byte) (err error) { + if t[8] != '-' || t[13] != '-' || t[18] != '-' || t[23] != '-' { + return fmt.Errorf("uuid: incorrect UUID format %s", t) + } + + src := t[:] + dst := u[:] + + for i, byteGroup := range byteGroups { + if i > 0 { + src = src[1:] // skip dash + } + _, err = hex.Decode(dst[:byteGroup/2], src[:byteGroup]) + if err != nil { + return + } + src = src[byteGroup:] + dst = dst[byteGroup/2:] + } + + return +} + +// decodeHashLike decodes UUID string in format +// "6ba7b8109dad11d180b400c04fd430c8". +func (u *UUID) decodeHashLike(t []byte) (err error) { + src := t[:] + dst := u[:] + + if _, err = hex.Decode(dst, src); err != nil { + return err + } + return +} + +// decodeBraced decodes UUID string in format +// "{6ba7b810-9dad-11d1-80b4-00c04fd430c8}" or in format +// "{6ba7b8109dad11d180b400c04fd430c8}". +func (u *UUID) decodeBraced(t []byte) (err error) { + l := len(t) + + if t[0] != '{' || t[l-1] != '}' { + return fmt.Errorf("uuid: incorrect UUID format %s", t) + } + + return u.decodePlain(t[1 : l-1]) +} + +// decodeURN decodes UUID string in format +// "urn:uuid:6ba7b810-9dad-11d1-80b4-00c04fd430c8" or in format +// "urn:uuid:6ba7b8109dad11d180b400c04fd430c8". +func (u *UUID) decodeURN(t []byte) (err error) { + total := len(t) + + urn_uuid_prefix := t[:9] + + if !bytes.Equal(urn_uuid_prefix, urnPrefix) { + return fmt.Errorf("uuid: incorrect UUID format: %s", t) + } + + return u.decodePlain(t[9:total]) +} + +// decodePlain decodes UUID string in canonical format +// "6ba7b810-9dad-11d1-80b4-00c04fd430c8" or in hash-like format +// "6ba7b8109dad11d180b400c04fd430c8". +func (u *UUID) decodePlain(t []byte) (err error) { + switch len(t) { + case 32: + return u.decodeHashLike(t) + case 36: + return u.decodeCanonical(t) + default: + return fmt.Errorf("uuid: incorrrect UUID length: %s", t) + } +} + +// MarshalBinary implements the encoding.BinaryMarshaler interface. +func (u UUID) MarshalBinary() (data []byte, err error) { + data = u.Bytes() + return +} + +// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface. +// It will return error if the slice isn't 16 bytes long. +func (u *UUID) UnmarshalBinary(data []byte) (err error) { + if len(data) != Size { + err = fmt.Errorf("uuid: UUID must be exactly 16 bytes long, got %d bytes", len(data)) + return + } + copy(u[:], data) + + return +} diff --git a/vendor/github.com/satori/go.uuid/generator.go b/vendor/github.com/satori/go.uuid/generator.go new file mode 100644 index 000000000..3f2f1da2d --- /dev/null +++ b/vendor/github.com/satori/go.uuid/generator.go @@ -0,0 +1,239 @@ +// Copyright (C) 2013-2018 by Maxim Bublis +// +// Permission is hereby granted, free of charge, to any person obtaining +// a copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to +// permit persons to whom the Software is furnished to do so, subject to +// the following conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +package uuid + +import ( + "crypto/md5" + "crypto/rand" + "crypto/sha1" + "encoding/binary" + "hash" + "net" + "os" + "sync" + "time" +) + +// Difference in 100-nanosecond intervals between +// UUID epoch (October 15, 1582) and Unix epoch (January 1, 1970). +const epochStart = 122192928000000000 + +var ( + global = newDefaultGenerator() + + epochFunc = unixTimeFunc + posixUID = uint32(os.Getuid()) + posixGID = uint32(os.Getgid()) +) + +// NewV1 returns UUID based on current timestamp and MAC address. +func NewV1() UUID { + return global.NewV1() +} + +// NewV2 returns DCE Security UUID based on POSIX UID/GID. +func NewV2(domain byte) UUID { + return global.NewV2(domain) +} + +// NewV3 returns UUID based on MD5 hash of namespace UUID and name. +func NewV3(ns UUID, name string) UUID { + return global.NewV3(ns, name) +} + +// NewV4 returns random generated UUID. +func NewV4() UUID { + return global.NewV4() +} + +// NewV5 returns UUID based on SHA-1 hash of namespace UUID and name. +func NewV5(ns UUID, name string) UUID { + return global.NewV5(ns, name) +} + +// Generator provides interface for generating UUIDs. +type Generator interface { + NewV1() UUID + NewV2(domain byte) UUID + NewV3(ns UUID, name string) UUID + NewV4() UUID + NewV5(ns UUID, name string) UUID +} + +// Default generator implementation. +type generator struct { + storageOnce sync.Once + storageMutex sync.Mutex + + lastTime uint64 + clockSequence uint16 + hardwareAddr [6]byte +} + +func newDefaultGenerator() Generator { + return &generator{} +} + +// NewV1 returns UUID based on current timestamp and MAC address. +func (g *generator) NewV1() UUID { + u := UUID{} + + timeNow, clockSeq, hardwareAddr := g.getStorage() + + binary.BigEndian.PutUint32(u[0:], uint32(timeNow)) + binary.BigEndian.PutUint16(u[4:], uint16(timeNow>>32)) + binary.BigEndian.PutUint16(u[6:], uint16(timeNow>>48)) + binary.BigEndian.PutUint16(u[8:], clockSeq) + + copy(u[10:], hardwareAddr) + + u.SetVersion(V1) + u.SetVariant(VariantRFC4122) + + return u +} + +// NewV2 returns DCE Security UUID based on POSIX UID/GID. +func (g *generator) NewV2(domain byte) UUID { + u := UUID{} + + timeNow, clockSeq, hardwareAddr := g.getStorage() + + switch domain { + case DomainPerson: + binary.BigEndian.PutUint32(u[0:], posixUID) + case DomainGroup: + binary.BigEndian.PutUint32(u[0:], posixGID) + } + + binary.BigEndian.PutUint16(u[4:], uint16(timeNow>>32)) + binary.BigEndian.PutUint16(u[6:], uint16(timeNow>>48)) + binary.BigEndian.PutUint16(u[8:], clockSeq) + u[9] = domain + + copy(u[10:], hardwareAddr) + + u.SetVersion(V2) + u.SetVariant(VariantRFC4122) + + return u +} + +// NewV3 returns UUID based on MD5 hash of namespace UUID and name. +func (g *generator) NewV3(ns UUID, name string) UUID { + u := newFromHash(md5.New(), ns, name) + u.SetVersion(V3) + u.SetVariant(VariantRFC4122) + + return u +} + +// NewV4 returns random generated UUID. +func (g *generator) NewV4() UUID { + u := UUID{} + g.safeRandom(u[:]) + u.SetVersion(V4) + u.SetVariant(VariantRFC4122) + + return u +} + +// NewV5 returns UUID based on SHA-1 hash of namespace UUID and name. +func (g *generator) NewV5(ns UUID, name string) UUID { + u := newFromHash(sha1.New(), ns, name) + u.SetVersion(V5) + u.SetVariant(VariantRFC4122) + + return u +} + +func (g *generator) initStorage() { + g.initClockSequence() + g.initHardwareAddr() +} + +func (g *generator) initClockSequence() { + buf := make([]byte, 2) + g.safeRandom(buf) + g.clockSequence = binary.BigEndian.Uint16(buf) +} + +func (g *generator) initHardwareAddr() { + interfaces, err := net.Interfaces() + if err == nil { + for _, iface := range interfaces { + if len(iface.HardwareAddr) >= 6 { + copy(g.hardwareAddr[:], iface.HardwareAddr) + return + } + } + } + + // Initialize hardwareAddr randomly in case + // of real network interfaces absence + g.safeRandom(g.hardwareAddr[:]) + + // Set multicast bit as recommended in RFC 4122 + g.hardwareAddr[0] |= 0x01 +} + +func (g *generator) safeRandom(dest []byte) { + if _, err := rand.Read(dest); err != nil { + panic(err) + } +} + +// Returns UUID v1/v2 storage state. +// Returns epoch timestamp, clock sequence, and hardware address. +func (g *generator) getStorage() (uint64, uint16, []byte) { + g.storageOnce.Do(g.initStorage) + + g.storageMutex.Lock() + defer g.storageMutex.Unlock() + + timeNow := epochFunc() + // Clock changed backwards since last UUID generation. + // Should increase clock sequence. + if timeNow <= g.lastTime { + g.clockSequence++ + } + g.lastTime = timeNow + + return timeNow, g.clockSequence, g.hardwareAddr[:] +} + +// Returns difference in 100-nanosecond intervals between +// UUID epoch (October 15, 1582) and current time. +// This is default epoch calculation function. +func unixTimeFunc() uint64 { + return epochStart + uint64(time.Now().UnixNano()/100) +} + +// Returns UUID based on hashing of namespace UUID and name. +func newFromHash(h hash.Hash, ns UUID, name string) UUID { + u := UUID{} + h.Write(ns[:]) + h.Write([]byte(name)) + copy(u[:], h.Sum(nil)) + + return u +} diff --git a/vendor/github.com/satori/go.uuid/sql.go b/vendor/github.com/satori/go.uuid/sql.go new file mode 100644 index 000000000..56759d390 --- /dev/null +++ b/vendor/github.com/satori/go.uuid/sql.go @@ -0,0 +1,78 @@ +// Copyright (C) 2013-2018 by Maxim Bublis +// +// Permission is hereby granted, free of charge, to any person obtaining +// a copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to +// permit persons to whom the Software is furnished to do so, subject to +// the following conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +package uuid + +import ( + "database/sql/driver" + "fmt" +) + +// Value implements the driver.Valuer interface. +func (u UUID) Value() (driver.Value, error) { + return u.String(), nil +} + +// Scan implements the sql.Scanner interface. +// A 16-byte slice is handled by UnmarshalBinary, while +// a longer byte slice or a string is handled by UnmarshalText. +func (u *UUID) Scan(src interface{}) error { + switch src := src.(type) { + case []byte: + if len(src) == Size { + return u.UnmarshalBinary(src) + } + return u.UnmarshalText(src) + + case string: + return u.UnmarshalText([]byte(src)) + } + + return fmt.Errorf("uuid: cannot convert %T to UUID", src) +} + +// NullUUID can be used with the standard sql package to represent a +// UUID value that can be NULL in the database +type NullUUID struct { + UUID UUID + Valid bool +} + +// Value implements the driver.Valuer interface. +func (u NullUUID) Value() (driver.Value, error) { + if !u.Valid { + return nil, nil + } + // Delegate to UUID Value function + return u.UUID.Value() +} + +// Scan implements the sql.Scanner interface. +func (u *NullUUID) Scan(src interface{}) error { + if src == nil { + u.UUID, u.Valid = Nil, false + return nil + } + + // Delegate to UUID Scan function + u.Valid = true + return u.UUID.Scan(src) +} diff --git a/vendor/github.com/satori/go.uuid/uuid.go b/vendor/github.com/satori/go.uuid/uuid.go new file mode 100644 index 000000000..a2b8e2ca2 --- /dev/null +++ b/vendor/github.com/satori/go.uuid/uuid.go @@ -0,0 +1,161 @@ +// Copyright (C) 2013-2018 by Maxim Bublis +// +// Permission is hereby granted, free of charge, to any person obtaining +// a copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to +// permit persons to whom the Software is furnished to do so, subject to +// the following conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +// Package uuid provides implementation of Universally Unique Identifier (UUID). +// Supported versions are 1, 3, 4 and 5 (as specified in RFC 4122) and +// version 2 (as specified in DCE 1.1). +package uuid + +import ( + "bytes" + "encoding/hex" +) + +// Size of a UUID in bytes. +const Size = 16 + +// UUID representation compliant with specification +// described in RFC 4122. +type UUID [Size]byte + +// UUID versions +const ( + _ byte = iota + V1 + V2 + V3 + V4 + V5 +) + +// UUID layout variants. +const ( + VariantNCS byte = iota + VariantRFC4122 + VariantMicrosoft + VariantFuture +) + +// UUID DCE domains. +const ( + DomainPerson = iota + DomainGroup + DomainOrg +) + +// String parse helpers. +var ( + urnPrefix = []byte("urn:uuid:") + byteGroups = []int{8, 4, 4, 4, 12} +) + +// Nil is special form of UUID that is specified to have all +// 128 bits set to zero. +var Nil = UUID{} + +// Predefined namespace UUIDs. +var ( + NamespaceDNS = Must(FromString("6ba7b810-9dad-11d1-80b4-00c04fd430c8")) + NamespaceURL = Must(FromString("6ba7b811-9dad-11d1-80b4-00c04fd430c8")) + NamespaceOID = Must(FromString("6ba7b812-9dad-11d1-80b4-00c04fd430c8")) + NamespaceX500 = Must(FromString("6ba7b814-9dad-11d1-80b4-00c04fd430c8")) +) + +// Equal returns true if u1 and u2 equals, otherwise returns false. +func Equal(u1 UUID, u2 UUID) bool { + return bytes.Equal(u1[:], u2[:]) +} + +// Version returns algorithm version used to generate UUID. +func (u UUID) Version() byte { + return u[6] >> 4 +} + +// Variant returns UUID layout variant. +func (u UUID) Variant() byte { + switch { + case (u[8] >> 7) == 0x00: + return VariantNCS + case (u[8] >> 6) == 0x02: + return VariantRFC4122 + case (u[8] >> 5) == 0x06: + return VariantMicrosoft + case (u[8] >> 5) == 0x07: + fallthrough + default: + return VariantFuture + } +} + +// Bytes returns bytes slice representation of UUID. +func (u UUID) Bytes() []byte { + return u[:] +} + +// Returns canonical string representation of UUID: +// xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx. +func (u UUID) String() string { + buf := make([]byte, 36) + + hex.Encode(buf[0:8], u[0:4]) + buf[8] = '-' + hex.Encode(buf[9:13], u[4:6]) + buf[13] = '-' + hex.Encode(buf[14:18], u[6:8]) + buf[18] = '-' + hex.Encode(buf[19:23], u[8:10]) + buf[23] = '-' + hex.Encode(buf[24:], u[10:]) + + return string(buf) +} + +// SetVersion sets version bits. +func (u *UUID) SetVersion(v byte) { + u[6] = (u[6] & 0x0f) | (v << 4) +} + +// SetVariant sets variant bits. +func (u *UUID) SetVariant(v byte) { + switch v { + case VariantNCS: + u[8] = (u[8]&(0xff>>1) | (0x00 << 7)) + case VariantRFC4122: + u[8] = (u[8]&(0xff>>2) | (0x02 << 6)) + case VariantMicrosoft: + u[8] = (u[8]&(0xff>>3) | (0x06 << 5)) + case VariantFuture: + fallthrough + default: + u[8] = (u[8]&(0xff>>3) | (0x07 << 5)) + } +} + +// Must is a helper that wraps a call to a function returning (UUID, error) +// and panics if the error is non-nil. It is intended for use in variable +// initializations such as +// var packageUUID = uuid.Must(uuid.FromString("123e4567-e89b-12d3-a456-426655440000")); +func Must(u UUID, err error) UUID { + if err != nil { + panic(err) + } + return u +} From c60a86c26f984740226b2fea65a71095c8480072 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sun, 28 Mar 2021 00:16:40 +0100 Subject: [PATCH 14/56] Fix UUIDSet GNO --- go/binlog/gomysql_reader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index 83e580f97..9d27a6076 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -168,7 +168,7 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha defer this.currentCoordinatesMutex.Unlock() this.currentCoordinates.GTIDSet = &gomysql.MysqlGTIDSet{ Sets: map[string]*gomysql.UUIDSet{ - sid.String(): gomysql.NewUUIDSet(sid, gomysql.Interval{event.GNO, 0}), + sid.String(): gomysql.NewUUIDSet(sid, gomysql.Interval{event.GNO, event.GNO + 1}), }, } }() From fbcbf5b8ab9523cfdfca57e782989f2a6a1790b5 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sun, 28 Mar 2021 20:43:48 +0200 Subject: [PATCH 15/56] Add .ParseGTIDBinlogCoordinates() --- go/binlog/gomysql_reader.go | 5 ++++- go/mysql/binlog.go | 14 ++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index 9d27a6076..70410d5af 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -168,7 +168,10 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha defer this.currentCoordinatesMutex.Unlock() this.currentCoordinates.GTIDSet = &gomysql.MysqlGTIDSet{ Sets: map[string]*gomysql.UUIDSet{ - sid.String(): gomysql.NewUUIDSet(sid, gomysql.Interval{event.GNO, event.GNO + 1}), + sid.String(): gomysql.NewUUIDSet(sid, gomysql.Interval{ + Start: event.GNO, + Stop: event.GNO + 1, + }), }, } }() diff --git a/go/mysql/binlog.go b/go/mysql/binlog.go index d5e7e508f..8a42a9b42 100644 --- a/go/mysql/binlog.go +++ b/go/mysql/binlog.go @@ -22,20 +22,26 @@ type BinlogCoordinates struct { EventSize int64 } -// ParseBinlogCoordinates will parse an InstanceKey from a string representation such as 127.0.0.1:3306 -func ParseBinlogCoordinates(logFileLogPos string) (*BinlogCoordinates, error) { +// ParseFileBinlogCoordinates parses a log file/position string into a *BinlogCoordinates struct. +func ParseFileBinlogCoordinates(logFileLogPos string) (*BinlogCoordinates, error) { tokens := strings.SplitN(logFileLogPos, ":", 2) if len(tokens) != 2 { - return nil, fmt.Errorf("ParseBinlogCoordinates: Cannot parse BinlogCoordinates from %s. Expected format is file:pos", logFileLogPos) + return nil, fmt.Errorf("ParseFileBinlogCoordinates: Cannot parse BinlogCoordinates from %s. Expected format is file:pos", logFileLogPos) } if logPos, err := strconv.ParseInt(tokens[1], 10, 0); err != nil { - return nil, fmt.Errorf("ParseBinlogCoordinates: invalid pos: %s", tokens[1]) + return nil, fmt.Errorf("ParseFileBinlogCoordinates: invalid pos: %s", tokens[1]) } else { return &BinlogCoordinates{LogFile: tokens[0], LogPos: logPos}, nil } } +// ParseGTIDBinlogCoordinates parses a MySQL GTID into a *BinlogCoordinates struct. +func ParseGTIDBinlogCoordinates(gtidSet string) (*BinlogCoordinates, error) { + set, err := gomysql.ParseMysqlGTIDSet(gtidSet) + return &BinlogCoordinates{GTIDSet: set}, err +} + // DisplayString returns a user-friendly string representation of these coordinates func (this *BinlogCoordinates) DisplayString() string { if this.GTIDSet != nil { From 21643daabcecda5d0da2a329989fab24666b7fc8 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Tue, 30 Mar 2021 00:40:04 +0200 Subject: [PATCH 16/56] Add missing smaller-than/equal logic --- go.mod | 1 - go.sum | 2 - go/binlog/gomysql_reader.go | 2 +- go/mysql/binlog.go | 18 +- go/mysql/binlog_test.go | 12 +- vendor/github.com/satori/go.uuid/.travis.yml | 23 -- vendor/github.com/satori/go.uuid/LICENSE | 20 -- vendor/github.com/satori/go.uuid/README.md | 65 ----- vendor/github.com/satori/go.uuid/codec.go | 206 --------------- vendor/github.com/satori/go.uuid/generator.go | 239 ------------------ vendor/github.com/satori/go.uuid/sql.go | 78 ------ vendor/github.com/satori/go.uuid/uuid.go | 161 ------------ vendor/modules.txt | 3 - 13 files changed, 25 insertions(+), 805 deletions(-) delete mode 100644 vendor/github.com/satori/go.uuid/.travis.yml delete mode 100644 vendor/github.com/satori/go.uuid/LICENSE delete mode 100644 vendor/github.com/satori/go.uuid/README.md delete mode 100644 vendor/github.com/satori/go.uuid/codec.go delete mode 100644 vendor/github.com/satori/go.uuid/generator.go delete mode 100644 vendor/github.com/satori/go.uuid/sql.go delete mode 100644 vendor/github.com/satori/go.uuid/uuid.go diff --git a/go.mod b/go.mod index b6b30774f..0cca873cf 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( github.com/google/uuid v1.6.0 github.com/hashicorp/go-version v1.7.0 github.com/openark/golib v0.0.0-20210531070646-355f37940af8 - github.com/satori/go.uuid v1.2.0 github.com/stretchr/testify v1.10.0 github.com/testcontainers/testcontainers-go v0.37.0 github.com/testcontainers/testcontainers-go/modules/mysql v0.37.0 diff --git a/go.sum b/go.sum index 60e9146f2..1a21c591a 100644 --- a/go.sum +++ b/go.sum @@ -114,8 +114,6 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= -github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/shirou/gopsutil/v4 v4.25.1 h1:QSWkTc+fu9LTAWfkZwZ6j8MSUk4A2LV7rbH0ZqmLjXs= github.com/shirou/gopsutil/v4 v4.25.1/go.mod h1:RoUCUpndaJFtT+2zsZzzmhvbfGoDCJ7nFXKJf8GqJbI= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index 70410d5af..0d3eacbd1 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -17,7 +17,7 @@ import ( gomysql "github.com/go-mysql-org/go-mysql/mysql" "github.com/go-mysql-org/go-mysql/replication" - uuid "github.com/satori/go.uuid" + uuid "github.com/google/uuid" "golang.org/x/net/context" ) diff --git a/go/mysql/binlog.go b/go/mysql/binlog.go index 8a42a9b42..50de122f3 100644 --- a/go/mysql/binlog.go +++ b/go/mysql/binlog.go @@ -36,8 +36,8 @@ func ParseFileBinlogCoordinates(logFileLogPos string) (*BinlogCoordinates, error } } -// ParseGTIDBinlogCoordinates parses a MySQL GTID into a *BinlogCoordinates struct. -func ParseGTIDBinlogCoordinates(gtidSet string) (*BinlogCoordinates, error) { +// ParseGTIDSetBinlogCoordinates parses a MySQL GTID set into a *BinlogCoordinates struct. +func ParseGTIDSetBinlogCoordinates(gtidSet string) (*BinlogCoordinates, error) { set, err := gomysql.ParseMysqlGTIDSet(gtidSet) return &BinlogCoordinates{GTIDSet: set}, err } @@ -60,12 +60,15 @@ func (this *BinlogCoordinates) Equals(other *BinlogCoordinates) bool { if other == nil { return false } + if this.GTIDSet != nil && !this.GTIDSet.Equal(other.GTIDSet) { + return false + } return this.LogFile == other.LogFile && this.LogPos == other.LogPos } -// IsEmpty returns true if the log file is empty, unnamed +// IsEmpty returns true if the log file and GTID set is empty, unnamed func (this *BinlogCoordinates) IsEmpty() bool { - return this.LogFile == "" + return this.LogFile == "" && this.GTIDSet == nil } // SmallerThan returns true if this coordinate is strictly smaller than the other. @@ -85,7 +88,12 @@ func (this *BinlogCoordinates) SmallerThanOrEquals(other *BinlogCoordinates) boo if this.SmallerThan(other) { return true } - return this.LogFile == other.LogFile && this.LogPos == other.LogPos + if this.GTIDSet != nil && !this.GTIDSet.Equal(other.GTIDSet) { + return false + } else if other.GTIDSet != nil { + return false + } + return this.LogFile == other.LogFile && this.LogPos == other.LogPos // No Type comparison } // IsLogPosOverflowBeyond4Bytes returns true if the coordinate endpos is overflow beyond 4 bytes. diff --git a/go/mysql/binlog_test.go b/go/mysql/binlog_test.go index 4e7a9c7db..6aa9d31d4 100644 --- a/go/mysql/binlog_test.go +++ b/go/mysql/binlog_test.go @@ -9,6 +9,7 @@ import ( "math" "testing" + gomysql "github.com/go-mysql-org/go-mysql/mysql" "github.com/openark/golib/log" "github.com/stretchr/testify/require" ) @@ -23,6 +24,13 @@ func TestBinlogCoordinates(t *testing.T) { c3 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 5000} c4 := BinlogCoordinates{LogFile: "mysql-bin.00112", LogPos: 104} + gtidSet1, _ := gomysql.ParseMysqlGTIDSet("3E11FA47-71CA-11E1-9E33-C80AA9429562:23") + gtidSet2, _ := gomysql.ParseMysqlGTIDSet("3E11FA47-71CA-11E1-9E33-C80AA9429562:100") + c5 := BinlogCoordinates{GTIDSet: gtidSet1} + c6 := BinlogCoordinates{GTIDSet: gtidSet1} + c7 := BinlogCoordinates{GTIDSet: gtidSet2} + + require.True(t, c5.Equals(&c6)) require.True(t, c1.Equals(&c2)) require.False(t, c1.Equals(&c3)) require.False(t, c1.Equals(&c4)) @@ -33,9 +41,11 @@ func TestBinlogCoordinates(t *testing.T) { require.False(t, c3.SmallerThan(&c2)) require.False(t, c4.SmallerThan(&c2)) require.False(t, c4.SmallerThan(&c3)) - require.True(t, c1.SmallerThanOrEquals(&c2)) require.True(t, c1.SmallerThanOrEquals(&c3)) + require.True(t, c1.SmallerThanOrEquals(&c2)) + require.True(t, c1.SmallerThanOrEquals(&c3)) + require.True(t, c6.SmallerThanOrEquals(&c7)) } func TestBinlogCoordinatesAsKey(t *testing.T) { diff --git a/vendor/github.com/satori/go.uuid/.travis.yml b/vendor/github.com/satori/go.uuid/.travis.yml deleted file mode 100644 index 20dd53b8d..000000000 --- a/vendor/github.com/satori/go.uuid/.travis.yml +++ /dev/null @@ -1,23 +0,0 @@ -language: go -sudo: false -go: - - 1.2 - - 1.3 - - 1.4 - - 1.5 - - 1.6 - - 1.7 - - 1.8 - - 1.9 - - tip -matrix: - allow_failures: - - go: tip - fast_finish: true -before_install: - - go get github.com/mattn/goveralls - - go get golang.org/x/tools/cmd/cover -script: - - $HOME/gopath/bin/goveralls -service=travis-ci -notifications: - email: false diff --git a/vendor/github.com/satori/go.uuid/LICENSE b/vendor/github.com/satori/go.uuid/LICENSE deleted file mode 100644 index 926d54987..000000000 --- a/vendor/github.com/satori/go.uuid/LICENSE +++ /dev/null @@ -1,20 +0,0 @@ -Copyright (C) 2013-2018 by Maxim Bublis - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/satori/go.uuid/README.md b/vendor/github.com/satori/go.uuid/README.md deleted file mode 100644 index 7b1a722df..000000000 --- a/vendor/github.com/satori/go.uuid/README.md +++ /dev/null @@ -1,65 +0,0 @@ -# UUID package for Go language - -[![Build Status](https://travis-ci.org/satori/go.uuid.png?branch=master)](https://travis-ci.org/satori/go.uuid) -[![Coverage Status](https://coveralls.io/repos/github/satori/go.uuid/badge.svg?branch=master)](https://coveralls.io/github/satori/go.uuid) -[![GoDoc](http://godoc.org/github.com/satori/go.uuid?status.png)](http://godoc.org/github.com/satori/go.uuid) - -This package provides pure Go implementation of Universally Unique Identifier (UUID). Supported both creation and parsing of UUIDs. - -With 100% test coverage and benchmarks out of box. - -Supported versions: -* Version 1, based on timestamp and MAC address (RFC 4122) -* Version 2, based on timestamp, MAC address and POSIX UID/GID (DCE 1.1) -* Version 3, based on MD5 hashing (RFC 4122) -* Version 4, based on random numbers (RFC 4122) -* Version 5, based on SHA-1 hashing (RFC 4122) - -## Installation - -Use the `go` command: - - $ go get github.com/satori/go.uuid - -## Requirements - -UUID package requires Go >= 1.2. - -## Example - -```go -package main - -import ( - "fmt" - "github.com/satori/go.uuid" -) - -func main() { - // Creating UUID Version 4 - u1 := uuid.NewV4() - fmt.Printf("UUIDv4: %s\n", u1) - - // Parsing UUID from string input - u2, err := uuid.FromString("6ba7b810-9dad-11d1-80b4-00c04fd430c8") - if err != nil { - fmt.Printf("Something gone wrong: %s", err) - } - fmt.Printf("Successfully parsed: %s", u2) -} -``` - -## Documentation - -[Documentation](http://godoc.org/github.com/satori/go.uuid) is hosted at GoDoc project. - -## Links -* [RFC 4122](http://tools.ietf.org/html/rfc4122) -* [DCE 1.1: Authentication and Security Services](http://pubs.opengroup.org/onlinepubs/9696989899/chap5.htm#tagcjh_08_02_01_01) - -## Copyright - -Copyright (C) 2013-2018 by Maxim Bublis . - -UUID package released under MIT License. -See [LICENSE](https://github.com/satori/go.uuid/blob/master/LICENSE) for details. diff --git a/vendor/github.com/satori/go.uuid/codec.go b/vendor/github.com/satori/go.uuid/codec.go deleted file mode 100644 index 656892c53..000000000 --- a/vendor/github.com/satori/go.uuid/codec.go +++ /dev/null @@ -1,206 +0,0 @@ -// Copyright (C) 2013-2018 by Maxim Bublis -// -// Permission is hereby granted, free of charge, to any person obtaining -// a copy of this software and associated documentation files (the -// "Software"), to deal in the Software without restriction, including -// without limitation the rights to use, copy, modify, merge, publish, -// distribute, sublicense, and/or sell copies of the Software, and to -// permit persons to whom the Software is furnished to do so, subject to -// the following conditions: -// -// The above copyright notice and this permission notice shall be -// included in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -package uuid - -import ( - "bytes" - "encoding/hex" - "fmt" -) - -// FromBytes returns UUID converted from raw byte slice input. -// It will return error if the slice isn't 16 bytes long. -func FromBytes(input []byte) (u UUID, err error) { - err = u.UnmarshalBinary(input) - return -} - -// FromBytesOrNil returns UUID converted from raw byte slice input. -// Same behavior as FromBytes, but returns a Nil UUID on error. -func FromBytesOrNil(input []byte) UUID { - uuid, err := FromBytes(input) - if err != nil { - return Nil - } - return uuid -} - -// FromString returns UUID parsed from string input. -// Input is expected in a form accepted by UnmarshalText. -func FromString(input string) (u UUID, err error) { - err = u.UnmarshalText([]byte(input)) - return -} - -// FromStringOrNil returns UUID parsed from string input. -// Same behavior as FromString, but returns a Nil UUID on error. -func FromStringOrNil(input string) UUID { - uuid, err := FromString(input) - if err != nil { - return Nil - } - return uuid -} - -// MarshalText implements the encoding.TextMarshaler interface. -// The encoding is the same as returned by String. -func (u UUID) MarshalText() (text []byte, err error) { - text = []byte(u.String()) - return -} - -// UnmarshalText implements the encoding.TextUnmarshaler interface. -// Following formats are supported: -// "6ba7b810-9dad-11d1-80b4-00c04fd430c8", -// "{6ba7b810-9dad-11d1-80b4-00c04fd430c8}", -// "urn:uuid:6ba7b810-9dad-11d1-80b4-00c04fd430c8" -// "6ba7b8109dad11d180b400c04fd430c8" -// ABNF for supported UUID text representation follows: -// uuid := canonical | hashlike | braced | urn -// plain := canonical | hashlike -// canonical := 4hexoct '-' 2hexoct '-' 2hexoct '-' 6hexoct -// hashlike := 12hexoct -// braced := '{' plain '}' -// urn := URN ':' UUID-NID ':' plain -// URN := 'urn' -// UUID-NID := 'uuid' -// 12hexoct := 6hexoct 6hexoct -// 6hexoct := 4hexoct 2hexoct -// 4hexoct := 2hexoct 2hexoct -// 2hexoct := hexoct hexoct -// hexoct := hexdig hexdig -// hexdig := '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | -// 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | -// 'A' | 'B' | 'C' | 'D' | 'E' | 'F' -func (u *UUID) UnmarshalText(text []byte) (err error) { - switch len(text) { - case 32: - return u.decodeHashLike(text) - case 36: - return u.decodeCanonical(text) - case 38: - return u.decodeBraced(text) - case 41: - fallthrough - case 45: - return u.decodeURN(text) - default: - return fmt.Errorf("uuid: incorrect UUID length: %s", text) - } -} - -// decodeCanonical decodes UUID string in format -// "6ba7b810-9dad-11d1-80b4-00c04fd430c8". -func (u *UUID) decodeCanonical(t []byte) (err error) { - if t[8] != '-' || t[13] != '-' || t[18] != '-' || t[23] != '-' { - return fmt.Errorf("uuid: incorrect UUID format %s", t) - } - - src := t[:] - dst := u[:] - - for i, byteGroup := range byteGroups { - if i > 0 { - src = src[1:] // skip dash - } - _, err = hex.Decode(dst[:byteGroup/2], src[:byteGroup]) - if err != nil { - return - } - src = src[byteGroup:] - dst = dst[byteGroup/2:] - } - - return -} - -// decodeHashLike decodes UUID string in format -// "6ba7b8109dad11d180b400c04fd430c8". -func (u *UUID) decodeHashLike(t []byte) (err error) { - src := t[:] - dst := u[:] - - if _, err = hex.Decode(dst, src); err != nil { - return err - } - return -} - -// decodeBraced decodes UUID string in format -// "{6ba7b810-9dad-11d1-80b4-00c04fd430c8}" or in format -// "{6ba7b8109dad11d180b400c04fd430c8}". -func (u *UUID) decodeBraced(t []byte) (err error) { - l := len(t) - - if t[0] != '{' || t[l-1] != '}' { - return fmt.Errorf("uuid: incorrect UUID format %s", t) - } - - return u.decodePlain(t[1 : l-1]) -} - -// decodeURN decodes UUID string in format -// "urn:uuid:6ba7b810-9dad-11d1-80b4-00c04fd430c8" or in format -// "urn:uuid:6ba7b8109dad11d180b400c04fd430c8". -func (u *UUID) decodeURN(t []byte) (err error) { - total := len(t) - - urn_uuid_prefix := t[:9] - - if !bytes.Equal(urn_uuid_prefix, urnPrefix) { - return fmt.Errorf("uuid: incorrect UUID format: %s", t) - } - - return u.decodePlain(t[9:total]) -} - -// decodePlain decodes UUID string in canonical format -// "6ba7b810-9dad-11d1-80b4-00c04fd430c8" or in hash-like format -// "6ba7b8109dad11d180b400c04fd430c8". -func (u *UUID) decodePlain(t []byte) (err error) { - switch len(t) { - case 32: - return u.decodeHashLike(t) - case 36: - return u.decodeCanonical(t) - default: - return fmt.Errorf("uuid: incorrrect UUID length: %s", t) - } -} - -// MarshalBinary implements the encoding.BinaryMarshaler interface. -func (u UUID) MarshalBinary() (data []byte, err error) { - data = u.Bytes() - return -} - -// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface. -// It will return error if the slice isn't 16 bytes long. -func (u *UUID) UnmarshalBinary(data []byte) (err error) { - if len(data) != Size { - err = fmt.Errorf("uuid: UUID must be exactly 16 bytes long, got %d bytes", len(data)) - return - } - copy(u[:], data) - - return -} diff --git a/vendor/github.com/satori/go.uuid/generator.go b/vendor/github.com/satori/go.uuid/generator.go deleted file mode 100644 index 3f2f1da2d..000000000 --- a/vendor/github.com/satori/go.uuid/generator.go +++ /dev/null @@ -1,239 +0,0 @@ -// Copyright (C) 2013-2018 by Maxim Bublis -// -// Permission is hereby granted, free of charge, to any person obtaining -// a copy of this software and associated documentation files (the -// "Software"), to deal in the Software without restriction, including -// without limitation the rights to use, copy, modify, merge, publish, -// distribute, sublicense, and/or sell copies of the Software, and to -// permit persons to whom the Software is furnished to do so, subject to -// the following conditions: -// -// The above copyright notice and this permission notice shall be -// included in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -package uuid - -import ( - "crypto/md5" - "crypto/rand" - "crypto/sha1" - "encoding/binary" - "hash" - "net" - "os" - "sync" - "time" -) - -// Difference in 100-nanosecond intervals between -// UUID epoch (October 15, 1582) and Unix epoch (January 1, 1970). -const epochStart = 122192928000000000 - -var ( - global = newDefaultGenerator() - - epochFunc = unixTimeFunc - posixUID = uint32(os.Getuid()) - posixGID = uint32(os.Getgid()) -) - -// NewV1 returns UUID based on current timestamp and MAC address. -func NewV1() UUID { - return global.NewV1() -} - -// NewV2 returns DCE Security UUID based on POSIX UID/GID. -func NewV2(domain byte) UUID { - return global.NewV2(domain) -} - -// NewV3 returns UUID based on MD5 hash of namespace UUID and name. -func NewV3(ns UUID, name string) UUID { - return global.NewV3(ns, name) -} - -// NewV4 returns random generated UUID. -func NewV4() UUID { - return global.NewV4() -} - -// NewV5 returns UUID based on SHA-1 hash of namespace UUID and name. -func NewV5(ns UUID, name string) UUID { - return global.NewV5(ns, name) -} - -// Generator provides interface for generating UUIDs. -type Generator interface { - NewV1() UUID - NewV2(domain byte) UUID - NewV3(ns UUID, name string) UUID - NewV4() UUID - NewV5(ns UUID, name string) UUID -} - -// Default generator implementation. -type generator struct { - storageOnce sync.Once - storageMutex sync.Mutex - - lastTime uint64 - clockSequence uint16 - hardwareAddr [6]byte -} - -func newDefaultGenerator() Generator { - return &generator{} -} - -// NewV1 returns UUID based on current timestamp and MAC address. -func (g *generator) NewV1() UUID { - u := UUID{} - - timeNow, clockSeq, hardwareAddr := g.getStorage() - - binary.BigEndian.PutUint32(u[0:], uint32(timeNow)) - binary.BigEndian.PutUint16(u[4:], uint16(timeNow>>32)) - binary.BigEndian.PutUint16(u[6:], uint16(timeNow>>48)) - binary.BigEndian.PutUint16(u[8:], clockSeq) - - copy(u[10:], hardwareAddr) - - u.SetVersion(V1) - u.SetVariant(VariantRFC4122) - - return u -} - -// NewV2 returns DCE Security UUID based on POSIX UID/GID. -func (g *generator) NewV2(domain byte) UUID { - u := UUID{} - - timeNow, clockSeq, hardwareAddr := g.getStorage() - - switch domain { - case DomainPerson: - binary.BigEndian.PutUint32(u[0:], posixUID) - case DomainGroup: - binary.BigEndian.PutUint32(u[0:], posixGID) - } - - binary.BigEndian.PutUint16(u[4:], uint16(timeNow>>32)) - binary.BigEndian.PutUint16(u[6:], uint16(timeNow>>48)) - binary.BigEndian.PutUint16(u[8:], clockSeq) - u[9] = domain - - copy(u[10:], hardwareAddr) - - u.SetVersion(V2) - u.SetVariant(VariantRFC4122) - - return u -} - -// NewV3 returns UUID based on MD5 hash of namespace UUID and name. -func (g *generator) NewV3(ns UUID, name string) UUID { - u := newFromHash(md5.New(), ns, name) - u.SetVersion(V3) - u.SetVariant(VariantRFC4122) - - return u -} - -// NewV4 returns random generated UUID. -func (g *generator) NewV4() UUID { - u := UUID{} - g.safeRandom(u[:]) - u.SetVersion(V4) - u.SetVariant(VariantRFC4122) - - return u -} - -// NewV5 returns UUID based on SHA-1 hash of namespace UUID and name. -func (g *generator) NewV5(ns UUID, name string) UUID { - u := newFromHash(sha1.New(), ns, name) - u.SetVersion(V5) - u.SetVariant(VariantRFC4122) - - return u -} - -func (g *generator) initStorage() { - g.initClockSequence() - g.initHardwareAddr() -} - -func (g *generator) initClockSequence() { - buf := make([]byte, 2) - g.safeRandom(buf) - g.clockSequence = binary.BigEndian.Uint16(buf) -} - -func (g *generator) initHardwareAddr() { - interfaces, err := net.Interfaces() - if err == nil { - for _, iface := range interfaces { - if len(iface.HardwareAddr) >= 6 { - copy(g.hardwareAddr[:], iface.HardwareAddr) - return - } - } - } - - // Initialize hardwareAddr randomly in case - // of real network interfaces absence - g.safeRandom(g.hardwareAddr[:]) - - // Set multicast bit as recommended in RFC 4122 - g.hardwareAddr[0] |= 0x01 -} - -func (g *generator) safeRandom(dest []byte) { - if _, err := rand.Read(dest); err != nil { - panic(err) - } -} - -// Returns UUID v1/v2 storage state. -// Returns epoch timestamp, clock sequence, and hardware address. -func (g *generator) getStorage() (uint64, uint16, []byte) { - g.storageOnce.Do(g.initStorage) - - g.storageMutex.Lock() - defer g.storageMutex.Unlock() - - timeNow := epochFunc() - // Clock changed backwards since last UUID generation. - // Should increase clock sequence. - if timeNow <= g.lastTime { - g.clockSequence++ - } - g.lastTime = timeNow - - return timeNow, g.clockSequence, g.hardwareAddr[:] -} - -// Returns difference in 100-nanosecond intervals between -// UUID epoch (October 15, 1582) and current time. -// This is default epoch calculation function. -func unixTimeFunc() uint64 { - return epochStart + uint64(time.Now().UnixNano()/100) -} - -// Returns UUID based on hashing of namespace UUID and name. -func newFromHash(h hash.Hash, ns UUID, name string) UUID { - u := UUID{} - h.Write(ns[:]) - h.Write([]byte(name)) - copy(u[:], h.Sum(nil)) - - return u -} diff --git a/vendor/github.com/satori/go.uuid/sql.go b/vendor/github.com/satori/go.uuid/sql.go deleted file mode 100644 index 56759d390..000000000 --- a/vendor/github.com/satori/go.uuid/sql.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright (C) 2013-2018 by Maxim Bublis -// -// Permission is hereby granted, free of charge, to any person obtaining -// a copy of this software and associated documentation files (the -// "Software"), to deal in the Software without restriction, including -// without limitation the rights to use, copy, modify, merge, publish, -// distribute, sublicense, and/or sell copies of the Software, and to -// permit persons to whom the Software is furnished to do so, subject to -// the following conditions: -// -// The above copyright notice and this permission notice shall be -// included in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -package uuid - -import ( - "database/sql/driver" - "fmt" -) - -// Value implements the driver.Valuer interface. -func (u UUID) Value() (driver.Value, error) { - return u.String(), nil -} - -// Scan implements the sql.Scanner interface. -// A 16-byte slice is handled by UnmarshalBinary, while -// a longer byte slice or a string is handled by UnmarshalText. -func (u *UUID) Scan(src interface{}) error { - switch src := src.(type) { - case []byte: - if len(src) == Size { - return u.UnmarshalBinary(src) - } - return u.UnmarshalText(src) - - case string: - return u.UnmarshalText([]byte(src)) - } - - return fmt.Errorf("uuid: cannot convert %T to UUID", src) -} - -// NullUUID can be used with the standard sql package to represent a -// UUID value that can be NULL in the database -type NullUUID struct { - UUID UUID - Valid bool -} - -// Value implements the driver.Valuer interface. -func (u NullUUID) Value() (driver.Value, error) { - if !u.Valid { - return nil, nil - } - // Delegate to UUID Value function - return u.UUID.Value() -} - -// Scan implements the sql.Scanner interface. -func (u *NullUUID) Scan(src interface{}) error { - if src == nil { - u.UUID, u.Valid = Nil, false - return nil - } - - // Delegate to UUID Scan function - u.Valid = true - return u.UUID.Scan(src) -} diff --git a/vendor/github.com/satori/go.uuid/uuid.go b/vendor/github.com/satori/go.uuid/uuid.go deleted file mode 100644 index a2b8e2ca2..000000000 --- a/vendor/github.com/satori/go.uuid/uuid.go +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright (C) 2013-2018 by Maxim Bublis -// -// Permission is hereby granted, free of charge, to any person obtaining -// a copy of this software and associated documentation files (the -// "Software"), to deal in the Software without restriction, including -// without limitation the rights to use, copy, modify, merge, publish, -// distribute, sublicense, and/or sell copies of the Software, and to -// permit persons to whom the Software is furnished to do so, subject to -// the following conditions: -// -// The above copyright notice and this permission notice shall be -// included in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -// Package uuid provides implementation of Universally Unique Identifier (UUID). -// Supported versions are 1, 3, 4 and 5 (as specified in RFC 4122) and -// version 2 (as specified in DCE 1.1). -package uuid - -import ( - "bytes" - "encoding/hex" -) - -// Size of a UUID in bytes. -const Size = 16 - -// UUID representation compliant with specification -// described in RFC 4122. -type UUID [Size]byte - -// UUID versions -const ( - _ byte = iota - V1 - V2 - V3 - V4 - V5 -) - -// UUID layout variants. -const ( - VariantNCS byte = iota - VariantRFC4122 - VariantMicrosoft - VariantFuture -) - -// UUID DCE domains. -const ( - DomainPerson = iota - DomainGroup - DomainOrg -) - -// String parse helpers. -var ( - urnPrefix = []byte("urn:uuid:") - byteGroups = []int{8, 4, 4, 4, 12} -) - -// Nil is special form of UUID that is specified to have all -// 128 bits set to zero. -var Nil = UUID{} - -// Predefined namespace UUIDs. -var ( - NamespaceDNS = Must(FromString("6ba7b810-9dad-11d1-80b4-00c04fd430c8")) - NamespaceURL = Must(FromString("6ba7b811-9dad-11d1-80b4-00c04fd430c8")) - NamespaceOID = Must(FromString("6ba7b812-9dad-11d1-80b4-00c04fd430c8")) - NamespaceX500 = Must(FromString("6ba7b814-9dad-11d1-80b4-00c04fd430c8")) -) - -// Equal returns true if u1 and u2 equals, otherwise returns false. -func Equal(u1 UUID, u2 UUID) bool { - return bytes.Equal(u1[:], u2[:]) -} - -// Version returns algorithm version used to generate UUID. -func (u UUID) Version() byte { - return u[6] >> 4 -} - -// Variant returns UUID layout variant. -func (u UUID) Variant() byte { - switch { - case (u[8] >> 7) == 0x00: - return VariantNCS - case (u[8] >> 6) == 0x02: - return VariantRFC4122 - case (u[8] >> 5) == 0x06: - return VariantMicrosoft - case (u[8] >> 5) == 0x07: - fallthrough - default: - return VariantFuture - } -} - -// Bytes returns bytes slice representation of UUID. -func (u UUID) Bytes() []byte { - return u[:] -} - -// Returns canonical string representation of UUID: -// xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx. -func (u UUID) String() string { - buf := make([]byte, 36) - - hex.Encode(buf[0:8], u[0:4]) - buf[8] = '-' - hex.Encode(buf[9:13], u[4:6]) - buf[13] = '-' - hex.Encode(buf[14:18], u[6:8]) - buf[18] = '-' - hex.Encode(buf[19:23], u[8:10]) - buf[23] = '-' - hex.Encode(buf[24:], u[10:]) - - return string(buf) -} - -// SetVersion sets version bits. -func (u *UUID) SetVersion(v byte) { - u[6] = (u[6] & 0x0f) | (v << 4) -} - -// SetVariant sets variant bits. -func (u *UUID) SetVariant(v byte) { - switch v { - case VariantNCS: - u[8] = (u[8]&(0xff>>1) | (0x00 << 7)) - case VariantRFC4122: - u[8] = (u[8]&(0xff>>2) | (0x02 << 6)) - case VariantMicrosoft: - u[8] = (u[8]&(0xff>>3) | (0x06 << 5)) - case VariantFuture: - fallthrough - default: - u[8] = (u[8]&(0xff>>3) | (0x07 << 5)) - } -} - -// Must is a helper that wraps a call to a function returning (UUID, error) -// and panics if the error is non-nil. It is intended for use in variable -// initializations such as -// var packageUUID = uuid.Must(uuid.FromString("123e4567-e89b-12d3-a456-426655440000")); -func Must(u UUID, err error) UUID { - if err != nil { - panic(err) - } - return u -} diff --git a/vendor/modules.txt b/vendor/modules.txt index 450f1ec8a..5cc85376c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -203,9 +203,6 @@ github.com/pmezard/go-difflib/difflib # github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c ## explicit; go 1.14 github.com/power-devops/perfstat -# github.com/satori/go.uuid v1.2.0 -## explicit -github.com/satori/go.uuid # github.com/shirou/gopsutil/v4 v4.25.1 ## explicit; go 1.18 github.com/shirou/gopsutil/v4/common From c34b437d4369d3de12fda1f1bd5536662c12ed79 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Tue, 30 Mar 2021 00:42:15 +0200 Subject: [PATCH 17/56] Comment-out WIP test --- go/mysql/binlog_test.go | 56 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/go/mysql/binlog_test.go b/go/mysql/binlog_test.go index 6aa9d31d4..7d7e8c54e 100644 --- a/go/mysql/binlog_test.go +++ b/go/mysql/binlog_test.go @@ -25,10 +25,10 @@ func TestBinlogCoordinates(t *testing.T) { c4 := BinlogCoordinates{LogFile: "mysql-bin.00112", LogPos: 104} gtidSet1, _ := gomysql.ParseMysqlGTIDSet("3E11FA47-71CA-11E1-9E33-C80AA9429562:23") - gtidSet2, _ := gomysql.ParseMysqlGTIDSet("3E11FA47-71CA-11E1-9E33-C80AA9429562:100") + //gtidSet2, _ := gomysql.ParseMysqlGTIDSet("3E11FA47-71CA-11E1-9E33-C80AA9429562:100") c5 := BinlogCoordinates{GTIDSet: gtidSet1} c6 := BinlogCoordinates{GTIDSet: gtidSet1} - c7 := BinlogCoordinates{GTIDSet: gtidSet2} + //c7 := BinlogCoordinates{GTIDSet: gtidSet2} require.True(t, c5.Equals(&c6)) require.True(t, c1.Equals(&c2)) @@ -48,6 +48,58 @@ func TestBinlogCoordinates(t *testing.T) { require.True(t, c6.SmallerThanOrEquals(&c7)) } +func TestBinlogNext(t *testing.T) { + c1 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 104} + cres, err := c1.NextFileCoordinates() + + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(c1.Type, cres.Type) + test.S(t).ExpectEquals(cres.LogFile, "mysql-bin.00018") + + c2 := BinlogCoordinates{LogFile: "mysql-bin.00099", LogPos: 104} + cres, err = c2.NextFileCoordinates() + + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(c1.Type, cres.Type) + test.S(t).ExpectEquals(cres.LogFile, "mysql-bin.00100") + + c3 := BinlogCoordinates{LogFile: "mysql.00.prod.com.00099", LogPos: 104} + cres, err = c3.NextFileCoordinates() + + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(c1.Type, cres.Type) + test.S(t).ExpectEquals(cres.LogFile, "mysql.00.prod.com.00100") +} + +func TestBinlogPrevious(t *testing.T) { + c1 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 104} + cres, err := c1.PreviousFileCoordinates() + + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(c1.Type, cres.Type) + test.S(t).ExpectEquals(cres.LogFile, "mysql-bin.00016") + + c2 := BinlogCoordinates{LogFile: "mysql-bin.00100", LogPos: 104} + cres, err = c2.PreviousFileCoordinates() + + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(c1.Type, cres.Type) + test.S(t).ExpectEquals(cres.LogFile, "mysql-bin.00099") + + c3 := BinlogCoordinates{LogFile: "mysql.00.prod.com.00100", LogPos: 104} + cres, err = c3.PreviousFileCoordinates() + + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(c1.Type, cres.Type) + test.S(t).ExpectEquals(cres.LogFile, "mysql.00.prod.com.00099") + + c4 := BinlogCoordinates{LogFile: "mysql.00.prod.com.00000", LogPos: 104} + _, err = c4.PreviousFileCoordinates() + + test.S(t).ExpectNotNil(err) +>>>>>>> 967ced57 (Comment-out WIP test) +} + func TestBinlogCoordinatesAsKey(t *testing.T) { m := make(map[BinlogCoordinates]bool) From 49e9920ad09a8f73869666f7a250b05535b0947b Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Thu, 1 Apr 2021 15:40:14 +0200 Subject: [PATCH 18/56] Fail on SID change --- go/binlog/gomysql_reader.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index 0d3eacbd1..4bf217e2e 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -28,6 +28,7 @@ type GoMySQLReader struct { binlogStreamer *replication.BinlogStreamer currentCoordinates mysql.BinlogCoordinates currentCoordinatesMutex *sync.Mutex + lastGtidSID string LastAppliedRowsEventHint mysql.BinlogCoordinates } @@ -163,6 +164,9 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha if err != nil { return err } + if this.lastGtidSID != "" && sid.String() != this.lastGtidSID { + return errors.New("GTID SID change is currently unsupported") + } func() { this.currentCoordinatesMutex.Lock() defer this.currentCoordinatesMutex.Unlock() @@ -174,6 +178,7 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha }), }, } + this.lastGtidSID = sid.String() }() case *replication.RotateEvent: if this.migrationContext.UseGTIDs { From 56e90800f60bdc76a2fb0887a0a6c132738bd5a0 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Thu, 1 Apr 2021 16:37:54 +0200 Subject: [PATCH 19/56] Fix import err --- go/binlog/gomysql_reader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index 4bf217e2e..9ef091169 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -165,7 +165,7 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha return err } if this.lastGtidSID != "" && sid.String() != this.lastGtidSID { - return errors.New("GTID SID change is currently unsupported") + return fmt.Errorf("Got unexpected GTID SID %q. SID change is currently unsupported", sid.String()) } func() { this.currentCoordinatesMutex.Lock() From 6e2c5b6c8337949efa0058f9e48631b507e1e0e5 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Fri, 2 Apr 2021 00:38:14 +0200 Subject: [PATCH 20/56] Add missing .Equals() check --- go/mysql/binlog.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go/mysql/binlog.go b/go/mysql/binlog.go index 50de122f3..25a7adc65 100644 --- a/go/mysql/binlog.go +++ b/go/mysql/binlog.go @@ -62,6 +62,8 @@ func (this *BinlogCoordinates) Equals(other *BinlogCoordinates) bool { } if this.GTIDSet != nil && !this.GTIDSet.Equal(other.GTIDSet) { return false + } else if other.GTIDSet != nil { + return false } return this.LogFile == other.LogFile && this.LogPos == other.LogPos } From 77d2c58106400955d4409603d9971a8e34166965 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sat, 3 Apr 2021 00:53:44 +0200 Subject: [PATCH 21/56] Fix .SmallerThan()/.SmallerThanEquals() funcs and tests --- go/binlog/gomysql_reader.go | 6 +++--- go/mysql/binlog.go | 37 ++++++++++++++++++++++++++++--------- go/mysql/binlog_test.go | 13 ++++++++----- go/mysql/utils.go | 2 +- 4 files changed, 40 insertions(+), 18 deletions(-) diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index 9ef091169..c155302b8 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -28,7 +28,7 @@ type GoMySQLReader struct { binlogStreamer *replication.BinlogStreamer currentCoordinates mysql.BinlogCoordinates currentCoordinatesMutex *sync.Mutex - lastGtidSID string + lastGtidSID *uuid.UUID LastAppliedRowsEventHint mysql.BinlogCoordinates } @@ -164,7 +164,7 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha if err != nil { return err } - if this.lastGtidSID != "" && sid.String() != this.lastGtidSID { + if this.lastGtidSID != nil && sid.String() != this.lastGtidSID.String() { return fmt.Errorf("Got unexpected GTID SID %q. SID change is currently unsupported", sid.String()) } func() { @@ -178,7 +178,7 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha }), }, } - this.lastGtidSID = sid.String() + this.lastGtidSID = &sid }() case *replication.RotateEvent: if this.migrationContext.UseGTIDs { diff --git a/go/mysql/binlog.go b/go/mysql/binlog.go index 25a7adc65..f6fc1d4f9 100644 --- a/go/mysql/binlog.go +++ b/go/mysql/binlog.go @@ -39,7 +39,7 @@ func ParseFileBinlogCoordinates(logFileLogPos string) (*BinlogCoordinates, error // ParseGTIDSetBinlogCoordinates parses a MySQL GTID set into a *BinlogCoordinates struct. func ParseGTIDSetBinlogCoordinates(gtidSet string) (*BinlogCoordinates, error) { set, err := gomysql.ParseMysqlGTIDSet(gtidSet) - return &BinlogCoordinates{GTIDSet: set}, err + return &BinlogCoordinates{GTIDSet: set.(*gomysql.MysqlGTIDSet)}, err } // DisplayString returns a user-friendly string representation of these coordinates @@ -60,10 +60,8 @@ func (this *BinlogCoordinates) Equals(other *BinlogCoordinates) bool { if other == nil { return false } - if this.GTIDSet != nil && !this.GTIDSet.Equal(other.GTIDSet) { - return false - } else if other.GTIDSet != nil { - return false + if this.GTIDSet != nil { + return this.GTIDSet.Equal(other.GTIDSet) } return this.LogFile == other.LogFile && this.LogPos == other.LogPos } @@ -75,6 +73,29 @@ func (this *BinlogCoordinates) IsEmpty() bool { // SmallerThan returns true if this coordinate is strictly smaller than the other. func (this *BinlogCoordinates) SmallerThan(other *BinlogCoordinates) bool { + // if GTID SIDs are equal we compare the interval stop points + // if GTID SIDs differ we have to assume there is a new/larger event + if this.GTIDSet != nil { + for sid, otherSet := range other.GTIDSet.Sets { + thisSet, ok := this.GTIDSet.Sets[sid] + if !ok { + return true // 'this' is missing a set + } + if len(thisSet.Intervals) < len(otherSet.Intervals) { + return true // 'this' has fewer intervals + } + for i, otherInterval := range otherSet.Intervals { + thisInterval := thisSet.Intervals[i] + if thisInterval.Start < otherInterval.Start { + return true + } + if thisInterval.Stop < otherInterval.Stop { + return true + } + } + } + } + if this.LogFile < other.LogFile { return true } @@ -90,10 +111,8 @@ func (this *BinlogCoordinates) SmallerThanOrEquals(other *BinlogCoordinates) boo if this.SmallerThan(other) { return true } - if this.GTIDSet != nil && !this.GTIDSet.Equal(other.GTIDSet) { - return false - } else if other.GTIDSet != nil { - return false + if this.GTIDSet != nil { + return this.GTIDSet.Equal(other.GTIDSet) } return this.LogFile == other.LogFile && this.LogPos == other.LogPos // No Type comparison } diff --git a/go/mysql/binlog_test.go b/go/mysql/binlog_test.go index 7d7e8c54e..9e2eb420c 100644 --- a/go/mysql/binlog_test.go +++ b/go/mysql/binlog_test.go @@ -1,5 +1,5 @@ /* - Copyright 2016 GitHub Inc. + Copyright 2021 GitHub Inc. See https://github.com/github/gh-ost/blob/master/LICENSE */ @@ -25,10 +25,12 @@ func TestBinlogCoordinates(t *testing.T) { c4 := BinlogCoordinates{LogFile: "mysql-bin.00112", LogPos: 104} gtidSet1, _ := gomysql.ParseMysqlGTIDSet("3E11FA47-71CA-11E1-9E33-C80AA9429562:23") - //gtidSet2, _ := gomysql.ParseMysqlGTIDSet("3E11FA47-71CA-11E1-9E33-C80AA9429562:100") - c5 := BinlogCoordinates{GTIDSet: gtidSet1} - c6 := BinlogCoordinates{GTIDSet: gtidSet1} - //c7 := BinlogCoordinates{GTIDSet: gtidSet2} + gtidSet2, _ := gomysql.ParseMysqlGTIDSet("3E11FA47-71CA-11E1-9E33-C80AA9429562:100") + gtidSet3, _ := gomysql.ParseMysqlGTIDSet("7F80FA47-FF33-71A1-AE01-B80CC7823548:100") + c5 := BinlogCoordinates{GTIDSet: gtidSet1.(*gomysql.MysqlGTIDSet)} + c6 := BinlogCoordinates{GTIDSet: gtidSet1.(*gomysql.MysqlGTIDSet)} + c7 := BinlogCoordinates{GTIDSet: gtidSet2.(*gomysql.MysqlGTIDSet)} + c8 := BinlogCoordinates{GTIDSet: gtidSet3.(*gomysql.MysqlGTIDSet)} require.True(t, c5.Equals(&c6)) require.True(t, c1.Equals(&c2)) @@ -46,6 +48,7 @@ func TestBinlogCoordinates(t *testing.T) { require.True(t, c1.SmallerThanOrEquals(&c2)) require.True(t, c1.SmallerThanOrEquals(&c3)) require.True(t, c6.SmallerThanOrEquals(&c7)) + require.True(t, c7.SmallerThanOrEquals(&c8)) } func TestBinlogNext(t *testing.T) { diff --git a/go/mysql/utils.go b/go/mysql/utils.go index e70ce18e6..916040856 100644 --- a/go/mysql/utils.go +++ b/go/mysql/utils.go @@ -188,7 +188,7 @@ func GetSelfBinlogCoordinates(dbVersion string, db *gosql.DB) (selfBinlogCoordin if err != nil { return err } - selfBinlogCoordinates.GTIDSet = gtidSet + selfBinlogCoordinates.GTIDSet = gtidSet.(*gomysql.MysqlGTIDSet) } return nil }) From 29953f6415e0b1a98fc5b37d3003a69d465c6608 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sat, 3 Apr 2021 00:55:40 +0200 Subject: [PATCH 22/56] Fix type change issues --- go/binlog/binlog_entry.go | 2 +- go/logic/streamer.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go/binlog/binlog_entry.go b/go/binlog/binlog_entry.go index 5cdc51f0c..d0bc7428c 100644 --- a/go/binlog/binlog_entry.go +++ b/go/binlog/binlog_entry.go @@ -20,7 +20,7 @@ type BinlogEntry struct { } // NewBinlogEntry creates an empty, ready to go BinlogEntry object -func NewBinlogEntry(logFile string, logPos uint64, gtidSet gomysql.GTIDSet) *BinlogEntry { +func NewBinlogEntry(logFile string, logPos uint64, gtidSet *gomysql.MysqlGTIDSet) *BinlogEntry { binlogEntry := &BinlogEntry{ Coordinates: mysql.BinlogCoordinates{LogFile: logFile, LogPos: int64(logPos), GTIDSet: gtidSet}, } diff --git a/go/logic/streamer.go b/go/logic/streamer.go index 83fbfa467..8366202e2 100644 --- a/go/logic/streamer.go +++ b/go/logic/streamer.go @@ -157,7 +157,7 @@ func (this *EventsStreamer) readCurrentBinlogCoordinates() error { if err != nil { return err } - this.initialBinlogCoordinates.GTIDSet = gtidSet + this.initialBinlogCoordinates.GTIDSet = gtidSet.(*gomysql.MysqlGTIDSet) } foundMasterStatus = true From 9166deaee22ae9809b7f1e6766e8e428ba3b4b84 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sat, 3 Apr 2021 02:36:36 +0200 Subject: [PATCH 23/56] Fix panics --- go/mysql/binlog.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/go/mysql/binlog.go b/go/mysql/binlog.go index f6fc1d4f9..80b24494e 100644 --- a/go/mysql/binlog.go +++ b/go/mysql/binlog.go @@ -76,6 +76,9 @@ func (this *BinlogCoordinates) SmallerThan(other *BinlogCoordinates) bool { // if GTID SIDs are equal we compare the interval stop points // if GTID SIDs differ we have to assume there is a new/larger event if this.GTIDSet != nil { + if other.GTIDSet == nil || other.GTIDSet.Sets == nil { + return false + } for sid, otherSet := range other.GTIDSet.Sets { thisSet, ok := this.GTIDSet.Sets[sid] if !ok { @@ -112,6 +115,9 @@ func (this *BinlogCoordinates) SmallerThanOrEquals(other *BinlogCoordinates) boo return true } if this.GTIDSet != nil { + if other.GTIDSet == nil { + return false + } return this.GTIDSet.Equal(other.GTIDSet) } return this.LogFile == other.LogFile && this.LogPos == other.LogPos // No Type comparison From baf92e70750829035a9471bc288418673914ed7a Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sat, 3 Apr 2021 03:20:19 +0200 Subject: [PATCH 24/56] Cleanup .SmallerThan() --- go/mysql/binlog.go | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/go/mysql/binlog.go b/go/mysql/binlog.go index 80b24494e..2c61212d1 100644 --- a/go/mysql/binlog.go +++ b/go/mysql/binlog.go @@ -75,35 +75,38 @@ func (this *BinlogCoordinates) IsEmpty() bool { func (this *BinlogCoordinates) SmallerThan(other *BinlogCoordinates) bool { // if GTID SIDs are equal we compare the interval stop points // if GTID SIDs differ we have to assume there is a new/larger event - if this.GTIDSet != nil { + if this.GTIDSet != nil && this.GTIDSet.Sets != nil { if other.GTIDSet == nil || other.GTIDSet.Sets == nil { return false } + if len(this.GTIDSet.Sets) < len(other.GTIDSet.Sets) { + return true + } for sid, otherSet := range other.GTIDSet.Sets { thisSet, ok := this.GTIDSet.Sets[sid] if !ok { - return true // 'this' is missing a set + return true // 'this' is missing an SID } if len(thisSet.Intervals) < len(otherSet.Intervals) { return true // 'this' has fewer intervals } for i, otherInterval := range otherSet.Intervals { - thisInterval := thisSet.Intervals[i] - if thisInterval.Start < otherInterval.Start { + if len(thisSet.Intervals)-1 > i { return true } - if thisInterval.Stop < otherInterval.Stop { + thisInterval := thisSet.Intervals[i] + if thisInterval.Start < otherInterval.Start || thisInterval.Stop < otherInterval.Stop { return true } } } - } - - if this.LogFile < other.LogFile { - return true - } - if this.LogFile == other.LogFile && this.LogPos < other.LogPos { - return true + } else { + if this.LogFile < other.LogFile { + return true + } + if this.LogFile == other.LogFile && this.LogPos < other.LogPos { + return true + } } return false } From ec04b98ee0d0d1f8736b9ae66657cddeaede638a Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sat, 3 Apr 2021 03:26:06 +0200 Subject: [PATCH 25/56] Add missing check in .Equals() --- go/mysql/binlog.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/go/mysql/binlog.go b/go/mysql/binlog.go index 2c61212d1..7b3261c4b 100644 --- a/go/mysql/binlog.go +++ b/go/mysql/binlog.go @@ -61,6 +61,9 @@ func (this *BinlogCoordinates) Equals(other *BinlogCoordinates) bool { return false } if this.GTIDSet != nil { + if other.GTIDSet == nil { + return false + } return this.GTIDSet.Equal(other.GTIDSet) } return this.LogFile == other.LogFile && this.LogPos == other.LogPos From 089ced99edacd1604e7225b79d5a6c82bba5d873 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sat, 3 Apr 2021 21:26:28 +0200 Subject: [PATCH 26/56] Add large GTID sets to test --- go/binlog/gomysql_reader.go | 14 ++++++-------- go/mysql/binlog.go | 5 +---- go/mysql/binlog_test.go | 25 +++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index c155302b8..41064bc72 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -170,14 +170,12 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha func() { this.currentCoordinatesMutex.Lock() defer this.currentCoordinatesMutex.Unlock() - this.currentCoordinates.GTIDSet = &gomysql.MysqlGTIDSet{ - Sets: map[string]*gomysql.UUIDSet{ - sid.String(): gomysql.NewUUIDSet(sid, gomysql.Interval{ - Start: event.GNO, - Stop: event.GNO + 1, - }), - }, - } + gtidSet := gomysql.MysqlGTIDSet{Sets: map[string]*gomysql.UUIDSet{}} + gtidSet.AddSet(gomysql.NewUUIDSet(sid, gomysql.Interval{ + Start: event.GNO, + Stop: event.GNO + 1, + })) + this.currentCoordinates.GTIDSet = >idSet this.lastGtidSID = &sid }() case *replication.RotateEvent: diff --git a/go/mysql/binlog.go b/go/mysql/binlog.go index 7b3261c4b..ee34e8dc9 100644 --- a/go/mysql/binlog.go +++ b/go/mysql/binlog.go @@ -61,10 +61,7 @@ func (this *BinlogCoordinates) Equals(other *BinlogCoordinates) bool { return false } if this.GTIDSet != nil { - if other.GTIDSet == nil { - return false - } - return this.GTIDSet.Equal(other.GTIDSet) + return other.GTIDSet != nil && this.GTIDSet.Equal(other.GTIDSet) } return this.LogFile == other.LogFile && this.LogPos == other.LogPos } diff --git a/go/mysql/binlog_test.go b/go/mysql/binlog_test.go index 9e2eb420c..68c44b8f0 100644 --- a/go/mysql/binlog_test.go +++ b/go/mysql/binlog_test.go @@ -27,10 +27,33 @@ func TestBinlogCoordinates(t *testing.T) { gtidSet1, _ := gomysql.ParseMysqlGTIDSet("3E11FA47-71CA-11E1-9E33-C80AA9429562:23") gtidSet2, _ := gomysql.ParseMysqlGTIDSet("3E11FA47-71CA-11E1-9E33-C80AA9429562:100") gtidSet3, _ := gomysql.ParseMysqlGTIDSet("7F80FA47-FF33-71A1-AE01-B80CC7823548:100") + gtidSetBig1, _ := gomysql.ParseMysqlGTIDSet(`08dc06d7-c27c-11ea-b204-e4434b77a5ce:1-1497873603, +0b4ff540-a712-11ea-9857-e4434b2a1c98:1-4315312982, +19636248-246d-11e9-ab0d-0263df733a8e:1, +1c8cd5dd-8c79-11eb-ae94-e4434b27ee9c:1-18850436, +3342d1ad-bda0-11ea-ba96-e4434b28e6e0:1-475232304, +3bcd300c-c811-11e9-9970-e4434b714c24:1-6209943929, +418b92ed-d6f6-11e8-b18f-246e961e5ed0:1-3299395227, +4465ebe1-2bcc-11e9-8913-e4434b21c560:1-4724945648, +48e2bc1d-d66d-11e8-bf56-a0369f9437b8:1, +492e2980-4518-11e9-92c6-e4434b3eca94:1-4926754392`) + gtidSetBig2, _ := gomysql.ParseMysqlGTIDSet(`08dc06d7-c27c-11ea-b204-e4434b77a5ce:1-1497873603, +0b4ff540-a712-11ea-9857-e4434b2a1c98:1-4315312982, +19636248-246d-11e9-ab0d-0263df733a8e:1, +1c8cd5dd-8c79-11eb-ae94-e4434b27ee9c:1-18850436, +3342d1ad-bda0-11ea-ba96-e4434b28e6e0:1-475232304, +3bcd300c-c811-11e9-9970-e4434b714c24:1-6209943929, +418b92ed-d6f6-11e8-b18f-246e961e5ed0:1-3299395227, +4465ebe1-2bcc-11e9-8913-e4434b21c560:1-4724945648, +48e2bc1d-d66d-11e8-bf56-a0369f9437b8:1, +492e2980-4518-11e9-92c6-e4434b3eca94:1-4926754399`) + c5 := BinlogCoordinates{GTIDSet: gtidSet1.(*gomysql.MysqlGTIDSet)} c6 := BinlogCoordinates{GTIDSet: gtidSet1.(*gomysql.MysqlGTIDSet)} c7 := BinlogCoordinates{GTIDSet: gtidSet2.(*gomysql.MysqlGTIDSet)} c8 := BinlogCoordinates{GTIDSet: gtidSet3.(*gomysql.MysqlGTIDSet)} + c9 := BinlogCoordinates{GTIDSet: gtidSetBig1.(*gomysql.MysqlGTIDSet)} + c10 := BinlogCoordinates{GTIDSet: gtidSetBig2.(*gomysql.MysqlGTIDSet)} require.True(t, c5.Equals(&c6)) require.True(t, c1.Equals(&c2)) @@ -49,6 +72,8 @@ func TestBinlogCoordinates(t *testing.T) { require.True(t, c1.SmallerThanOrEquals(&c3)) require.True(t, c6.SmallerThanOrEquals(&c7)) require.True(t, c7.SmallerThanOrEquals(&c8)) + require.True(t, c9.SmallerThanOrEquals(&c9)) + require.True(t, c10.SmallerThanOrEquals(&c9)) } func TestBinlogNext(t *testing.T) { From 51e0276cdb219a4696747b23ca24df4bda82b2d2 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sat, 3 Apr 2021 21:32:24 +0200 Subject: [PATCH 27/56] Simplify if cond --- go/mysql/binlog.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/go/mysql/binlog.go b/go/mysql/binlog.go index ee34e8dc9..9f611cf63 100644 --- a/go/mysql/binlog.go +++ b/go/mysql/binlog.go @@ -118,10 +118,7 @@ func (this *BinlogCoordinates) SmallerThanOrEquals(other *BinlogCoordinates) boo return true } if this.GTIDSet != nil { - if other.GTIDSet == nil { - return false - } - return this.GTIDSet.Equal(other.GTIDSet) + return other.GTIDSet != nil && this.GTIDSet.Equal(other.GTIDSet) } return this.LogFile == other.LogFile && this.LogPos == other.LogPos // No Type comparison } From dee9dfe6bb9d5471ac2bac72b61c402fa84f7e02 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Fri, 9 Jul 2021 22:19:12 +0200 Subject: [PATCH 28/56] Add localtest/gtid --- localtests/gtid/create.sql | 19 +++++++++++++++++++ localtests/gtid/extra_args | 1 + localtests/gtid/gtid_mode | 1 + localtests/gtid/ignore_versions | 1 + localtests/test.sh | 29 +++++++++++++++++++++++++++++ 5 files changed, 51 insertions(+) create mode 100644 localtests/gtid/create.sql create mode 100644 localtests/gtid/extra_args create mode 100644 localtests/gtid/gtid_mode create mode 100644 localtests/gtid/ignore_versions diff --git a/localtests/gtid/create.sql b/localtests/gtid/create.sql new file mode 100644 index 000000000..c3e7f8d7d --- /dev/null +++ b/localtests/gtid/create.sql @@ -0,0 +1,19 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + t varchar(128) charset utf8mb4, + primary key(id) +) auto_increment=1; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test values (null, md5(rand())); +end ;; diff --git a/localtests/gtid/extra_args b/localtests/gtid/extra_args new file mode 100644 index 000000000..ac1e8e795 --- /dev/null +++ b/localtests/gtid/extra_args @@ -0,0 +1 @@ +--gtid diff --git a/localtests/gtid/gtid_mode b/localtests/gtid/gtid_mode new file mode 100644 index 000000000..76371f28f --- /dev/null +++ b/localtests/gtid/gtid_mode @@ -0,0 +1 @@ +ON diff --git a/localtests/gtid/ignore_versions b/localtests/gtid/ignore_versions new file mode 100644 index 000000000..7acd3f06f --- /dev/null +++ b/localtests/gtid/ignore_versions @@ -0,0 +1 @@ +(5.5) diff --git a/localtests/test.sh b/localtests/test.sh index e467c113b..2b7d9b393 100755 --- a/localtests/test.sh +++ b/localtests/test.sh @@ -136,6 +136,35 @@ test_single() { start_replication echo_dot + current_gtid_mode=$(gh-ost-test-mysql-master -s -s -e "select @@global.gtid_mode" 2>/dev/null) + target_gtid_mode=OFF + if [ -f $tests_path/$test_name/gtid_mode ] ; then + target_gtid_mode=$(cat $tests_path/$test_name/gtid_mode) + if [ "$current_gtid_mode" == "OFF" ] && [ "$target_gtid_mode" == "ON" ] ; then + gh-ost-test-mysql-master --default-character-set=utf8mb4 test -e "set @@global.enforce_gtid_consistency=ON" + gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "set @@global.enforce_gtid_consistency=ON" + for mode in "OFF_PERMISSIVE" "ON_PERMISSIVE" "ON"; do + gh-ost-test-mysql-master --default-character-set=utf8mb4 test -e "set @@global.gtid_mode=${mode}" + done + else + echo "gtid_mode transition from ${current_gtid_mode} to ${target_gtid_mode} is unsupported!" + exit 1 + fi + elif [ "$current_gtid_mode" == "ON" ] ; then + for mode in "ON_PERMISSIVE" "OFF_PERMISSIVE" "OFF"; do + gh-ost-test-mysql-master --default-character-set=utf8mb4 test -e "set @@global.gtid_mode=${mode}" + gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "set @@global.gtid_mode=${mode}" + done + gh-ost-test-mysql-master --default-character-set=utf8mb4 test -e "set @@global.enforce_gtid_consistency=OFF" + gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "set @@global.enforce_gtid_consistency=OFF" + fi + + current_gtid_mode=$(gh-ost-test-mysql-master -s -s -e "select @@global.gtid_mode" 2>/dev/null) + if [ ! -z "$current_gtid_mode" ] && [ "$current_gtid_mode" != "$target_gtid_mode" ] ; then + echo "gtid_transition from ${current_gtid_mode} to ${target_gtid_mode} failed!" + exit 1 + fi + if [ -f $tests_path/$test_name/sql_mode ] ; then gh-ost-test-mysql-master --default-character-set=utf8mb4 test -e "set @@global.sql_mode='$(cat $tests_path/$test_name/sql_mode)'" gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "set @@global.sql_mode='$(cat $tests_path/$test_name/sql_mode)'" From 0413c2a418ede7e931beaf87170e504672453c60 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Fri, 9 Jul 2021 22:49:28 +0200 Subject: [PATCH 29/56] simplify localtest gtid_mode check --- localtests/test.sh | 26 +++----------------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/localtests/test.sh b/localtests/test.sh index 2b7d9b393..ba01b43c3 100755 --- a/localtests/test.sh +++ b/localtests/test.sh @@ -136,33 +136,13 @@ test_single() { start_replication echo_dot - current_gtid_mode=$(gh-ost-test-mysql-master -s -s -e "select @@global.gtid_mode" 2>/dev/null) - target_gtid_mode=OFF if [ -f $tests_path/$test_name/gtid_mode ] ; then + current_gtid_mode=$(gh-ost-test-mysql-master -s -s -e "select @@global.gtid_mode") target_gtid_mode=$(cat $tests_path/$test_name/gtid_mode) - if [ "$current_gtid_mode" == "OFF" ] && [ "$target_gtid_mode" == "ON" ] ; then - gh-ost-test-mysql-master --default-character-set=utf8mb4 test -e "set @@global.enforce_gtid_consistency=ON" - gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "set @@global.enforce_gtid_consistency=ON" - for mode in "OFF_PERMISSIVE" "ON_PERMISSIVE" "ON"; do - gh-ost-test-mysql-master --default-character-set=utf8mb4 test -e "set @@global.gtid_mode=${mode}" - done - else - echo "gtid_mode transition from ${current_gtid_mode} to ${target_gtid_mode} is unsupported!" + if [ "$current_gtid_mode" != "$target_gtid_mode" ] ; then + echo "gtid_mode is ${current_gtid_mode}, expected ${target_gtid_mode}" exit 1 fi - elif [ "$current_gtid_mode" == "ON" ] ; then - for mode in "ON_PERMISSIVE" "OFF_PERMISSIVE" "OFF"; do - gh-ost-test-mysql-master --default-character-set=utf8mb4 test -e "set @@global.gtid_mode=${mode}" - gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "set @@global.gtid_mode=${mode}" - done - gh-ost-test-mysql-master --default-character-set=utf8mb4 test -e "set @@global.enforce_gtid_consistency=OFF" - gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "set @@global.enforce_gtid_consistency=OFF" - fi - - current_gtid_mode=$(gh-ost-test-mysql-master -s -s -e "select @@global.gtid_mode" 2>/dev/null) - if [ ! -z "$current_gtid_mode" ] && [ "$current_gtid_mode" != "$target_gtid_mode" ] ; then - echo "gtid_transition from ${current_gtid_mode} to ${target_gtid_mode} failed!" - exit 1 fi if [ -f $tests_path/$test_name/sql_mode ] ; then From d5a163d4eacca4d1212b433717a9d00588d99d0e Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Fri, 9 Jul 2021 23:04:25 +0200 Subject: [PATCH 30/56] print gtid config --- localtests/test.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/localtests/test.sh b/localtests/test.sh index ba01b43c3..18b9fb724 100755 --- a/localtests/test.sh +++ b/localtests/test.sh @@ -24,6 +24,7 @@ master_port= replica_host= replica_port= original_sql_mode= +current_gtid_mode= OPTIND=1 while getopts "b:s:d" OPTION @@ -56,6 +57,10 @@ verify_master_and_replica() { original_sql_mode="$(gh-ost-test-mysql-master -e "select @@global.sql_mode" -s -s)" echo "sql_mode on master is ${original_sql_mode}" + current_gtid_mode=$(gh-ost-test-mysql-master -s -s -e "select @@global.gtid_mode" 2>/dev/null || echo unsupported) + current_enforce_gtid_consistency=$(gh-ost-test-mysql-master -s -s -e "select @@global.enforce_gtid_consistency" 2>/dev/null || echo unsupported) + echo "gtid_mode on master is ${current_gtid_mode} with enforce_gtid_consistency=${enforce_gtid_consistency}" + echo "Gracefully sleeping for 3 seconds while replica is setting up..." sleep 3 @@ -137,7 +142,6 @@ test_single() { echo_dot if [ -f $tests_path/$test_name/gtid_mode ] ; then - current_gtid_mode=$(gh-ost-test-mysql-master -s -s -e "select @@global.gtid_mode") target_gtid_mode=$(cat $tests_path/$test_name/gtid_mode) if [ "$current_gtid_mode" != "$target_gtid_mode" ] ; then echo "gtid_mode is ${current_gtid_mode}, expected ${target_gtid_mode}" From d614b4009c76723baf305f5072fc8a90a38efdc6 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Fri, 9 Jul 2021 23:23:22 +0200 Subject: [PATCH 31/56] Fix var typo --- localtests/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/localtests/test.sh b/localtests/test.sh index 18b9fb724..58c6b1c3e 100755 --- a/localtests/test.sh +++ b/localtests/test.sh @@ -59,7 +59,7 @@ verify_master_and_replica() { current_gtid_mode=$(gh-ost-test-mysql-master -s -s -e "select @@global.gtid_mode" 2>/dev/null || echo unsupported) current_enforce_gtid_consistency=$(gh-ost-test-mysql-master -s -s -e "select @@global.enforce_gtid_consistency" 2>/dev/null || echo unsupported) - echo "gtid_mode on master is ${current_gtid_mode} with enforce_gtid_consistency=${enforce_gtid_consistency}" + echo "gtid_mode on master is ${current_gtid_mode} with enforce_gtid_consistency=${current_enforce_gtid_consistency}" echo "Gracefully sleeping for 3 seconds while replica is setting up..." sleep 3 From 551e6d0a6298d1cfd7387ac8024460264b3732fa Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Fri, 9 Jul 2021 23:30:39 +0200 Subject: [PATCH 32/56] support enforce_gtid_consistency=1 --- go/logic/inspect.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index cb8c1f62e..7ea5e0572 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -386,7 +386,7 @@ func (this *Inspector) validateBinlogsAndGTID() error { if err := this.db.QueryRow(query).Scan(&hasBinaryLogs, &this.migrationContext.OriginalBinlogFormat, >idMode, &enforceGtidConsistency); err != nil { return err } - if gtidMode != "ON" || enforceGtidConsistency != "ON" { + if gtidMode != "ON" || (enforceGtidConsistency != "ON" && enforceGtidConsistency != "1") { return fmt.Errorf("%s:%d must have gtid_mode=ON and enforce_gtid_consistency=ON to use GTID support", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) } } else { From 76691cb98772bd0ebacae8a0408fff2b25abbff8 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sun, 11 Jul 2021 01:58:23 +0200 Subject: [PATCH 33/56] fix test --- go/binlog/gomysql_reader.go | 35 ++++++----- go/logic/streamer.go | 6 +- localtests/test.sh | 3 + script/cibuild-gh-ost-replica-tests | 91 +++++++++++++++++++++++++++++ 4 files changed, 116 insertions(+), 19 deletions(-) create mode 100755 script/cibuild-gh-ost-replica-tests diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index 41064bc72..21d00cfea 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -28,7 +28,7 @@ type GoMySQLReader struct { binlogStreamer *replication.BinlogStreamer currentCoordinates mysql.BinlogCoordinates currentCoordinatesMutex *sync.Mutex - lastGtidSID *uuid.UUID + nextCoordinates mysql.BinlogCoordinates LastAppliedRowsEventHint mysql.BinlogCoordinates } @@ -85,13 +85,14 @@ func (this *GoMySQLReader) GetCurrentBinlogCoordinates() *mysql.BinlogCoordinate // StreamEvents func (this *GoMySQLReader) handleRowsEvent(ev *replication.BinlogEvent, rowsEvent *replication.RowsEvent, entriesChannel chan<- *BinlogEntry) error { - if this.currentCoordinates.IsLogPosOverflowBeyond4Bytes(&this.LastAppliedRowsEventHint) { - return fmt.Errorf("Unexpected rows event at %+v, the binlog end_log_pos is overflow 4 bytes", this.currentCoordinates) - } - - if this.currentCoordinates.SmallerThanOrEquals(&this.LastAppliedRowsEventHint) { - this.migrationContext.Log.Debugf("Skipping handled query at %+v", this.currentCoordinates) - return nil + if !this.migrationContext.UseGTIDs { + if this.currentCoordinates.IsLogPosOverflowBeyond4Bytes(&this.LastAppliedRowsEventHint) { + return fmt.Errorf("Unexpected rows event at %+v, the binlog end_log_pos is overflow 4 bytes", this.currentCoordinates) + } + if this.currentCoordinates.SmallerThanOrEquals(&this.LastAppliedRowsEventHint) { + this.migrationContext.Log.Debugf("Skipping handled query at %+v", this.currentCoordinates) + return nil + } } dml := ToEventDML(ev.Header.EventType.String()) @@ -131,6 +132,11 @@ func (this *GoMySQLReader) handleRowsEvent(ev *replication.BinlogEvent, rowsEven // In reality, reads will be synchronous entriesChannel <- binlogEntry } + if this.migrationContext.UseGTIDs { + this.currentCoordinatesMutex.Lock() + defer this.currentCoordinatesMutex.Unlock() + this.currentCoordinates = this.nextCoordinates + } this.LastAppliedRowsEventHint = this.currentCoordinates return nil } @@ -164,19 +170,12 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha if err != nil { return err } - if this.lastGtidSID != nil && sid.String() != this.lastGtidSID.String() { - return fmt.Errorf("Got unexpected GTID SID %q. SID change is currently unsupported", sid.String()) - } func() { this.currentCoordinatesMutex.Lock() defer this.currentCoordinatesMutex.Unlock() - gtidSet := gomysql.MysqlGTIDSet{Sets: map[string]*gomysql.UUIDSet{}} - gtidSet.AddSet(gomysql.NewUUIDSet(sid, gomysql.Interval{ - Start: event.GNO, - Stop: event.GNO + 1, - })) - this.currentCoordinates.GTIDSet = >idSet - this.lastGtidSID = &sid + this.nextCoordinates = this.currentCoordinates + interval := gomysql.Interval{Start: event.GNO, Stop: event.GNO + 1} + this.nextCoordinates.GTIDSet.AddSet(gomysql.NewUUIDSet(sid, interval)) }() case *replication.RotateEvent: if this.migrationContext.UseGTIDs { diff --git a/go/logic/streamer.go b/go/logic/streamer.go index 8366202e2..2a4b9d5b2 100644 --- a/go/logic/streamer.go +++ b/go/logic/streamer.go @@ -139,7 +139,11 @@ func (this *EventsStreamer) GetCurrentBinlogCoordinates() *mysql.BinlogCoordinat } func (this *EventsStreamer) GetReconnectBinlogCoordinates() *mysql.BinlogCoordinates { - return &mysql.BinlogCoordinates{LogFile: this.GetCurrentBinlogCoordinates().LogFile, LogPos: 4} + current := this.GetCurrentBinlogCoordinates() + if current.GTIDSet != nil { + return &mysql.BinlogCoordinates{GTIDSet: current.GTIDSet} + } + return &mysql.BinlogCoordinates{LogFile: current.LogFile, LogPos: 4} } // readCurrentBinlogCoordinates reads master status from hooked server diff --git a/localtests/test.sh b/localtests/test.sh index 58c6b1c3e..8410ef2d5 100755 --- a/localtests/test.sh +++ b/localtests/test.sh @@ -59,7 +59,10 @@ verify_master_and_replica() { current_gtid_mode=$(gh-ost-test-mysql-master -s -s -e "select @@global.gtid_mode" 2>/dev/null || echo unsupported) current_enforce_gtid_consistency=$(gh-ost-test-mysql-master -s -s -e "select @@global.enforce_gtid_consistency" 2>/dev/null || echo unsupported) + current_master_server_uuid=$(gh-ost-test-mysql-master -s -s -e "select @@global.server_uuid" 2>/dev/null || echo unsupported) + current_replica_server_uuid=$(gh-ost-test-mysql-replica -s -s -e "select @@global.server_uuid" 2>/dev/null || echo unsupported) echo "gtid_mode on master is ${current_gtid_mode} with enforce_gtid_consistency=${current_enforce_gtid_consistency}" + echo "server_uuid on master is ${current_master_server_uuid}, replica is ${current_replica_server_uuid}" echo "Gracefully sleeping for 3 seconds while replica is setting up..." sleep 3 diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests new file mode 100755 index 000000000..c4dbfd292 --- /dev/null +++ b/script/cibuild-gh-ost-replica-tests @@ -0,0 +1,91 @@ +#!/bin/bash + +set -e + +whoami + +fetch_ci_env() { + # Clone gh-ost-ci-env + # Only clone if not already running locally at latest commit + remote_commit=$(git ls-remote https://github.com/github/gh-ost-ci-env.git HEAD | cut -f1) + local_commit="unknown" + [ -d "gh-ost-ci-env" ] && local_commit=$(cd gh-ost-ci-env && git log --format="%H" -n 1) + + echo "remote commit is: $remote_commit" + echo "local commit is: $local_commit" + + if [ "$remote_commit" != "$local_commit" ] ; then + rm -rf ./gh-ost-ci-env + git clone https://github.com/github/gh-ost-ci-env.git + fi +} + +test_dbdeployer() { + gh-ost-ci-env/bin/linux/dbdeployer --version +} + +test_mysql_version() { + local mysql_version + mysql_version="$1" + + echo "##### Testing $mysql_version" + + echo "### Setting up sandbox for $mysql_version" + + find sandboxes -name "stop_all" | bash + + mkdir -p sandbox/binary + rm -rf sandbox/binary/* + gh-ost-ci-env/bin/linux/dbdeployer unpack gh-ost-ci-env/mysql-tarballs/"$mysql_version".tar.xz --sandbox-binary ${PWD}/sandbox/binary + + mkdir -p sandboxes + rm -rf sandboxes/* + + local mysql_version_num=${mysql_version#*-} + if echo "$mysql_version_num" | egrep "5[.]5[.]" ; then + gtid="" + else + gtid="--gtid" + fi + gh-ost-ci-env/bin/linux/dbdeployer deploy replication "$mysql_version_num" --nodes 2 --sandbox-binary ${PWD}/sandbox/binary --sandbox-home ${PWD}/sandboxes ${gtid} --my-cnf-options log_slave_updates --my-cnf-options log_bin --my-cnf-options binlog_format=ROW --sandbox-directory rsandbox + + sed '/sandboxes/d' -i gh-ost-ci-env/bin/gh-ost-test-mysql-master + echo 'sandboxes/rsandbox/m "$@"' >> gh-ost-ci-env/bin/gh-ost-test-mysql-master + + sed '/sandboxes/d' -i gh-ost-ci-env/bin/gh-ost-test-mysql-replica + echo 'sandboxes/rsandbox/s1 "$@"' >> gh-ost-ci-env/bin/gh-ost-test-mysql-replica + + export PATH="${PWD}/gh-ost-ci-env/bin/:${PATH}" + + gh-ost-test-mysql-master -uroot -e "create user 'gh-ost'@'%' identified by 'gh-ost'" + gh-ost-test-mysql-master -uroot -e "grant all on *.* to 'gh-ost'@'%'" + + echo "### Running gh-ost tests for $mysql_version" + ./localtests/test.sh -b bin/gh-ost + + find sandboxes -name "stop_all" | bash +} + +main() { + fetch_ci_env + test_dbdeployer + + echo "Building..." + . script/build + + # TEST_MYSQL_VERSION is set by the replica-tests CI job + if [ -z "$TEST_MYSQL_VERSION" ]; then + # Test all versions: + find gh-ost-ci-env/mysql-tarballs/ -name "*.tar.xz" | while read f ; do basename $f ".tar.xz" ; done | sort -r | while read mysql_version ; do + echo "found MySQL version: $mysql_version" + done + find gh-ost-ci-env/mysql-tarballs/ -name "*.tar.xz" | while read f ; do basename $f ".tar.xz" ; done | sort -r | while read mysql_version ; do + test_mysql_version "$mysql_version" + done + else + echo "found MySQL version: $TEST_MYSQL_VERSION" + test_mysql_version "$TEST_MYSQL_VERSION" + fi +} + +main From 472abbed3ac19df97199751efb506558f9b074de Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sun, 11 Jul 2021 03:09:34 +0200 Subject: [PATCH 34/56] Handle PreviousGTIDsEvent --- go/binlog/gomysql_reader.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index 21d00cfea..e79d363e2 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -162,6 +162,17 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha }() switch event := ev.Event.(type) { + case *replication.PreviousGTIDsEvent: + if !this.migrationContext.UseGTIDs { + continue + } + func() { + this.currentCoordinatesMutex.Lock() + defer this.currentCoordinatesMutex.Unlock() + if err := this.currentCoordinates.GTIDSet.Update(event.GTIDSets); err != nil { + this.migrationContext.Log.Errorf("Failed to parse GTID set: %v", err) + } + }() case *replication.GTIDEvent: if !this.migrationContext.UseGTIDs { continue From 7cd643b3f6b8c3a664a466b2704f7660ea4f147d Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Tue, 22 Feb 2022 16:56:47 +0100 Subject: [PATCH 35/56] gofmt --- go/binlog/gomysql_reader.go | 1 + go/mysql/binlog_test.go | 52 ------------------------------------- 2 files changed, 1 insertion(+), 52 deletions(-) diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index e79d363e2..2cc63682d 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -18,6 +18,7 @@ import ( gomysql "github.com/go-mysql-org/go-mysql/mysql" "github.com/go-mysql-org/go-mysql/replication" uuid "github.com/google/uuid" + "golang.org/x/net/context" ) diff --git a/go/mysql/binlog_test.go b/go/mysql/binlog_test.go index 68c44b8f0..54ae6e5c9 100644 --- a/go/mysql/binlog_test.go +++ b/go/mysql/binlog_test.go @@ -76,58 +76,6 @@ func TestBinlogCoordinates(t *testing.T) { require.True(t, c10.SmallerThanOrEquals(&c9)) } -func TestBinlogNext(t *testing.T) { - c1 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 104} - cres, err := c1.NextFileCoordinates() - - test.S(t).ExpectNil(err) - test.S(t).ExpectEquals(c1.Type, cres.Type) - test.S(t).ExpectEquals(cres.LogFile, "mysql-bin.00018") - - c2 := BinlogCoordinates{LogFile: "mysql-bin.00099", LogPos: 104} - cres, err = c2.NextFileCoordinates() - - test.S(t).ExpectNil(err) - test.S(t).ExpectEquals(c1.Type, cres.Type) - test.S(t).ExpectEquals(cres.LogFile, "mysql-bin.00100") - - c3 := BinlogCoordinates{LogFile: "mysql.00.prod.com.00099", LogPos: 104} - cres, err = c3.NextFileCoordinates() - - test.S(t).ExpectNil(err) - test.S(t).ExpectEquals(c1.Type, cres.Type) - test.S(t).ExpectEquals(cres.LogFile, "mysql.00.prod.com.00100") -} - -func TestBinlogPrevious(t *testing.T) { - c1 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 104} - cres, err := c1.PreviousFileCoordinates() - - test.S(t).ExpectNil(err) - test.S(t).ExpectEquals(c1.Type, cres.Type) - test.S(t).ExpectEquals(cres.LogFile, "mysql-bin.00016") - - c2 := BinlogCoordinates{LogFile: "mysql-bin.00100", LogPos: 104} - cres, err = c2.PreviousFileCoordinates() - - test.S(t).ExpectNil(err) - test.S(t).ExpectEquals(c1.Type, cres.Type) - test.S(t).ExpectEquals(cres.LogFile, "mysql-bin.00099") - - c3 := BinlogCoordinates{LogFile: "mysql.00.prod.com.00100", LogPos: 104} - cres, err = c3.PreviousFileCoordinates() - - test.S(t).ExpectNil(err) - test.S(t).ExpectEquals(c1.Type, cres.Type) - test.S(t).ExpectEquals(cres.LogFile, "mysql.00.prod.com.00099") - - c4 := BinlogCoordinates{LogFile: "mysql.00.prod.com.00000", LogPos: 104} - _, err = c4.PreviousFileCoordinates() - - test.S(t).ExpectNotNil(err) ->>>>>>> 967ced57 (Comment-out WIP test) -} - func TestBinlogCoordinatesAsKey(t *testing.T) { m := make(map[BinlogCoordinates]bool) From 9db9a32c3eee16cdf2b328e9fb7322139f1f2a80 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 23 Feb 2022 22:24:37 +0100 Subject: [PATCH 36/56] Create `validateGTIDConfig` func --- go/logic/inspect.go | 45 +++++++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 7ea5e0572..abad70c45 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -70,9 +70,14 @@ func (this *Inspector) InitDBConnections() (err error) { if err := this.validateGrants(); err != nil { return err } - if err := this.validateBinlogsAndGTID(); err != nil { + if err := this.validateBinlogs(); err != nil { return err } + if this.migrationContext.UseGTIDs { + if err := this.validateGTIDConfig(); err != nil { + return err + } + } if err := this.applyBinlogFormat(); err != nil { return err } @@ -377,23 +382,12 @@ func (this *Inspector) applyBinlogFormat() error { return nil } -// validateBinlogsAndGTID checks that binary log and optional GTID configuration is good to go -func (this *Inspector) validateBinlogsAndGTID() error { +// validateBinlogs checks that binary log configuration is good to go +func (this *Inspector) validateBinlogs() error { + query := `select @@global.log_bin, @@global.binlog_format` var hasBinaryLogs bool - if this.migrationContext.UseGTIDs { - var gtidMode, enforceGtidConsistency string - query := `select @@global.log_bin, @@global.binlog_format, @@global.gtid_mode, @@global.enforce_gtid_consistency` - if err := this.db.QueryRow(query).Scan(&hasBinaryLogs, &this.migrationContext.OriginalBinlogFormat, >idMode, &enforceGtidConsistency); err != nil { - return err - } - if gtidMode != "ON" || (enforceGtidConsistency != "ON" && enforceGtidConsistency != "1") { - return fmt.Errorf("%s:%d must have gtid_mode=ON and enforce_gtid_consistency=ON to use GTID support", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) - } - } else { - query := `select @@global.log_bin, @@global.binlog_format` - if err := this.db.QueryRow(query).Scan(&hasBinaryLogs, &this.migrationContext.OriginalBinlogFormat); err != nil { - return err - } + if err := this.db.QueryRow(query).Scan(&hasBinaryLogs, &this.migrationContext.OriginalBinlogFormat); err != nil { + return err } if !hasBinaryLogs { return fmt.Errorf("%s must have binary logs enabled", this.connectionConfig.Key.String()) @@ -416,7 +410,7 @@ func (this *Inspector) validateBinlogsAndGTID() error { } this.migrationContext.Log.Infof("%s has %s binlog_format. I will change it to ROW, and will NOT change it back, even in the event of failure.", this.connectionConfig.Key.String(), this.migrationContext.OriginalBinlogFormat) } - query := `select /* gh-ost */ @@global.binlog_row_image` + query = `select /* gh-ost */ @@global.binlog_row_image` if err := this.db.QueryRow(query).Scan(&this.migrationContext.OriginalBinlogRowImage); err != nil { return err } @@ -429,6 +423,21 @@ func (this *Inspector) validateBinlogsAndGTID() error { return nil } +// validateGTIDConfig checks that the GTID configuration is good to go +func (this *Inspector) validateGTIDConfig() error { + var enforceGtidConsistency, gtidMode string + query := `select @@global.gtid_mode, @@global.enforce_gtid_consistency` + if err := this.db.QueryRow(query).Scan(>idMode, &enforceGtidConsistency); err != nil { + return err + } + if gtidMode != "ON" || (enforceGtidConsistency != "ON" && enforceGtidConsistency != "1") { + return fmt.Errorf("%s:%d must have gtid_mode=ON and enforce_gtid_consistency=ON to use GTID support", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) + } + + this.migrationContext.Log.Infof("gtid config validated on %s:%d", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) + return nil +} + // validateLogSlaveUpdates checks that binary log log_slave_updates is set. This test is not required when migrating on replica or when migrating directly on master func (this *Inspector) validateLogSlaveUpdates() error { query := `select /* gh-ost */ @@global.log_slave_updates` From f472fbd966533650e4dc8d4513231d08e4fe5344 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 23 Feb 2022 23:08:10 +0100 Subject: [PATCH 37/56] Update copyrights --- go/binlog/binlog_entry.go | 2 +- go/logic/inspect.go | 4 ++-- go/mysql/utils.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go/binlog/binlog_entry.go b/go/binlog/binlog_entry.go index d0bc7428c..e4283901e 100644 --- a/go/binlog/binlog_entry.go +++ b/go/binlog/binlog_entry.go @@ -1,5 +1,5 @@ /* - Copyright 2021 GitHub Inc. + Copyright 2022 GitHub Inc. See https://github.com/github/gh-ost/blob/master/LICENSE */ diff --git a/go/logic/inspect.go b/go/logic/inspect.go index abad70c45..47ef5e892 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -431,10 +431,10 @@ func (this *Inspector) validateGTIDConfig() error { return err } if gtidMode != "ON" || (enforceGtidConsistency != "ON" && enforceGtidConsistency != "1") { - return fmt.Errorf("%s:%d must have gtid_mode=ON and enforce_gtid_consistency=ON to use GTID support", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) + return fmt.Errorf("%s must have gtid_mode=ON and enforce_gtid_consistency=ON to use GTID support", this.connectionConfig.Key.String()) } - this.migrationContext.Log.Infof("gtid config validated on %s:%d", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) + this.migrationContext.Log.Infof("gtid config validated on %s", this.connectionConfig.Key.String()) return nil } diff --git a/go/mysql/utils.go b/go/mysql/utils.go index 916040856..6a134e1f4 100644 --- a/go/mysql/utils.go +++ b/go/mysql/utils.go @@ -1,5 +1,5 @@ /* - Copyright 2021 GitHub Inc. + Copyright 2022 GitHub Inc. See https://github.com/github/gh-ost/blob/master/LICENSE */ From 13d218e6aceeb68947881e6d10c1167c29a44faa Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 23 Feb 2022 23:12:08 +0100 Subject: [PATCH 38/56] Update copyrights pt 2 --- go/mysql/binlog_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/mysql/binlog_test.go b/go/mysql/binlog_test.go index 54ae6e5c9..b2fd2377c 100644 --- a/go/mysql/binlog_test.go +++ b/go/mysql/binlog_test.go @@ -1,5 +1,5 @@ /* - Copyright 2021 GitHub Inc. + Copyright 2022 GitHub Inc. See https://github.com/github/gh-ost/blob/master/LICENSE */ From 713f70bf89f97d42d0ca7eb5dda3d582b9bd42f5 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Thu, 24 Feb 2022 02:09:01 +0100 Subject: [PATCH 39/56] Copyrights again --- go/base/context_test.go | 2 +- go/logic/applier.go | 2 +- go/logic/inspect.go | 2 +- go/logic/server.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go/base/context_test.go b/go/base/context_test.go index f3323b9c1..f87bc9f13 100644 --- a/go/base/context_test.go +++ b/go/base/context_test.go @@ -1,5 +1,5 @@ /* - Copyright 2022 GitHub Inc. + Copyright 2021 GitHub Inc. See https://github.com/github/gh-ost/blob/master/LICENSE */ diff --git a/go/logic/applier.go b/go/logic/applier.go index c575a2fb9..f2cef9d0f 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -1,5 +1,5 @@ /* - Copyright 2022 GitHub Inc. + Copyright 2021 GitHub Inc. See https://github.com/github/gh-ost/blob/master/LICENSE */ diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 47ef5e892..4e8d4724c 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -425,7 +425,7 @@ func (this *Inspector) validateBinlogs() error { // validateGTIDConfig checks that the GTID configuration is good to go func (this *Inspector) validateGTIDConfig() error { - var enforceGtidConsistency, gtidMode string + var gtidMode, enforceGtidConsistency string query := `select @@global.gtid_mode, @@global.enforce_gtid_consistency` if err := this.db.QueryRow(query).Scan(>idMode, &enforceGtidConsistency); err != nil { return err diff --git a/go/logic/server.go b/go/logic/server.go index b5d05b758..45e5b2bd4 100644 --- a/go/logic/server.go +++ b/go/logic/server.go @@ -1,5 +1,5 @@ /* - Copyright 2022 GitHub Inc. + Copyright 2021 GitHub Inc. See https://github.com/github/gh-ost/blob/master/LICENSE */ From a2b178b017fa2afc64152e865854f2b2e1c02e0c Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Thu, 24 Feb 2022 19:10:02 +0100 Subject: [PATCH 40/56] Update docs --- doc/command-line-flags.md | 2 +- go/binlog/gomysql_reader.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/command-line-flags.md b/doc/command-line-flags.md index 153c5265a..7b5efd9fb 100644 --- a/doc/command-line-flags.md +++ b/doc/command-line-flags.md @@ -162,7 +162,7 @@ Add this flag when executing on a 1st generation Google Cloud Platform (GCP). ### gtid -Add this flag to enable support for [MySQL GTIDs](https://dev.mysql.com/doc/refman/5.7/en/replication-gtids-concepts.html) for replication binlog positioning. This requires `gtid_mode` and `enforce_gtid_consistency` to be set to `ON`. +Add this flag to enable support for [MySQL replication GTIDs](https://dev.mysql.com/doc/refman/5.7/en/replication-gtids-concepts.html) for replication positioning. This requires `gtid_mode` and `enforce_gtid_consistency` to be set to `ON`. ### heartbeat-interval-millis diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index 2cc63682d..e79d363e2 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -18,7 +18,6 @@ import ( gomysql "github.com/go-mysql-org/go-mysql/mysql" "github.com/go-mysql-org/go-mysql/replication" uuid "github.com/google/uuid" - "golang.org/x/net/context" ) From a328c221301db6894323c46754778edd64b409b5 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Tue, 1 Mar 2022 00:59:10 +0100 Subject: [PATCH 41/56] WIP --- go/logic/inspect.go | 3 +- go/mysql/binlog.go | 147 +------------ go/mysql/binlog_file.go | 206 ++++++++++++++++++ .../{binlog_test.go => binlog_file_test.go} | 0 go/mysql/binlog_gtid.go | 84 +++++++ 5 files changed, 300 insertions(+), 140 deletions(-) create mode 100644 go/mysql/binlog_file.go rename go/mysql/{binlog_test.go => binlog_file_test.go} (100%) create mode 100644 go/mysql/binlog_gtid.go diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 4e8d4724c..f41f4032f 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -430,7 +430,8 @@ func (this *Inspector) validateGTIDConfig() error { if err := this.db.QueryRow(query).Scan(>idMode, &enforceGtidConsistency); err != nil { return err } - if gtidMode != "ON" || (enforceGtidConsistency != "ON" && enforceGtidConsistency != "1") { + enforceGtidConsistency = strings.ToUpper(enforceGtidConsistency) + if strings.ToUpper(gtidMode) != "ON" || (enforceGtidConsistency != "ON" && enforceGtidConsistency != "1") { return fmt.Errorf("%s must have gtid_mode=ON and enforce_gtid_consistency=ON to use GTID support", this.connectionConfig.Key.String()) } diff --git a/go/mysql/binlog.go b/go/mysql/binlog.go index 9f611cf63..9a9838fb7 100644 --- a/go/mysql/binlog.go +++ b/go/mysql/binlog.go @@ -6,143 +6,12 @@ package mysql -import ( - "fmt" - "strconv" - "strings" - - gomysql "github.com/go-mysql-org/go-mysql/mysql" -) - -// BinlogCoordinates described binary log coordinates in the form of a GTID set and/or log file & log position. -type BinlogCoordinates struct { - GTIDSet gomysql.GTIDSet - LogFile string - LogPos int64 - EventSize int64 -} - -// ParseFileBinlogCoordinates parses a log file/position string into a *BinlogCoordinates struct. -func ParseFileBinlogCoordinates(logFileLogPos string) (*BinlogCoordinates, error) { - tokens := strings.SplitN(logFileLogPos, ":", 2) - if len(tokens) != 2 { - return nil, fmt.Errorf("ParseFileBinlogCoordinates: Cannot parse BinlogCoordinates from %s. Expected format is file:pos", logFileLogPos) - } - - if logPos, err := strconv.ParseInt(tokens[1], 10, 0); err != nil { - return nil, fmt.Errorf("ParseFileBinlogCoordinates: invalid pos: %s", tokens[1]) - } else { - return &BinlogCoordinates{LogFile: tokens[0], LogPos: logPos}, nil - } -} - -// ParseGTIDSetBinlogCoordinates parses a MySQL GTID set into a *BinlogCoordinates struct. -func ParseGTIDSetBinlogCoordinates(gtidSet string) (*BinlogCoordinates, error) { - set, err := gomysql.ParseMysqlGTIDSet(gtidSet) - return &BinlogCoordinates{GTIDSet: set.(*gomysql.MysqlGTIDSet)}, err -} - -// DisplayString returns a user-friendly string representation of these coordinates -func (this *BinlogCoordinates) DisplayString() string { - if this.GTIDSet != nil { - return this.GTIDSet.String() - } - return fmt.Sprintf("%s:%d", this.LogFile, this.LogPos) -} - -// String returns a user-friendly string representation of these coordinates -func (this BinlogCoordinates) String() string { - return this.DisplayString() -} - -// Equals tests equality of this coordinate and another one. -func (this *BinlogCoordinates) Equals(other *BinlogCoordinates) bool { - if other == nil { - return false - } - if this.GTIDSet != nil { - return other.GTIDSet != nil && this.GTIDSet.Equal(other.GTIDSet) - } - return this.LogFile == other.LogFile && this.LogPos == other.LogPos -} - -// IsEmpty returns true if the log file and GTID set is empty, unnamed -func (this *BinlogCoordinates) IsEmpty() bool { - return this.LogFile == "" && this.GTIDSet == nil -} - -// SmallerThan returns true if this coordinate is strictly smaller than the other. -func (this *BinlogCoordinates) SmallerThan(other *BinlogCoordinates) bool { - // if GTID SIDs are equal we compare the interval stop points - // if GTID SIDs differ we have to assume there is a new/larger event - if this.GTIDSet != nil && this.GTIDSet.Sets != nil { - if other.GTIDSet == nil || other.GTIDSet.Sets == nil { - return false - } - if len(this.GTIDSet.Sets) < len(other.GTIDSet.Sets) { - return true - } - for sid, otherSet := range other.GTIDSet.Sets { - thisSet, ok := this.GTIDSet.Sets[sid] - if !ok { - return true // 'this' is missing an SID - } - if len(thisSet.Intervals) < len(otherSet.Intervals) { - return true // 'this' has fewer intervals - } - for i, otherInterval := range otherSet.Intervals { - if len(thisSet.Intervals)-1 > i { - return true - } - thisInterval := thisSet.Intervals[i] - if thisInterval.Start < otherInterval.Start || thisInterval.Stop < otherInterval.Stop { - return true - } - } - } - } else { - if this.LogFile < other.LogFile { - return true - } - if this.LogFile == other.LogFile && this.LogPos < other.LogPos { - return true - } - } - return false -} - -// SmallerThanOrEquals returns true if this coordinate is the same or equal to the other one. -// We do NOT compare the type so we can not use this.Equals() -func (this *BinlogCoordinates) SmallerThanOrEquals(other *BinlogCoordinates) bool { - if this.SmallerThan(other) { - return true - } - if this.GTIDSet != nil { - return other.GTIDSet != nil && this.GTIDSet.Equal(other.GTIDSet) - } - return this.LogFile == other.LogFile && this.LogPos == other.LogPos // No Type comparison -} - -// IsLogPosOverflowBeyond4Bytes returns true if the coordinate endpos is overflow beyond 4 bytes. -// The binlog event end_log_pos field type is defined as uint32, 4 bytes. -// https://github.com/go-mysql-org/go-mysql/blob/master/replication/event.go -// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_replication_binlog_event.html#sect_protocol_replication_binlog_event_header -// Issue: https://github.com/github/gh-ost/issues/1366 -func (this *BinlogCoordinates) IsLogPosOverflowBeyond4Bytes(preCoordinate *BinlogCoordinates) bool { - if preCoordinate == nil { - return false - } - if preCoordinate.IsEmpty() { - return false - } - - if this.LogFile != preCoordinate.LogFile { - return false - } - - if preCoordinate.LogPos+this.EventSize >= 1<<32 { - // Unexpected rows event, the previous binlog log_pos + current binlog event_size is overflow 4 bytes - return true - } - return false +type BinlogCoordinates interface { + Name() string + String() string + DisplayString() string + IsEmpty() bool + Equals(other BinlogCoordinates) bool + SmallerThan(other BinlogCoordinates) bool + SmallerThanOrEquals(other BinlogCoordinates) bool } diff --git a/go/mysql/binlog_file.go b/go/mysql/binlog_file.go new file mode 100644 index 000000000..b251a63d5 --- /dev/null +++ b/go/mysql/binlog_file.go @@ -0,0 +1,206 @@ +/* + Copyright 2015 Shlomi Noach, courtesy Booking.com + Copyright 2022 GitHub Inc. + See https://github.com/github/gh-ost/blob/master/LICENSE +*/ + +package mysql + +import ( + "errors" + "fmt" + "regexp" + "strconv" + "strings" +) + +var detachPattern *regexp.Regexp + +func init() { + detachPattern, _ = regexp.Compile(`//([^/:]+):([\d]+)`) // e.g. `//binlog.01234:567890` +} + +type BinlogType int + +const ( + BinaryLog BinlogType = iota + RelayLog +) + +// FileBinlogCoordinates described binary log coordinates in the form of a binlog file & log position. +type FileBinlogCoordinates struct { + LogFile string + LogPos int64 + Type BinlogType + EventSize int64 +} + +// ParseFileBinlogCoordinates parses a log file/position string into a *BinlogCoordinates struct. +func ParseFileBinlogCoordinates(logFileLogPos string) (*FileBinlogCoordinates, error) { + tokens := strings.SplitN(logFileLogPos, ":", 2) + if len(tokens) != 2 { + return nil, fmt.Errorf("ParseFileBinlogCoordinates: Cannot parse BinlogCoordinates from %s. Expected format is file:pos", logFileLogPos) + } + + if logPos, err := strconv.ParseInt(tokens[1], 10, 0); err != nil { + return nil, fmt.Errorf("ParseFileBinlogCoordinates: invalid pos: %s", tokens[1]) + } else { + return &FileBinlogCoordinates{LogFile: tokens[0], LogPos: logPos}, nil + } +} + +// Name returns the name of the BinlogCoordinates interface implementation +func (this *FileBinlogCoordinates) Name() string { + return "file" +} + +// DisplayString returns a user-friendly string representation of these coordinates +func (this *FileBinlogCoordinates) DisplayString() string { + return fmt.Sprintf("%s:%d", this.LogFile, this.LogPos) +} + +// String returns a user-friendly string representation of these coordinates +func (this FileBinlogCoordinates) String() string { + return this.DisplayString() +} + +// Equals tests equality of this coordinate and another one. +func (this *FileBinlogCoordinates) Equals(other BinlogCoordinates) bool { + coord, ok := other.(*FileBinlogCoordinates) + if !ok || other == nil { + return false + } + return this.LogFile == coord.LogFile && this.LogPos == coord.LogPos && this.Type == coord.Type +} + +// IsEmpty returns true if the log file is empty, unnamed +func (this *FileBinlogCoordinates) IsEmpty() bool { + return this.LogFile == "" +} + +// SmallerThan returns true if this coordinate is strictly smaller than the other. +func (this *FileBinlogCoordinates) SmallerThan(other BinlogCoordinates) bool { + coord, ok := other.(*FileBinlogCoordinates) + if !ok || other == nil { + return false + } + if this.LogFile < coord.LogFile { + return true + } + if this.LogFile == coord.LogFile && this.LogPos < coord.LogPos { + return true + } + return false +} + +// SmallerThanOrEquals returns true if this coordinate is the same or equal to the other one. +// We do NOT compare the type so we can not use this.Equals() +func (this *FileBinlogCoordinates) SmallerThanOrEquals(other BinlogCoordinates) bool { + coord, ok := other.(*FileBinlogCoordinates) + if !ok || other == nil { + return false + } + if this.SmallerThan(other) { + return true + } + return this.LogFile == coord.LogFile && this.LogPos == coord.LogPos // No Type comparison +} + +// FileSmallerThan returns true if this coordinate's file is strictly smaller than the other's. +func (this *FileBinlogCoordinates) FileSmallerThan(other BinlogCoordinates) bool { + coord, ok := other.(*FileBinlogCoordinates) + if !ok || other == nil { + return false + } + return this.LogFile < coord.LogFile +} + +// FileNumberDistance returns the numeric distance between this coordinate's file number and the other's. +// Effectively it means "how many rotates/FLUSHes would make these coordinates's file reach the other's" +func (this *FileBinlogCoordinates) FileNumberDistance(other *FileBinlogCoordinates) int { + thisNumber, _ := this.FileNumber() + otherNumber, _ := other.FileNumber() + return otherNumber - thisNumber +} + +// FileNumber returns the numeric value of the file, and the length in characters representing the number in the filename. +// Example: FileNumber() of mysqld.log.000789 is (789, 6) +func (this *FileBinlogCoordinates) FileNumber() (int, int) { + tokens := strings.Split(this.LogFile, ".") + numPart := tokens[len(tokens)-1] + numLen := len(numPart) + fileNum, err := strconv.Atoi(numPart) + if err != nil { + return 0, 0 + } + return fileNum, numLen +} + +// PreviousFileCoordinatesBy guesses the filename of the previous binlog/relaylog, by given offset (number of files back) +func (this *FileBinlogCoordinates) PreviousFileCoordinatesBy(offset int) (BinlogCoordinates, error) { + result := &FileBinlogCoordinates{LogPos: 0, Type: this.Type} + + fileNum, numLen := this.FileNumber() + if fileNum == 0 { + return result, errors.New("Log file number is zero, cannot detect previous file") + } + newNumStr := fmt.Sprintf("%d", (fileNum - offset)) + newNumStr = strings.Repeat("0", numLen-len(newNumStr)) + newNumStr + + tokens := strings.Split(this.LogFile, ".") + tokens[len(tokens)-1] = newNumStr + result.LogFile = strings.Join(tokens, ".") + return result, nil +} + +// PreviousFileCoordinates guesses the filename of the previous binlog/relaylog +func (this *FileBinlogCoordinates) PreviousFileCoordinates() (BinlogCoordinates, error) { + return this.PreviousFileCoordinatesBy(1) +} + +// PreviousFileCoordinates guesses the filename of the previous binlog/relaylog +func (this *FileBinlogCoordinates) NextFileCoordinates() (BinlogCoordinates, error) { + result := &FileBinlogCoordinates{LogPos: 0, Type: this.Type} + + fileNum, numLen := this.FileNumber() + newNumStr := fmt.Sprintf("%d", (fileNum + 1)) + newNumStr = strings.Repeat("0", numLen-len(newNumStr)) + newNumStr + + tokens := strings.Split(this.LogFile, ".") + tokens[len(tokens)-1] = newNumStr + result.LogFile = strings.Join(tokens, ".") + return result, nil +} + +// FileSmallerThan returns true if this coordinate's file is strictly smaller than the other's. +func (this *FileBinlogCoordinates) DetachedCoordinates() (isDetached bool, detachedLogFile string, detachedLogPos string) { + detachedCoordinatesSubmatch := detachPattern.FindStringSubmatch(this.LogFile) + if len(detachedCoordinatesSubmatch) == 0 { + return false, "", "" + } + return true, detachedCoordinatesSubmatch[1], detachedCoordinatesSubmatch[2] +} + +// IsLogPosOverflowBeyond4Bytes returns true if the coordinate endpos is overflow beyond 4 bytes. +// The binlog event end_log_pos field type is defined as uint32, 4 bytes. +// https://github.com/go-mysql-org/go-mysql/blob/master/replication/event.go +// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_replication_binlog_event.html#sect_protocol_replication_binlog_event_header +// Issue: https://github.com/github/gh-ost/issues/1366 +func (this *FileBinlogCoordinates) IsLogPosOverflowBeyond4Bytes(preCoordinate *FileBinlogCoordinates) bool { + if preCoordinate == nil { + return false + } + if preCoordinate.IsEmpty() { + return false + } + + if this.LogFile != preCoordinate.LogFile { + return false + } + + if preCoordinate.LogPos+this.EventSize >= 1<<32 { + // Unexpected rows event, the previous binlog log_pos + current binlog event_size is overflow 4 bytes + return true + } + return false +} diff --git a/go/mysql/binlog_test.go b/go/mysql/binlog_file_test.go similarity index 100% rename from go/mysql/binlog_test.go rename to go/mysql/binlog_file_test.go diff --git a/go/mysql/binlog_gtid.go b/go/mysql/binlog_gtid.go new file mode 100644 index 000000000..d66586ef8 --- /dev/null +++ b/go/mysql/binlog_gtid.go @@ -0,0 +1,84 @@ +/* + Copyright 2022 GitHub Inc. + See https://github.com/github/gh-ost/blob/master/LICENSE +*/ + +package mysql + +import ( + gomysql "github.com/go-mysql-org/go-mysql/mysql" +) + +// GTIDBinlogCoordinates described binary log coordinates in the form of a binlog file & log position. +type GTIDBinlogCoordinates struct { + Set *gomysql.MysqlGTIDSet +} + +// ParseGTIDSetBinlogCoordinates parses a MySQL GTID set into a *GTIDBinlogCoordinates struct. +func ParseGTIDSetBinlogCoordinates(gtidSet string) (*GTIDBinlogCoordinates, error) { + set, err := gomysql.ParseMysqlGTIDSet(gtidSet) + return >IDBinlogCoordinates{set.(*gomysql.MysqlGTIDSet)}, err +} + +// DisplayString returns a user-friendly string representation of these coordinates +func (this *GTIDBinlogCoordinates) DisplayString() string { + return this.Set.String() +} + +// String returns a user-friendly string representation of these coordinates +func (this GTIDBinlogCoordinates) String() string { + return this.DisplayString() +} + +// Equals tests equality of this coordinate and another one. +func (this *GTIDBinlogCoordinates) Equals(other *GTIDBinlogCoordinates) bool { + if other == nil { + return false + } + return other.Set != nil && this.Set.Equal(other.Set) +} + +// IsEmpty returns true if the GTID set is empty, unnamed +func (this *GTIDBinlogCoordinates) IsEmpty() bool { + return this.Set == nil +} + +// SmallerThan returns true if this coordinate is strictly smaller than the other. +func (this *GTIDBinlogCoordinates) SmallerThan(other *GTIDBinlogCoordinates) bool { + // if GTID SIDs are equal we compare the interval stop points + // if GTID SIDs differ we have to assume there is a new/larger event + if other.Set == nil || other.Set.Sets == nil { + return false + } + if len(this.Set.Sets) < len(other.Set.Sets) { + return true + } + for sid, otherSet := range other.Set.Sets { + thisSet, ok := this.Set.Sets[sid] + if !ok { + return true // 'this' is missing an SID + } + if len(thisSet.Intervals) < len(otherSet.Intervals) { + return true // 'this' has fewer intervals + } + for i, otherInterval := range otherSet.Intervals { + if len(thisSet.Intervals)-1 > i { + return true + } + thisInterval := thisSet.Intervals[i] + if thisInterval.Start < otherInterval.Start || thisInterval.Stop < otherInterval.Stop { + return true + } + } + } + return false +} + +// SmallerThanOrEquals returns true if this coordinate is the same or equal to the other one. +// We do NOT compare the type so we can not use this.Equals() +func (this *GTIDBinlogCoordinates) SmallerThanOrEquals(other *GTIDBinlogCoordinates) bool { + if this.SmallerThan(other) { + return true + } + return other.Set != nil && this.Set.Equal(other.Set) +} From ca0499b4b0d38fc7881f0ad604ec3f2eb9a46cc2 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Fri, 27 May 2022 21:20:38 +0200 Subject: [PATCH 42/56] WIP --- go/cmd/gh-ost/main.go | 2 +- go/logic/applier.go | 2 +- go/mysql/binlog.go | 20 ++++++++- go/mysql/binlog_file.go | 26 +++++------ go/mysql/binlog_gtid.go | 96 +++++++++++++++++++++-------------------- go/mysql/binlog_test.go | 36 ++++++++++++++++ go/mysql/utils.go | 47 +++++++++++--------- 7 files changed, 144 insertions(+), 85 deletions(-) create mode 100644 go/mysql/binlog_test.go diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 940cb1737..16e548429 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -87,7 +87,7 @@ func main() { flag.BoolVar(&migrationContext.AliyunRDS, "aliyun-rds", false, "set to 'true' when you execute on Aliyun RDS.") flag.BoolVar(&migrationContext.GoogleCloudPlatform, "gcp", false, "set to 'true' when you execute on a 1st generation Google Cloud Platform (GCP).") flag.BoolVar(&migrationContext.AzureMySQL, "azure", false, "set to 'true' when you execute on Azure Database on MySQL.") - flag.BoolVar(&migrationContext.UseGTIDs, "gtid", false, "set to 'true' to enable MySQL GTIDs for replication binlog positioning.") + flag.BoolVar(&migrationContext.UseGTIDs, "gtid", false, "(experimental) set to 'true' to use MySQL GTIDs for binlog positioning.") executeFlag := flag.Bool("execute", false, "actually execute the alter & migrate the table. Default is noop: do some tests and exit") flag.BoolVar(&migrationContext.TestOnReplica, "test-on-replica", false, "Have the migration run on a replica, not on the master. At the end of migration replication is stopped, and tables are swapped and immediately swap-revert. Replication remains stopped and you can compare the two tables for building trust") diff --git a/go/logic/applier.go b/go/logic/applier.go index f2cef9d0f..6a661cebd 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -989,7 +989,7 @@ func (this *Applier) StopReplication() error { return err } - readBinlogCoordinates, executeBinlogCoordinates, err := mysql.GetReplicationBinlogCoordinates(this.migrationContext.ApplierMySQLVersion, this.db) + readBinlogCoordinates, executeBinlogCoordinates, err := mysql.GetReplicationBinlogCoordinates(this.migrationContext.ApplierMySQLVersion, this.db, this.migrationContext.UseGTIDs) if err != nil { return err } diff --git a/go/mysql/binlog.go b/go/mysql/binlog.go index 9a9838fb7..51a57d2d1 100644 --- a/go/mysql/binlog.go +++ b/go/mysql/binlog.go @@ -6,8 +6,9 @@ package mysql +import "errors" + type BinlogCoordinates interface { - Name() string String() string DisplayString() string IsEmpty() bool @@ -15,3 +16,20 @@ type BinlogCoordinates interface { SmallerThan(other BinlogCoordinates) bool SmallerThanOrEquals(other BinlogCoordinates) bool } + +func binlogCoordinatesToImplementation(in BinlogCoordinates, out interface{}) (err error) { + var ok bool + switch out.(type) { + case *FileBinlogCoordinates: + out, ok = in.(*FileBinlogCoordinates) + case *GTIDBinlogCoordinates: + out, ok = in.(*GTIDBinlogCoordinates) + default: + err = errors.New("unrecognized BinlogCoordinates implementation") + } + + if !ok { + err = errors.New("failed to reflect BinlogCoordinates implementation") + } + return err +} diff --git a/go/mysql/binlog_file.go b/go/mysql/binlog_file.go index b251a63d5..65cc6da68 100644 --- a/go/mysql/binlog_file.go +++ b/go/mysql/binlog_file.go @@ -20,21 +20,20 @@ func init() { detachPattern, _ = regexp.Compile(`//([^/:]+):([\d]+)`) // e.g. `//binlog.01234:567890` } -type BinlogType int - -const ( - BinaryLog BinlogType = iota - RelayLog -) - // FileBinlogCoordinates described binary log coordinates in the form of a binlog file & log position. type FileBinlogCoordinates struct { LogFile string LogPos int64 - Type BinlogType EventSize int64 } +func NewFileBinlogCoordinates(logFile string, logPos int64) *FileBinlogCoordinates { + return &FileBinlogCoordinates{ + LogFile: logFile, + LogPos: logPos, + } +} + // ParseFileBinlogCoordinates parses a log file/position string into a *BinlogCoordinates struct. func ParseFileBinlogCoordinates(logFileLogPos string) (*FileBinlogCoordinates, error) { tokens := strings.SplitN(logFileLogPos, ":", 2) @@ -49,11 +48,6 @@ func ParseFileBinlogCoordinates(logFileLogPos string) (*FileBinlogCoordinates, e } } -// Name returns the name of the BinlogCoordinates interface implementation -func (this *FileBinlogCoordinates) Name() string { - return "file" -} - // DisplayString returns a user-friendly string representation of these coordinates func (this *FileBinlogCoordinates) DisplayString() string { return fmt.Sprintf("%s:%d", this.LogFile, this.LogPos) @@ -70,7 +64,7 @@ func (this *FileBinlogCoordinates) Equals(other BinlogCoordinates) bool { if !ok || other == nil { return false } - return this.LogFile == coord.LogFile && this.LogPos == coord.LogPos && this.Type == coord.Type + return this.LogFile == coord.LogFile && this.LogPos == coord.LogPos } // IsEmpty returns true if the log file is empty, unnamed @@ -138,7 +132,7 @@ func (this *FileBinlogCoordinates) FileNumber() (int, int) { // PreviousFileCoordinatesBy guesses the filename of the previous binlog/relaylog, by given offset (number of files back) func (this *FileBinlogCoordinates) PreviousFileCoordinatesBy(offset int) (BinlogCoordinates, error) { - result := &FileBinlogCoordinates{LogPos: 0, Type: this.Type} + result := &FileBinlogCoordinates{} fileNum, numLen := this.FileNumber() if fileNum == 0 { @@ -160,7 +154,7 @@ func (this *FileBinlogCoordinates) PreviousFileCoordinates() (BinlogCoordinates, // PreviousFileCoordinates guesses the filename of the previous binlog/relaylog func (this *FileBinlogCoordinates) NextFileCoordinates() (BinlogCoordinates, error) { - result := &FileBinlogCoordinates{LogPos: 0, Type: this.Type} + result := &FileBinlogCoordinates{} fileNum, numLen := this.FileNumber() newNumStr := fmt.Sprintf("%d", (fileNum + 1)) diff --git a/go/mysql/binlog_gtid.go b/go/mysql/binlog_gtid.go index d66586ef8..cec14d0f4 100644 --- a/go/mysql/binlog_gtid.go +++ b/go/mysql/binlog_gtid.go @@ -6,79 +6,83 @@ package mysql import ( + "errors" + gomysql "github.com/go-mysql-org/go-mysql/mysql" ) -// GTIDBinlogCoordinates described binary log coordinates in the form of a binlog file & log position. +// GTIDBinlogCoordinates describe binary log coordinates in MySQL GTID format. type GTIDBinlogCoordinates struct { - Set *gomysql.MysqlGTIDSet + GTIDSet *gomysql.MysqlGTIDSet + UUIDSet *gomysql.UUIDSet } -// ParseGTIDSetBinlogCoordinates parses a MySQL GTID set into a *GTIDBinlogCoordinates struct. -func ParseGTIDSetBinlogCoordinates(gtidSet string) (*GTIDBinlogCoordinates, error) { +// NewGTIDBinlogCoordinates parses a MySQL GTID set into a *GTIDBinlogCoordinates struct. +func NewGTIDBinlogCoordinates(gtidSet string) (*GTIDBinlogCoordinates, error) { set, err := gomysql.ParseMysqlGTIDSet(gtidSet) - return >IDBinlogCoordinates{set.(*gomysql.MysqlGTIDSet)}, err + return >IDBinlogCoordinates{ + GTIDSet: set.(*gomysql.MysqlGTIDSet), + }, err } -// DisplayString returns a user-friendly string representation of these coordinates +// DisplayString returns a user-friendly string representation of these current UUID set or the full GTID set. func (this *GTIDBinlogCoordinates) DisplayString() string { - return this.Set.String() + if this.UUIDSet != nil { + return this.UUIDSet.String() + } + return this.String() } -// String returns a user-friendly string representation of these coordinates +// String returns a user-friendly string representation of these full GTID set. func (this GTIDBinlogCoordinates) String() string { - return this.DisplayString() + return this.GTIDSet.String() } // Equals tests equality of this coordinate and another one. -func (this *GTIDBinlogCoordinates) Equals(other *GTIDBinlogCoordinates) bool { - if other == nil { +func (this *GTIDBinlogCoordinates) Equals(other BinlogCoordinates) bool { + if other == nil || this.IsEmpty() || other.IsEmpty() { return false } - return other.Set != nil && this.Set.Equal(other.Set) + + otherBinlogCoordinates := >IDBinlogCoordinates{} + if err := binlogCoordinatesToImplementation(other, otherBinlogCoordinates); err != nil { + panic(err) + } + + return this.GTIDSet.Equal(otherBinlogCoordinates.GTIDSet) } -// IsEmpty returns true if the GTID set is empty, unnamed +// IsEmpty returns true if the GTID set is empty. func (this *GTIDBinlogCoordinates) IsEmpty() bool { - return this.Set == nil + return this.GTIDSet == nil } // SmallerThan returns true if this coordinate is strictly smaller than the other. -func (this *GTIDBinlogCoordinates) SmallerThan(other *GTIDBinlogCoordinates) bool { - // if GTID SIDs are equal we compare the interval stop points - // if GTID SIDs differ we have to assume there is a new/larger event - if other.Set == nil || other.Set.Sets == nil { - return false +func (this *GTIDBinlogCoordinates) SmallerThan(other BinlogCoordinates) bool { + otherBinlogCoordinates := >IDBinlogCoordinates{} + if err := binlogCoordinatesToImplementation(other, otherBinlogCoordinates); err != nil { + panic(err) } - if len(this.Set.Sets) < len(other.Set.Sets) { - return true - } - for sid, otherSet := range other.Set.Sets { - thisSet, ok := this.Set.Sets[sid] - if !ok { - return true // 'this' is missing an SID - } - if len(thisSet.Intervals) < len(otherSet.Intervals) { - return true // 'this' has fewer intervals - } - for i, otherInterval := range otherSet.Intervals { - if len(thisSet.Intervals)-1 > i { - return true - } - thisInterval := thisSet.Intervals[i] - if thisInterval.Start < otherInterval.Start || thisInterval.Stop < otherInterval.Stop { - return true - } - } - } - return false + + // if 'this' does not contain the same sets we assume we are behind 'other'. + // there are probably edge cases where this isn't true + return !this.GTIDSet.Contain(other.GTIDSet) } // SmallerThanOrEquals returns true if this coordinate is the same or equal to the other one. -// We do NOT compare the type so we can not use this.Equals() -func (this *GTIDBinlogCoordinates) SmallerThanOrEquals(other *GTIDBinlogCoordinates) bool { - if this.SmallerThan(other) { - return true +func (this *GTIDBinlogCoordinates) SmallerThanOrEquals(other BinlogCoordinates) bool { + return this.Equals(other) || this.SmallerThan(other) +} + +func (this *GTIDBinlogCoordinates) Update(update interface{}) error { + switch u := update.(type) { + case *gomysql.UUIDSet: + this.GTIDSet.AddSet(u) + this.UUIDSet = u + case *gomysql.MysqlGTIDSet: + this.GTIDSet = u + default: + return errors.New("unsupported update") } - return other.Set != nil && this.Set.Equal(other.Set) + return nil } diff --git a/go/mysql/binlog_test.go b/go/mysql/binlog_test.go new file mode 100644 index 000000000..cd75149e9 --- /dev/null +++ b/go/mysql/binlog_test.go @@ -0,0 +1,36 @@ +/* + Copyright 2022 GitHub Inc. + See https://github.com/github/gh-ost/blob/master/LICENSE +*/ + +package mysql + +import ( + "testing" + + "github.com/openark/golib/log" + test "github.com/openark/golib/tests" +) + +func init() { + log.SetLevel(log.ERROR) +} + +func TestBinlogCoordinatesToImplementation(t *testing.T) { + test.S(t).ExpectNil(binlogCoordinatesToImplementation( + &FileBinlogCoordinates{}, + &FileBinlogCoordinates{}, + )) + test.S(t).ExpectNil(binlogCoordinatesToImplementation( + >IDBinlogCoordinates{}, + >IDBinlogCoordinates{}, + )) + test.S(t).ExpectNotNil(binlogCoordinatesToImplementation( + &FileBinlogCoordinates{}, + >IDBinlogCoordinates{}, + )) + test.S(t).ExpectNotNil(binlogCoordinatesToImplementation( + &FileBinlogCoordinates{}, + map[string]string{}, + )) +} diff --git a/go/mysql/utils.go b/go/mysql/utils.go index 6a134e1f4..54522df12 100644 --- a/go/mysql/utils.go +++ b/go/mysql/utils.go @@ -14,7 +14,6 @@ import ( "github.com/github/gh-ost/go/sql" - gomysql "github.com/go-mysql-org/go-mysql/mysql" "github.com/openark/golib/log" "github.com/openark/golib/sqlutils" ) @@ -160,35 +159,43 @@ func GetMasterConnectionConfigSafe(dbVersion string, connectionConfig *Connectio return GetMasterConnectionConfigSafe(dbVersion, masterConfig, visitedKeys, allowMasterMaster) } -func GetReplicationBinlogCoordinates(dbVersion string, db *gosql.DB) (readBinlogCoordinates *BinlogCoordinates, executeBinlogCoordinates *BinlogCoordinates, err error) { +func GetReplicationBinlogCoordinates(dbVersion string, db *gosql.DB, gtid bool) (readBinlogCoordinates, executeBinlogCoordinates BinlogCoordinates, err error) { showReplicaStatusQuery := fmt.Sprintf("show %s", ReplicaTermFor(dbVersion, `slave status`)) err = sqlutils.QueryRowsMap(db, showReplicaStatusQuery, func(m sqlutils.RowMap) error { - readBinlogCoordinates = &BinlogCoordinates{ - LogFile: m.GetString(ReplicaTermFor(dbVersion, "Master_Log_File")), - LogPos: m.GetInt64(ReplicaTermFor(dbVersion, "Read_Master_Log_Pos")), - } - executeBinlogCoordinates = &BinlogCoordinates{ - LogFile: m.GetString(ReplicaTermFor(dbVersion, "Relay_Master_Log_File")), - LogPos: m.GetInt64(ReplicaTermFor(dbVersion, "Exec_Master_Log_Pos")), + if gtid { + executeBinlogCoordinates, err = NewGTIDBinlogCoordinates(m.GetString("Executed_Gtid_Set")) + if err != nil { + return err + } + readBinlogCoordinates, err = NewGTIDBinlogCoordinates(m.GetString("Retrieved_Gtid_Set")) + if err != nil { + return err + } + } else { + readBinlogCoordinates = NewFileBinlogCoordinates( + m.GetString("Master_Log_File"), + m.GetInt64("Read_Master_Log_Pos"), + ) + executeBinlogCoordinates = NewFileBinlogCoordinates( + m.GetString("Relay_Master_Log_File"), + m.GetInt64("Exec_Master_Log_Pos"), + ) } return nil }) return readBinlogCoordinates, executeBinlogCoordinates, err } -func GetSelfBinlogCoordinates(dbVersion string, db *gosql.DB) (selfBinlogCoordinates *BinlogCoordinates, err error) { +func GetSelfBinlogCoordinates(dbVersion string, db *gosql.DB, gtid bool) (selfBinlogCoordinates BinlogCoordinates, err error) { binaryLogStatusTerm := ReplicaTermFor(dbVersion, "master status") err = sqlutils.QueryRowsMap(db, fmt.Sprintf("show %s", binaryLogStatusTerm), func(m sqlutils.RowMap) error { - selfBinlogCoordinates = &BinlogCoordinates{ - LogFile: m.GetString("File"), - LogPos: m.GetInt64("Position"), - } - if execGtidSet := m.GetString("Executed_Gtid_Set"); execGtidSet != "" { - gtidSet, err := gomysql.ParseMysqlGTIDSet(execGtidSet) - if err != nil { - return err - } - selfBinlogCoordinates.GTIDSet = gtidSet.(*gomysql.MysqlGTIDSet) + if gtid { + selfBinlogCoordinates, err = NewGTIDBinlogCoordinates(m.GetString("Executed_Gtid_Set")) + } else { + selfBinlogCoordinates = NewFileBinlogCoordinates( + m.GetString("File"), + m.GetInt64("Position"), + ) } return nil }) From 788911dbb3aff4414b82511de0055ebdd13bbcc5 Mon Sep 17 00:00:00 2001 From: meiji163 Date: Tue, 23 Sep 2025 15:16:07 -0700 Subject: [PATCH 43/56] fix BinlogCoordinates interface usage --- go/binlog/binlog_entry.go | 15 --------- go/binlog/gomysql_reader.go | 55 +++++++++++++++++++------------ go/logic/applier.go | 2 +- go/logic/migrator.go | 4 +-- go/logic/streamer.go | 38 +++++++++++---------- go/mysql/binlog.go | 20 +---------- go/mysql/binlog_file.go | 8 +++++ go/mysql/binlog_file_test.go | 64 ++++++++++++++++++------------------ go/mysql/binlog_gtid.go | 30 ++++++++++++----- go/mysql/binlog_test.go | 36 -------------------- 10 files changed, 121 insertions(+), 151 deletions(-) delete mode 100644 go/mysql/binlog_test.go diff --git a/go/binlog/binlog_entry.go b/go/binlog/binlog_entry.go index e4283901e..69a2fc31d 100644 --- a/go/binlog/binlog_entry.go +++ b/go/binlog/binlog_entry.go @@ -9,8 +9,6 @@ import ( "fmt" "github.com/github/gh-ost/go/mysql" - - gomysql "github.com/go-mysql-org/go-mysql/mysql" ) // BinlogEntry describes an entry in the binary log @@ -19,14 +17,6 @@ type BinlogEntry struct { DmlEvent *BinlogDMLEvent } -// NewBinlogEntry creates an empty, ready to go BinlogEntry object -func NewBinlogEntry(logFile string, logPos uint64, gtidSet *gomysql.MysqlGTIDSet) *BinlogEntry { - binlogEntry := &BinlogEntry{ - Coordinates: mysql.BinlogCoordinates{LogFile: logFile, LogPos: int64(logPos), GTIDSet: gtidSet}, - } - return binlogEntry -} - // NewBinlogEntryAt creates an empty, ready to go BinlogEntry object func NewBinlogEntryAt(coordinates mysql.BinlogCoordinates) *BinlogEntry { binlogEntry := &BinlogEntry{ @@ -35,11 +25,6 @@ func NewBinlogEntryAt(coordinates mysql.BinlogCoordinates) *BinlogEntry { return binlogEntry } -// Duplicate creates and returns a new binlog entry, with some of the attributes pre-assigned -func (this *BinlogEntry) Duplicate() *BinlogEntry { - return NewBinlogEntry(this.Coordinates.LogFile, uint64(this.Coordinates.LogPos), this.Coordinates.GTIDSet) -} - // String() returns a string representation of this binlog entry func (this *BinlogEntry) String() string { return fmt.Sprintf("[BinlogEntry at %+v; dml:%+v]", this.Coordinates, this.DmlEvent) diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index e79d363e2..d8102e080 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -37,7 +37,6 @@ func NewGoMySQLReader(migrationContext *base.MigrationContext) *GoMySQLReader { return &GoMySQLReader{ migrationContext: migrationContext, connectionConfig: connectionConfig, - currentCoordinates: mysql.BinlogCoordinates{}, currentCoordinatesMutex: &sync.Mutex{}, binlogSyncer: replication.NewBinlogSyncer(replication.BinlogSyncerConfig{ ServerID: uint32(migrationContext.ReplicaServerId), @@ -65,32 +64,39 @@ func (this *GoMySQLReader) ConnectBinlogStreamer(coordinates mysql.BinlogCoordin // Start sync with specified GTID set or binlog file and position if this.migrationContext.UseGTIDs { - this.binlogStreamer, err = this.binlogSyncer.StartSyncGTID(this.currentCoordinates.GTIDSet) + coords := this.currentCoordinates.(*mysql.GTIDBinlogCoordinates) + this.binlogStreamer, err = this.binlogSyncer.StartSyncGTID(coords.GTIDSet) } else { + coords := this.currentCoordinates.(*mysql.FileBinlogCoordinates) this.binlogStreamer, err = this.binlogSyncer.StartSync(gomysql.Position{ - Name: this.currentCoordinates.LogFile, - Pos: uint32(this.currentCoordinates.LogPos)}, + Name: coords.LogFile, + Pos: uint32(coords.LogPos)}, ) } return err } -func (this *GoMySQLReader) GetCurrentBinlogCoordinates() *mysql.BinlogCoordinates { +func (this *GoMySQLReader) GetCurrentBinlogCoordinates() mysql.BinlogCoordinates { this.currentCoordinatesMutex.Lock() defer this.currentCoordinatesMutex.Unlock() returnCoordinates := this.currentCoordinates - return &returnCoordinates + return returnCoordinates } -// StreamEvents func (this *GoMySQLReader) handleRowsEvent(ev *replication.BinlogEvent, rowsEvent *replication.RowsEvent, entriesChannel chan<- *BinlogEntry) error { + this.currentCoordinatesMutex.Lock() + currentCoords := this.currentCoordinates + lastApplied := this.LastAppliedRowsEventHint + this.currentCoordinatesMutex.Unlock() + if !this.migrationContext.UseGTIDs { - if this.currentCoordinates.IsLogPosOverflowBeyond4Bytes(&this.LastAppliedRowsEventHint) { - return fmt.Errorf("Unexpected rows event at %+v, the binlog end_log_pos is overflow 4 bytes", this.currentCoordinates) + currentFileCoords := currentCoords.(*mysql.FileBinlogCoordinates) + if lastApplied != nil && currentFileCoords.IsLogPosOverflowBeyond4Bytes(lastApplied.(*mysql.FileBinlogCoordinates)) { + return fmt.Errorf("Unexpected rows event at %+v, the binlog end_log_pos is overflow 4 bytes", currentCoords) } - if this.currentCoordinates.SmallerThanOrEquals(&this.LastAppliedRowsEventHint) { - this.migrationContext.Log.Debugf("Skipping handled query at %+v", this.currentCoordinates) + if currentCoords.SmallerThanOrEquals(lastApplied) { + this.migrationContext.Log.Debugf("Skipping handled query at %+v (last applied is %+v)", currentCoords, lastApplied) return nil } } @@ -105,7 +111,7 @@ func (this *GoMySQLReader) handleRowsEvent(ev *replication.BinlogEvent, rowsEven // We do both at the same time continue } - binlogEntry := NewBinlogEntryAt(this.currentCoordinates) + binlogEntry := NewBinlogEntryAt(currentCoords) binlogEntry.DmlEvent = NewBinlogDMLEvent( string(rowsEvent.Table.Schema), string(rowsEvent.Table.Table), @@ -126,18 +132,19 @@ func (this *GoMySQLReader) handleRowsEvent(ev *replication.BinlogEvent, rowsEven binlogEntry.DmlEvent.WhereColumnValues = sql.ToColumnValues(row) } } + // The channel will do the throttling. Whoever is reading from the channel // decides whether action is taken synchronously (meaning we wait before // next iteration) or asynchronously (we keep pushing more events) // In reality, reads will be synchronous entriesChannel <- binlogEntry } + this.currentCoordinatesMutex.Lock() + this.LastAppliedRowsEventHint = this.currentCoordinates.Clone() if this.migrationContext.UseGTIDs { - this.currentCoordinatesMutex.Lock() - defer this.currentCoordinatesMutex.Unlock() this.currentCoordinates = this.nextCoordinates } - this.LastAppliedRowsEventHint = this.currentCoordinates + defer this.currentCoordinatesMutex.Unlock() return nil } @@ -157,8 +164,11 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha func() { this.currentCoordinatesMutex.Lock() defer this.currentCoordinatesMutex.Unlock() - this.currentCoordinates.LogPos = int64(ev.Header.LogPos) - this.currentCoordinates.EventSize = int64(ev.Header.EventSize) + if !this.migrationContext.UseGTIDs { + coords := this.currentCoordinates.(*mysql.FileBinlogCoordinates) + coords.LogPos = int64(ev.Header.LogPos) + coords.EventSize = int64(ev.Header.EventSize) + } }() switch event := ev.Event.(type) { @@ -169,7 +179,8 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha func() { this.currentCoordinatesMutex.Lock() defer this.currentCoordinatesMutex.Unlock() - if err := this.currentCoordinates.GTIDSet.Update(event.GTIDSets); err != nil { + coords := this.currentCoordinates.(*mysql.GTIDBinlogCoordinates) + if err := coords.GTIDSet.Update(event.GTIDSets); err != nil { this.migrationContext.Log.Errorf("Failed to parse GTID set: %v", err) } }() @@ -186,7 +197,8 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha defer this.currentCoordinatesMutex.Unlock() this.nextCoordinates = this.currentCoordinates interval := gomysql.Interval{Start: event.GNO, Stop: event.GNO + 1} - this.nextCoordinates.GTIDSet.AddSet(gomysql.NewUUIDSet(sid, interval)) + coords := this.currentCoordinates.(*mysql.GTIDBinlogCoordinates) + coords.GTIDSet.AddSet(gomysql.NewUUIDSet(sid, interval)) }() case *replication.RotateEvent: if this.migrationContext.UseGTIDs { @@ -195,9 +207,10 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha func() { this.currentCoordinatesMutex.Lock() defer this.currentCoordinatesMutex.Unlock() - this.currentCoordinates.LogFile = string(event.NextLogName) + coords := this.currentCoordinates.(*mysql.FileBinlogCoordinates) + coords.LogFile = string(event.NextLogName) + this.migrationContext.Log.Infof("rotate to next log from %s:%d to %s", coords.LogFile, int64(ev.Header.LogPos), event.NextLogName) }() - this.migrationContext.Log.Infof("rotate to next log from %s:%d to %s", this.currentCoordinates.LogFile, int64(ev.Header.LogPos), event.NextLogName) case *replication.RowsEvent: if err := this.handleRowsEvent(ev, event, entriesChannel); err != nil { return err diff --git a/go/logic/applier.go b/go/logic/applier.go index 6a661cebd..be226c7dc 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -993,7 +993,7 @@ func (this *Applier) StopReplication() error { if err != nil { return err } - this.migrationContext.Log.Infof("Replication IO thread at %+v. SQL thread is at %+v", *readBinlogCoordinates, *executeBinlogCoordinates) + this.migrationContext.Log.Infof("Replication IO thread at %+v. SQL thread is at %+v", readBinlogCoordinates, executeBinlogCoordinates) return nil } diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 880b9b4c5..4d7074b22 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -1075,7 +1075,7 @@ func (this *Migrator) printStatus(rule PrintStatusRule, writers ...io.Writer) { return } - currentBinlogCoordinates := *this.eventsStreamer.GetCurrentBinlogCoordinates() + currentBinlogCoordinates := this.eventsStreamer.GetCurrentBinlogCoordinates() status := fmt.Sprintf("Copy: %d/%d %.1f%%; Applied: %d; Backlog: %d/%d; Time: %+v(total), %+v(copy); streamer: %+v; Lag: %.2fs, HeartbeatLag: %.2fs, State: %s; ETA: %s", totalRowsCopied, rowsEstimate, progressPct, @@ -1140,7 +1140,7 @@ func (this *Migrator) initiateStreaming() error { if atomic.LoadInt64(&this.finishedMigrating) > 0 { return } - this.migrationContext.SetRecentBinlogCoordinates(*this.eventsStreamer.GetCurrentBinlogCoordinates()) + this.migrationContext.SetRecentBinlogCoordinates(this.eventsStreamer.GetCurrentBinlogCoordinates()) } }() return nil diff --git a/go/logic/streamer.go b/go/logic/streamer.go index 2a4b9d5b2..4fd6dd315 100644 --- a/go/logic/streamer.go +++ b/go/logic/streamer.go @@ -39,7 +39,7 @@ type EventsStreamer struct { db *gosql.DB dbVersion string migrationContext *base.MigrationContext - initialBinlogCoordinates *mysql.BinlogCoordinates + initialBinlogCoordinates mysql.BinlogCoordinates listeners [](*BinlogEventListener) listenersMutex *sync.Mutex eventsChannel chan *binlog.BinlogEntry @@ -125,25 +125,28 @@ func (this *EventsStreamer) InitDBConnections() (err error) { } // initBinlogReader creates and connects the reader: we hook up to a MySQL server as a replica -func (this *EventsStreamer) initBinlogReader(binlogCoordinates *mysql.BinlogCoordinates) error { +func (this *EventsStreamer) initBinlogReader(binlogCoordinates mysql.BinlogCoordinates) error { goMySQLReader := binlog.NewGoMySQLReader(this.migrationContext) - if err := goMySQLReader.ConnectBinlogStreamer(*binlogCoordinates); err != nil { + if err := goMySQLReader.ConnectBinlogStreamer(binlogCoordinates); err != nil { return err } this.binlogReader = goMySQLReader return nil } -func (this *EventsStreamer) GetCurrentBinlogCoordinates() *mysql.BinlogCoordinates { +func (this *EventsStreamer) GetCurrentBinlogCoordinates() mysql.BinlogCoordinates { return this.binlogReader.GetCurrentBinlogCoordinates() } -func (this *EventsStreamer) GetReconnectBinlogCoordinates() *mysql.BinlogCoordinates { +func (this *EventsStreamer) GetReconnectBinlogCoordinates() mysql.BinlogCoordinates { current := this.GetCurrentBinlogCoordinates() - if current.GTIDSet != nil { - return &mysql.BinlogCoordinates{GTIDSet: current.GTIDSet} + switch coords := current.(type) { + case *mysql.FileBinlogCoordinates: + return &mysql.FileBinlogCoordinates{LogFile: coords.LogFile, LogPos: 4} + case *mysql.GTIDBinlogCoordinates: + return &mysql.GTIDBinlogCoordinates{GTIDSet: coords.GTIDSet} } - return &mysql.BinlogCoordinates{LogFile: current.LogFile, LogPos: 4} + return nil } // readCurrentBinlogCoordinates reads master status from hooked server @@ -152,19 +155,20 @@ func (this *EventsStreamer) readCurrentBinlogCoordinates() error { query := fmt.Sprintf("show /* gh-ost readCurrentBinlogCoordinates */ %s", binaryLogStatusTerm) foundMasterStatus := false err := sqlutils.QueryRowsMap(this.db, query, func(m sqlutils.RowMap) error { - this.initialBinlogCoordinates = &mysql.BinlogCoordinates{ - LogFile: m.GetString("File"), - LogPos: m.GetInt64("Position"), - } - if execGtidSet := m.GetString("Executed_Gtid_Set"); execGtidSet != "" && this.migrationContext.UseGTIDs { + if this.migrationContext.UseGTIDs { + execGtidSet := m.GetString("Executed_Gtid_Set") gtidSet, err := gomysql.ParseMysqlGTIDSet(execGtidSet) if err != nil { return err } - this.initialBinlogCoordinates.GTIDSet = gtidSet.(*gomysql.MysqlGTIDSet) + this.initialBinlogCoordinates = &mysql.GTIDBinlogCoordinates{GTIDSet: gtidSet.(*gomysql.MysqlGTIDSet)} + } else { + this.initialBinlogCoordinates = &mysql.FileBinlogCoordinates{ + LogFile: m.GetString("File"), + LogPos: m.GetInt64("Position"), + } } foundMasterStatus = true - return nil }) if err != nil { @@ -173,7 +177,7 @@ func (this *EventsStreamer) readCurrentBinlogCoordinates() error { if !foundMasterStatus { return fmt.Errorf("Got no results from SHOW %s. Bailing out", strings.ToUpper(binaryLogStatusTerm)) } - this.migrationContext.Log.Debugf("Streamer binlog coordinates: %+v", *this.initialBinlogCoordinates) + this.migrationContext.Log.Debugf("Streamer binlog coordinates: %+v", this.initialBinlogCoordinates) return nil } @@ -204,7 +208,7 @@ func (this *EventsStreamer) StreamEvents(canStopStreaming func() bool) error { time.Sleep(ReconnectStreamerSleepSeconds * time.Second) // See if there's retry overflow - if this.binlogReader.LastAppliedRowsEventHint.Equals(&lastAppliedRowsEventHint) { + if this.binlogReader.LastAppliedRowsEventHint.Equals(lastAppliedRowsEventHint) { successiveFailures += 1 } else { successiveFailures = 0 diff --git a/go/mysql/binlog.go b/go/mysql/binlog.go index 51a57d2d1..01790da29 100644 --- a/go/mysql/binlog.go +++ b/go/mysql/binlog.go @@ -6,8 +6,6 @@ package mysql -import "errors" - type BinlogCoordinates interface { String() string DisplayString() string @@ -15,21 +13,5 @@ type BinlogCoordinates interface { Equals(other BinlogCoordinates) bool SmallerThan(other BinlogCoordinates) bool SmallerThanOrEquals(other BinlogCoordinates) bool -} - -func binlogCoordinatesToImplementation(in BinlogCoordinates, out interface{}) (err error) { - var ok bool - switch out.(type) { - case *FileBinlogCoordinates: - out, ok = in.(*FileBinlogCoordinates) - case *GTIDBinlogCoordinates: - out, ok = in.(*GTIDBinlogCoordinates) - default: - err = errors.New("unrecognized BinlogCoordinates implementation") - } - - if !ok { - err = errors.New("failed to reflect BinlogCoordinates implementation") - } - return err + Clone() BinlogCoordinates } diff --git a/go/mysql/binlog_file.go b/go/mysql/binlog_file.go index 65cc6da68..426e54076 100644 --- a/go/mysql/binlog_file.go +++ b/go/mysql/binlog_file.go @@ -175,6 +175,14 @@ func (this *FileBinlogCoordinates) DetachedCoordinates() (isDetached bool, detac return true, detachedCoordinatesSubmatch[1], detachedCoordinatesSubmatch[2] } +func (this *FileBinlogCoordinates) Clone() BinlogCoordinates { + return &FileBinlogCoordinates{ + LogPos: this.LogPos, + LogFile: this.LogFile, + EventSize: this.EventSize, + } +} + // IsLogPosOverflowBeyond4Bytes returns true if the coordinate endpos is overflow beyond 4 bytes. // The binlog event end_log_pos field type is defined as uint32, 4 bytes. // https://github.com/go-mysql-org/go-mysql/blob/master/replication/event.go diff --git a/go/mysql/binlog_file_test.go b/go/mysql/binlog_file_test.go index b2fd2377c..f12f5514f 100644 --- a/go/mysql/binlog_file_test.go +++ b/go/mysql/binlog_file_test.go @@ -19,10 +19,10 @@ func init() { } func TestBinlogCoordinates(t *testing.T) { - c1 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 104} - c2 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 104} - c3 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 5000} - c4 := BinlogCoordinates{LogFile: "mysql-bin.00112", LogPos: 104} + c1 := FileBinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 104} + c2 := FileBinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 104} + c3 := FileBinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 5000} + c4 := FileBinlogCoordinates{LogFile: "mysql-bin.00112", LogPos: 104} gtidSet1, _ := gomysql.ParseMysqlGTIDSet("3E11FA47-71CA-11E1-9E33-C80AA9429562:23") gtidSet2, _ := gomysql.ParseMysqlGTIDSet("3E11FA47-71CA-11E1-9E33-C80AA9429562:100") @@ -48,12 +48,12 @@ func TestBinlogCoordinates(t *testing.T) { 48e2bc1d-d66d-11e8-bf56-a0369f9437b8:1, 492e2980-4518-11e9-92c6-e4434b3eca94:1-4926754399`) - c5 := BinlogCoordinates{GTIDSet: gtidSet1.(*gomysql.MysqlGTIDSet)} - c6 := BinlogCoordinates{GTIDSet: gtidSet1.(*gomysql.MysqlGTIDSet)} - c7 := BinlogCoordinates{GTIDSet: gtidSet2.(*gomysql.MysqlGTIDSet)} - c8 := BinlogCoordinates{GTIDSet: gtidSet3.(*gomysql.MysqlGTIDSet)} - c9 := BinlogCoordinates{GTIDSet: gtidSetBig1.(*gomysql.MysqlGTIDSet)} - c10 := BinlogCoordinates{GTIDSet: gtidSetBig2.(*gomysql.MysqlGTIDSet)} + c5 := GTIDBinlogCoordinates{GTIDSet: gtidSet1.(*gomysql.MysqlGTIDSet)} + c6 := GTIDBinlogCoordinates{GTIDSet: gtidSet1.(*gomysql.MysqlGTIDSet)} + c7 := GTIDBinlogCoordinates{GTIDSet: gtidSet2.(*gomysql.MysqlGTIDSet)} + c8 := GTIDBinlogCoordinates{GTIDSet: gtidSet3.(*gomysql.MysqlGTIDSet)} + c9 := GTIDBinlogCoordinates{GTIDSet: gtidSetBig1.(*gomysql.MysqlGTIDSet)} + c10 := GTIDBinlogCoordinates{GTIDSet: gtidSetBig2.(*gomysql.MysqlGTIDSet)} require.True(t, c5.Equals(&c6)) require.True(t, c1.Equals(&c2)) @@ -73,64 +73,64 @@ func TestBinlogCoordinates(t *testing.T) { require.True(t, c6.SmallerThanOrEquals(&c7)) require.True(t, c7.SmallerThanOrEquals(&c8)) require.True(t, c9.SmallerThanOrEquals(&c9)) - require.True(t, c10.SmallerThanOrEquals(&c9)) + require.True(t, c9.SmallerThanOrEquals(&c10)) } func TestBinlogCoordinatesAsKey(t *testing.T) { m := make(map[BinlogCoordinates]bool) - c1 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 104} - c2 := BinlogCoordinates{LogFile: "mysql-bin.00022", LogPos: 104} - c3 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 104} - c4 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 222} + c1 := &FileBinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 104} + c2 := &FileBinlogCoordinates{LogFile: "mysql-bin.00022", LogPos: 104} + c3 := &FileBinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 104} + c4 := &FileBinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 222} m[c1] = true m[c2] = true m[c3] = true m[c4] = true - require.Len(t, m, 3) + require.Len(t, m, 4) } func TestIsLogPosOverflowBeyond4Bytes(t *testing.T) { { - var preCoordinates *BinlogCoordinates - curCoordinates := &BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 10321, EventSize: 1100} + var preCoordinates *FileBinlogCoordinates + curCoordinates := &FileBinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 10321, EventSize: 1100} require.False(t, curCoordinates.IsLogPosOverflowBeyond4Bytes(preCoordinates)) } { - preCoordinates := &BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 1100, EventSize: 1100} - curCoordinates := &BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: int64(uint32(preCoordinates.LogPos + 1100)), EventSize: 1100} + preCoordinates := &FileBinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 1100, EventSize: 1100} + curCoordinates := &FileBinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: int64(uint32(preCoordinates.LogPos + 1100)), EventSize: 1100} require.False(t, curCoordinates.IsLogPosOverflowBeyond4Bytes(preCoordinates)) } { - preCoordinates := &BinlogCoordinates{LogFile: "mysql-bin.00016", LogPos: 1100, EventSize: 1100} - curCoordinates := &BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: int64(uint32(preCoordinates.LogPos + 1100)), EventSize: 1100} + preCoordinates := &FileBinlogCoordinates{LogFile: "mysql-bin.00016", LogPos: 1100, EventSize: 1100} + curCoordinates := &FileBinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: int64(uint32(preCoordinates.LogPos + 1100)), EventSize: 1100} require.False(t, curCoordinates.IsLogPosOverflowBeyond4Bytes(preCoordinates)) } { - preCoordinates := &BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: math.MaxUint32 - 1001, EventSize: 1000} - curCoordinates := &BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: int64(uint32(preCoordinates.LogPos + 1000)), EventSize: 1000} + preCoordinates := &FileBinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: math.MaxUint32 - 1001, EventSize: 1000} + curCoordinates := &FileBinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: int64(uint32(preCoordinates.LogPos + 1000)), EventSize: 1000} require.False(t, curCoordinates.IsLogPosOverflowBeyond4Bytes(preCoordinates)) } { - preCoordinates := &BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: math.MaxUint32 - 1000, EventSize: 1000} - curCoordinates := &BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: int64(uint32(preCoordinates.LogPos + 1000)), EventSize: 1000} + preCoordinates := &FileBinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: math.MaxUint32 - 1000, EventSize: 1000} + curCoordinates := &FileBinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: int64(uint32(preCoordinates.LogPos + 1000)), EventSize: 1000} require.False(t, curCoordinates.IsLogPosOverflowBeyond4Bytes(preCoordinates)) } { - preCoordinates := &BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: math.MaxUint32 - 999, EventSize: 1000} - curCoordinates := &BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: int64(uint32(preCoordinates.LogPos + 1000)), EventSize: 1000} + preCoordinates := &FileBinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: math.MaxUint32 - 999, EventSize: 1000} + curCoordinates := &FileBinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: int64(uint32(preCoordinates.LogPos + 1000)), EventSize: 1000} require.True(t, curCoordinates.IsLogPosOverflowBeyond4Bytes(preCoordinates)) } { - preCoordinates := &BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: int64(uint32(math.MaxUint32 - 500)), EventSize: 1000} - curCoordinates := &BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: int64(uint32(preCoordinates.LogPos + 1000)), EventSize: 1000} + preCoordinates := &FileBinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: int64(uint32(math.MaxUint32 - 500)), EventSize: 1000} + curCoordinates := &FileBinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: int64(uint32(preCoordinates.LogPos + 1000)), EventSize: 1000} require.True(t, curCoordinates.IsLogPosOverflowBeyond4Bytes(preCoordinates)) } { - preCoordinates := &BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: math.MaxUint32, EventSize: 1000} - curCoordinates := &BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: int64(uint32(preCoordinates.LogPos + 1000)), EventSize: 1000} + preCoordinates := &FileBinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: math.MaxUint32, EventSize: 1000} + curCoordinates := &FileBinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: int64(uint32(preCoordinates.LogPos + 1000)), EventSize: 1000} require.True(t, curCoordinates.IsLogPosOverflowBeyond4Bytes(preCoordinates)) } } diff --git a/go/mysql/binlog_gtid.go b/go/mysql/binlog_gtid.go index cec14d0f4..c02b666c2 100644 --- a/go/mysql/binlog_gtid.go +++ b/go/mysql/binlog_gtid.go @@ -44,12 +44,12 @@ func (this *GTIDBinlogCoordinates) Equals(other BinlogCoordinates) bool { return false } - otherBinlogCoordinates := >IDBinlogCoordinates{} - if err := binlogCoordinatesToImplementation(other, otherBinlogCoordinates); err != nil { - panic(err) + otherCoords, ok := other.(*GTIDBinlogCoordinates) + if !ok { + return false } - return this.GTIDSet.Equal(otherBinlogCoordinates.GTIDSet) + return this.GTIDSet.Equal(otherCoords.GTIDSet) } // IsEmpty returns true if the GTID set is empty. @@ -59,14 +59,17 @@ func (this *GTIDBinlogCoordinates) IsEmpty() bool { // SmallerThan returns true if this coordinate is strictly smaller than the other. func (this *GTIDBinlogCoordinates) SmallerThan(other BinlogCoordinates) bool { - otherBinlogCoordinates := >IDBinlogCoordinates{} - if err := binlogCoordinatesToImplementation(other, otherBinlogCoordinates); err != nil { - panic(err) + if other == nil || this.IsEmpty() || other.IsEmpty() { + return false + } + otherCoords, ok := other.(*GTIDBinlogCoordinates) + if !ok { + return false } // if 'this' does not contain the same sets we assume we are behind 'other'. // there are probably edge cases where this isn't true - return !this.GTIDSet.Contain(other.GTIDSet) + return !this.GTIDSet.Contain(otherCoords.GTIDSet) } // SmallerThanOrEquals returns true if this coordinate is the same or equal to the other one. @@ -86,3 +89,14 @@ func (this *GTIDBinlogCoordinates) Update(update interface{}) error { } return nil } + +func (this *GTIDBinlogCoordinates) Clone() BinlogCoordinates { + out := >IDBinlogCoordinates{} + if this.GTIDSet != nil { + out.GTIDSet = this.GTIDSet.Clone().(*gomysql.MysqlGTIDSet) + } + if this.UUIDSet != nil { + out.UUIDSet = this.UUIDSet.Clone() + } + return out +} diff --git a/go/mysql/binlog_test.go b/go/mysql/binlog_test.go deleted file mode 100644 index cd75149e9..000000000 --- a/go/mysql/binlog_test.go +++ /dev/null @@ -1,36 +0,0 @@ -/* - Copyright 2022 GitHub Inc. - See https://github.com/github/gh-ost/blob/master/LICENSE -*/ - -package mysql - -import ( - "testing" - - "github.com/openark/golib/log" - test "github.com/openark/golib/tests" -) - -func init() { - log.SetLevel(log.ERROR) -} - -func TestBinlogCoordinatesToImplementation(t *testing.T) { - test.S(t).ExpectNil(binlogCoordinatesToImplementation( - &FileBinlogCoordinates{}, - &FileBinlogCoordinates{}, - )) - test.S(t).ExpectNil(binlogCoordinatesToImplementation( - >IDBinlogCoordinates{}, - >IDBinlogCoordinates{}, - )) - test.S(t).ExpectNotNil(binlogCoordinatesToImplementation( - &FileBinlogCoordinates{}, - >IDBinlogCoordinates{}, - )) - test.S(t).ExpectNotNil(binlogCoordinatesToImplementation( - &FileBinlogCoordinates{}, - map[string]string{}, - )) -} From 9e76411a5157782bf2458be0ea086981a7220ede Mon Sep 17 00:00:00 2001 From: meiji163 Date: Tue, 7 Oct 2025 15:51:35 -0700 Subject: [PATCH 44/56] add binlog stream retry and toxiproxy test --- doc/local-tests.md | 8 + go/binlog/gomysql_reader.go | 104 +++--- go/cmd/gh-ost/main.go | 1 + go/logic/streamer.go | 23 +- localtests/test.sh | 490 +++++++++++++++-------------- script/docker-gh-ost-replica-tests | 61 +++- 6 files changed, 380 insertions(+), 307 deletions(-) diff --git a/doc/local-tests.md b/doc/local-tests.md index 411418f3b..fb13a79ed 100644 --- a/doc/local-tests.md +++ b/doc/local-tests.md @@ -34,3 +34,11 @@ TEST_MYSQL_IMAGE="mysql-server:8.0.16" ./script/docker-gh-ost-replica-tests up # cleanup containers ./script/docker-gh-ost-replica-tests down ``` + +Pass the `-t` flag to run the tests with a toxiproxy between gh-ost and the MySQL replica. This simulates network conditions where MySQL connections are closed unexpectedly. + +```shell +# run tests with toxiproxy +./script/docker-gh-ost-replica-tests up -t +./script/docker-gh-ost-replica-tests run -t +``` diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index d8102e080..0033ef888 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -22,14 +22,14 @@ import ( ) type GoMySQLReader struct { - migrationContext *base.MigrationContext - connectionConfig *mysql.ConnectionConfig - binlogSyncer *replication.BinlogSyncer - binlogStreamer *replication.BinlogStreamer - currentCoordinates mysql.BinlogCoordinates - currentCoordinatesMutex *sync.Mutex - nextCoordinates mysql.BinlogCoordinates - LastAppliedRowsEventHint mysql.BinlogCoordinates + migrationContext *base.MigrationContext + connectionConfig *mysql.ConnectionConfig + binlogSyncer *replication.BinlogSyncer + binlogStreamer *replication.BinlogStreamer + currentCoordinates mysql.BinlogCoordinates + currentCoordinatesMutex *sync.Mutex + // LastTrxCoords is the coordinates of the last transaction read. + LastTrxCoords mysql.BinlogCoordinates } func NewGoMySQLReader(migrationContext *base.MigrationContext) *GoMySQLReader { @@ -47,8 +47,8 @@ func NewGoMySQLReader(migrationContext *base.MigrationContext) *GoMySQLReader { Password: connectionConfig.Password, TLSConfig: connectionConfig.TLSConfig(), UseDecimal: true, - MaxReconnectAttempts: migrationContext.BinlogSyncerMaxReconnectAttempts, TimestampStringLocation: time.UTC, + DisableRetrySync: true, // we implement our own reconnect. }), } } @@ -80,25 +80,17 @@ func (this *GoMySQLReader) ConnectBinlogStreamer(coordinates mysql.BinlogCoordin func (this *GoMySQLReader) GetCurrentBinlogCoordinates() mysql.BinlogCoordinates { this.currentCoordinatesMutex.Lock() defer this.currentCoordinatesMutex.Unlock() - returnCoordinates := this.currentCoordinates - return returnCoordinates + return this.currentCoordinates.Clone() } func (this *GoMySQLReader) handleRowsEvent(ev *replication.BinlogEvent, rowsEvent *replication.RowsEvent, entriesChannel chan<- *BinlogEntry) error { this.currentCoordinatesMutex.Lock() currentCoords := this.currentCoordinates - lastApplied := this.LastAppliedRowsEventHint this.currentCoordinatesMutex.Unlock() + lastTrx := this.LastTrxCoords - if !this.migrationContext.UseGTIDs { - currentFileCoords := currentCoords.(*mysql.FileBinlogCoordinates) - if lastApplied != nil && currentFileCoords.IsLogPosOverflowBeyond4Bytes(lastApplied.(*mysql.FileBinlogCoordinates)) { - return fmt.Errorf("Unexpected rows event at %+v, the binlog end_log_pos is overflow 4 bytes", currentCoords) - } - if currentCoords.SmallerThanOrEquals(lastApplied) { - this.migrationContext.Log.Debugf("Skipping handled query at %+v (last applied is %+v)", currentCoords, lastApplied) - return nil - } + if currentCoords.SmallerThan(lastTrx) { + this.migrationContext.Log.Debugf("Skipping handled transaction %+v (last trx is %+v)", currentCoords, lastTrx) } dml := ToEventDML(ev.Header.EventType.String()) @@ -139,12 +131,6 @@ func (this *GoMySQLReader) handleRowsEvent(ev *replication.BinlogEvent, rowsEven // In reality, reads will be synchronous entriesChannel <- binlogEntry } - this.currentCoordinatesMutex.Lock() - this.LastAppliedRowsEventHint = this.currentCoordinates.Clone() - if this.migrationContext.UseGTIDs { - this.currentCoordinates = this.nextCoordinates - } - defer this.currentCoordinatesMutex.Unlock() return nil } @@ -161,29 +147,33 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha if err != nil { return err } - func() { + + // update binlog coords if using file-based coords + if !this.migrationContext.UseGTIDs { this.currentCoordinatesMutex.Lock() - defer this.currentCoordinatesMutex.Unlock() - if !this.migrationContext.UseGTIDs { - coords := this.currentCoordinates.(*mysql.FileBinlogCoordinates) - coords.LogPos = int64(ev.Header.LogPos) - coords.EventSize = int64(ev.Header.EventSize) + coords := this.currentCoordinates.(*mysql.FileBinlogCoordinates) + prevCoords := coords.Clone().(*mysql.FileBinlogCoordinates) + coords.LogPos = int64(ev.Header.LogPos) + coords.EventSize = int64(ev.Header.EventSize) + if coords.IsLogPosOverflowBeyond4Bytes(prevCoords) { + this.currentCoordinatesMutex.Unlock() + return fmt.Errorf("Unexpected rows event at %+v, the binlog end_log_pos is overflow 4 bytes", coords) } - }() + this.currentCoordinatesMutex.Unlock() + } switch event := ev.Event.(type) { case *replication.PreviousGTIDsEvent: if !this.migrationContext.UseGTIDs { continue } - func() { - this.currentCoordinatesMutex.Lock() - defer this.currentCoordinatesMutex.Unlock() - coords := this.currentCoordinates.(*mysql.GTIDBinlogCoordinates) - if err := coords.GTIDSet.Update(event.GTIDSets); err != nil { - this.migrationContext.Log.Errorf("Failed to parse GTID set: %v", err) - } - }() + this.currentCoordinatesMutex.Lock() + coords := this.currentCoordinates.(*mysql.GTIDBinlogCoordinates) + if err := coords.GTIDSet.Update(event.GTIDSets); err != nil { + this.currentCoordinatesMutex.Unlock() + return this.migrationContext.Log.Errorf("Failed to parse GTID set: %v", err) + } + this.currentCoordinatesMutex.Unlock() case *replication.GTIDEvent: if !this.migrationContext.UseGTIDs { continue @@ -192,25 +182,25 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha if err != nil { return err } - func() { - this.currentCoordinatesMutex.Lock() - defer this.currentCoordinatesMutex.Unlock() - this.nextCoordinates = this.currentCoordinates - interval := gomysql.Interval{Start: event.GNO, Stop: event.GNO + 1} - coords := this.currentCoordinates.(*mysql.GTIDBinlogCoordinates) - coords.GTIDSet.AddSet(gomysql.NewUUIDSet(sid, interval)) - }() + interval := gomysql.Interval{Start: event.GNO, Stop: event.GNO + 1} + + this.currentCoordinatesMutex.Lock() + coords := this.currentCoordinates.(*mysql.GTIDBinlogCoordinates) + coords.GTIDSet.AddSet(gomysql.NewUUIDSet(sid, interval)) + this.currentCoordinatesMutex.Unlock() case *replication.RotateEvent: if this.migrationContext.UseGTIDs { continue } - func() { - this.currentCoordinatesMutex.Lock() - defer this.currentCoordinatesMutex.Unlock() - coords := this.currentCoordinates.(*mysql.FileBinlogCoordinates) - coords.LogFile = string(event.NextLogName) - this.migrationContext.Log.Infof("rotate to next log from %s:%d to %s", coords.LogFile, int64(ev.Header.LogPos), event.NextLogName) - }() + this.currentCoordinatesMutex.Lock() + coords := this.currentCoordinates.(*mysql.FileBinlogCoordinates) + coords.LogFile = string(event.NextLogName) + this.migrationContext.Log.Infof("rotate to next log from %s:%d to %s", coords.LogFile, int64(ev.Header.LogPos), event.NextLogName) + this.currentCoordinatesMutex.Unlock() + case *replication.XIDEvent: + this.currentCoordinatesMutex.Lock() + this.LastTrxCoords = this.currentCoordinates.Clone() + this.currentCoordinatesMutex.Unlock() case *replication.RowsEvent: if err := this.handleRowsEvent(ev, event, entriesChannel); err != nil { return err diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 16e548429..6391cf4fb 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -144,6 +144,7 @@ func main() { flag.BoolVar(&migrationContext.IncludeTriggers, "include-triggers", false, "When true, the triggers (if exist) will be created on the new table") flag.StringVar(&migrationContext.TriggerSuffix, "trigger-suffix", "", "Add a suffix to the trigger name (i.e '_v2'). Requires '--include-triggers'") flag.BoolVar(&migrationContext.RemoveTriggerSuffix, "remove-trigger-suffix-if-exists", false, "Remove given suffix from name of trigger. Requires '--include-triggers' and '--trigger-suffix'") + flag.BoolVar(&migrationContext.SkipPortValidation, "skip-port-validation", false, "Skip port validation for MySQL connections") maxLoad := flag.String("max-load", "", "Comma delimited status-name=threshold. e.g: 'Threads_running=100,Threads_connected=500'. When status exceeds threshold, app throttles writes") criticalLoad := flag.String("critical-load", "", "Comma delimited status-name=threshold, same format as --max-load. When status exceeds threshold, app panics and quits") diff --git a/go/logic/streamer.go b/go/logic/streamer.go index 4fd6dd315..d222a48f7 100644 --- a/go/logic/streamer.go +++ b/go/logic/streamer.go @@ -29,7 +29,7 @@ type BinlogEventListener struct { const ( EventsChannelBufferSize = 1 - ReconnectStreamerSleepSeconds = 5 + ReconnectStreamerSleepSeconds = 1 ) // EventsStreamer reads data from binary logs and streams it on. It acts as a publisher, @@ -192,8 +192,8 @@ func (this *EventsStreamer) StreamEvents(canStopStreaming func() bool) error { } }() // The next should block and execute forever, unless there's a serious error - var successiveFailures int64 - var lastAppliedRowsEventHint mysql.BinlogCoordinates + var successiveFailures int + var reconnectCoords mysql.BinlogCoordinates for { if canStopStreaming() { return nil @@ -208,22 +208,25 @@ func (this *EventsStreamer) StreamEvents(canStopStreaming func() bool) error { time.Sleep(ReconnectStreamerSleepSeconds * time.Second) // See if there's retry overflow - if this.binlogReader.LastAppliedRowsEventHint.Equals(lastAppliedRowsEventHint) { + if this.binlogReader.LastTrxCoords.SmallerThanOrEquals(reconnectCoords) { successiveFailures += 1 } else { successiveFailures = 0 } - if successiveFailures >= this.migrationContext.MaxRetries() { + if successiveFailures >= this.migrationContext.BinlogSyncerMaxReconnectAttempts { return fmt.Errorf("%d successive failures in streamer reconnect at coordinates %+v", successiveFailures, this.GetReconnectBinlogCoordinates()) } - // Reposition at same binlog file. - lastAppliedRowsEventHint = this.binlogReader.LastAppliedRowsEventHint - this.migrationContext.Log.Infof("Reconnecting... Will resume at %+v", lastAppliedRowsEventHint) - if err := this.initBinlogReader(this.GetReconnectBinlogCoordinates()); err != nil { + // Reposition at same coordinates + reconnectCoords = this.binlogReader.LastTrxCoords.Clone() + if reconnectCoords.IsEmpty() { + // no transactions were handled yet + reconnectCoords = this.initialBinlogCoordinates.Clone() + } + this.migrationContext.Log.Infof("Reconnecting EventsStreamer... Will resume at %+v", reconnectCoords) + if err := this.initBinlogReader(reconnectCoords); err != nil { return err } - this.binlogReader.LastAppliedRowsEventHint = lastAppliedRowsEventHint } } } diff --git a/localtests/test.sh b/localtests/test.sh index 8410ef2d5..e7ff02ff5 100755 --- a/localtests/test.sh +++ b/localtests/test.sh @@ -12,6 +12,8 @@ test_logfile=/tmp/gh-ost-test.log default_ghost_binary=/tmp/gh-ost-test ghost_binary="" docker=false +toxiproxy=false +gtid=false storage_engine=innodb exec_command_file=/tmp/gh-ost-test.bash ghost_structure_output_file=/tmp/gh-ost-test.ghost.structure.sql @@ -27,167 +29,187 @@ original_sql_mode= current_gtid_mode= OPTIND=1 -while getopts "b:s:d" OPTION -do - case $OPTION in +while getopts "b:s:dtg" OPTION; do + case $OPTION in b) - ghost_binary="$OPTARG";; + ghost_binary="$OPTARG" + ;; s) - storage_engine="$OPTARG";; + storage_engine="$OPTARG" + ;; d) - docker=true;; - esac + docker=true + ;; + t) + toxiproxy=true + ;; + g) + gtid=true + ;; + esac done -shift $((OPTIND-1)) +shift $((OPTIND - 1)) test_pattern="${1:-.}" verify_master_and_replica() { - if [ "$(gh-ost-test-mysql-master -e "select 1" -ss)" != "1" ] ; then - echo "Cannot verify gh-ost-test-mysql-master" - exit 1 - fi - read master_host master_port <<< $(gh-ost-test-mysql-master -e "select @@hostname, @@port" -ss) - [ "$master_host" == "$(hostname)" ] && master_host="127.0.0.1" - echo "# master verified at $master_host:$master_port" - if ! gh-ost-test-mysql-master -e "set global event_scheduler := 1" ; then - echo "Cannot enable event_scheduler on master" - exit 1 - fi - original_sql_mode="$(gh-ost-test-mysql-master -e "select @@global.sql_mode" -s -s)" - echo "sql_mode on master is ${original_sql_mode}" - - current_gtid_mode=$(gh-ost-test-mysql-master -s -s -e "select @@global.gtid_mode" 2>/dev/null || echo unsupported) - current_enforce_gtid_consistency=$(gh-ost-test-mysql-master -s -s -e "select @@global.enforce_gtid_consistency" 2>/dev/null || echo unsupported) - current_master_server_uuid=$(gh-ost-test-mysql-master -s -s -e "select @@global.server_uuid" 2>/dev/null || echo unsupported) - current_replica_server_uuid=$(gh-ost-test-mysql-replica -s -s -e "select @@global.server_uuid" 2>/dev/null || echo unsupported) - echo "gtid_mode on master is ${current_gtid_mode} with enforce_gtid_consistency=${current_enforce_gtid_consistency}" - echo "server_uuid on master is ${current_master_server_uuid}, replica is ${current_replica_server_uuid}" - - echo "Gracefully sleeping for 3 seconds while replica is setting up..." - sleep 3 - - if [ "$(gh-ost-test-mysql-replica -e "select 1" -ss)" != "1" ] ; then - echo "Cannot verify gh-ost-test-mysql-replica" - exit 1 - fi - if [ "$(gh-ost-test-mysql-replica -e "select @@global.binlog_format" -ss)" != "ROW" ] ; then - echo "Expecting test replica to have binlog_format=ROW" - exit 1 - fi - read replica_host replica_port <<< $(gh-ost-test-mysql-replica -e "select @@hostname, @@port" -ss) - [ "$replica_host" == "$(hostname)" ] && replica_host="127.0.0.1" - echo "# replica verified at $replica_host:$replica_port" + if [ "$(gh-ost-test-mysql-master -e "select 1" -ss)" != "1" ]; then + echo "Cannot verify gh-ost-test-mysql-master" + exit 1 + fi + read master_host master_port <<<$(gh-ost-test-mysql-master -e "select @@hostname, @@port" -ss) + [ "$master_host" == "$(hostname)" ] && master_host="127.0.0.1" + echo "# master verified at $master_host:$master_port" + if ! gh-ost-test-mysql-master -e "set global event_scheduler := 1"; then + echo "Cannot enable event_scheduler on master" + exit 1 + fi + original_sql_mode="$(gh-ost-test-mysql-master -e "select @@global.sql_mode" -s -s)" + echo "sql_mode on master is ${original_sql_mode}" + + current_gtid_mode=$(gh-ost-test-mysql-master -s -s -e "select @@global.gtid_mode" 2>/dev/null || echo unsupported) + current_enforce_gtid_consistency=$(gh-ost-test-mysql-master -s -s -e "select @@global.enforce_gtid_consistency" 2>/dev/null || echo unsupported) + current_master_server_uuid=$(gh-ost-test-mysql-master -s -s -e "select @@global.server_uuid" 2>/dev/null || echo unsupported) + current_replica_server_uuid=$(gh-ost-test-mysql-replica -s -s -e "select @@global.server_uuid" 2>/dev/null || echo unsupported) + echo "gtid_mode on master is ${current_gtid_mode} with enforce_gtid_consistency=${current_enforce_gtid_consistency}" + echo "server_uuid on master is ${current_master_server_uuid}, replica is ${current_replica_server_uuid}" + + echo "Gracefully sleeping for 3 seconds while replica is setting up..." + sleep 3 + + if [ "$(gh-ost-test-mysql-replica -e "select 1" -ss)" != "1" ]; then + echo "Cannot verify gh-ost-test-mysql-replica" + exit 1 + fi + if [ "$(gh-ost-test-mysql-replica -e "select @@global.binlog_format" -ss)" != "ROW" ]; then + echo "Expecting test replica to have binlog_format=ROW" + exit 1 + fi + read replica_host replica_port <<<$(gh-ost-test-mysql-replica -e "select @@hostname, @@port" -ss) + [ "$replica_host" == "$(hostname)" ] && replica_host="127.0.0.1" + echo "# replica verified at $replica_host:$replica_port" + + if [ "$docker" = true ]; then + master_host="0.0.0.0" + master_port="3307" + echo "# using docker master at $master_host:$master_port" + replica_host="0.0.0.0" + if [ "$toxiproxy" = true ]; then + replica_port="23308" + echo "# using toxiproxy replica at $replica_host:$replica_port" + else + replica_port="3308" + echo "# using docker replica at $replica_host:$replica_port" + fi + fi } exec_cmd() { - echo "$@" - command "$@" 1> $test_logfile 2>&1 - return $? + echo "$@" + command "$@" 1>$test_logfile 2>&1 + return $? } echo_dot() { - echo -n "." + echo -n "." } start_replication() { - mysql_version="$(gh-ost-test-mysql-replica -e "select @@version")" - if [[ $mysql_version =~ "8.4" ]]; then - seconds_behind_source="Seconds_Behind_Source" - replica_terminology="replica" - else - seconds_behind_source="Seconds_Behind_Master" - replica_terminology="slave" - fi - gh-ost-test-mysql-replica -e "stop $replica_terminology; start $replica_terminology;" - - num_attempts=0 - while gh-ost-test-mysql-replica -e "show $replica_terminology status\G" | grep $seconds_behind_source | grep -q NULL ; do - ((num_attempts=num_attempts+1)) - if [ $num_attempts -gt 10 ] ; then - echo - echo "ERROR replication failure" - exit 1 + mysql_version="$(gh-ost-test-mysql-replica -e "select @@version")" + if [[ $mysql_version =~ "8.4" ]]; then + seconds_behind_source="Seconds_Behind_Source" + replica_terminology="replica" + else + seconds_behind_source="Seconds_Behind_Master" + replica_terminology="slave" fi - echo_dot - sleep 1 - done + gh-ost-test-mysql-replica -e "stop $replica_terminology; start $replica_terminology;" + + num_attempts=0 + while gh-ost-test-mysql-replica -e "show $replica_terminology status\G" | grep $seconds_behind_source | grep -q NULL; do + ((num_attempts = num_attempts + 1)) + if [ $num_attempts -gt 10 ]; then + echo + echo "ERROR replication failure" + exit 1 + fi + echo_dot + sleep 1 + done } test_single() { - local test_name - test_name="$1" - - if [ "$docker" = true ]; then - master_host="0.0.0.0" - master_port="3307" - replica_host="0.0.0.0" - replica_port="3308" - fi - - if [ -f $tests_path/$test_name/ignore_versions ] ; then - ignore_versions=$(cat $tests_path/$test_name/ignore_versions) - mysql_version=$(gh-ost-test-mysql-master -s -s -e "select @@version") - mysql_version_comment=$(gh-ost-test-mysql-master -s -s -e "select @@version_comment") - if echo "$mysql_version" | egrep -q "^${ignore_versions}" ; then - echo -n "Skipping: $test_name" - return 0 - elif echo "$mysql_version_comment" | egrep -i -q "^${ignore_versions}" ; then - echo -n "Skipping: $test_name" - return 0 + local test_name + test_name="$1" + if [ -f $tests_path/$test_name/ignore_versions ]; then + ignore_versions=$(cat $tests_path/$test_name/ignore_versions) + mysql_version=$(gh-ost-test-mysql-master -s -s -e "select @@version") + mysql_version_comment=$(gh-ost-test-mysql-master -s -s -e "select @@version_comment") + if echo "$mysql_version" | egrep -q "^${ignore_versions}"; then + echo -n "Skipping: $test_name" + return 0 + elif echo "$mysql_version_comment" | egrep -i -q "^${ignore_versions}"; then + echo -n "Skipping: $test_name" + return 0 + fi fi - fi - echo -n "Testing: $test_name" + echo -n "Testing: $test_name" - echo_dot - start_replication - echo_dot + echo_dot + start_replication + echo_dot - if [ -f $tests_path/$test_name/gtid_mode ] ; then - target_gtid_mode=$(cat $tests_path/$test_name/gtid_mode) - if [ "$current_gtid_mode" != "$target_gtid_mode" ] ; then - echo "gtid_mode is ${current_gtid_mode}, expected ${target_gtid_mode}" - exit 1 + if [ -f $tests_path/$test_name/gtid_mode ]; then + target_gtid_mode=$(cat $tests_path/$test_name/gtid_mode) + if [ "$current_gtid_mode" != "$target_gtid_mode" ]; then + echo "gtid_mode is ${current_gtid_mode}, expected ${target_gtid_mode}" + exit 1 + fi + fi + + if [ -f $tests_path/$test_name/sql_mode ]; then + gh-ost-test-mysql-master --default-character-set=utf8mb4 test -e "set @@global.sql_mode='$(cat $tests_path/$test_name/sql_mode)'" + gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "set @@global.sql_mode='$(cat $tests_path/$test_name/sql_mode)'" + fi + + gh-ost-test-mysql-master --default-character-set=utf8mb4 test <$tests_path/$test_name/create.sql + test_create_result=$? + + if [ $test_create_result -ne 0 ]; then + echo + echo "ERROR $test_name create failure. cat $tests_path/$test_name/create.sql:" + cat $tests_path/$test_name/create.sql + return 1 + fi + + extra_args="" + if [ -f $tests_path/$test_name/extra_args ]; then + extra_args=$(cat $tests_path/$test_name/extra_args) + fi + if [ "$gtid" = true ]; then + extra_args+=" --gtid" + fi + if [ "$toxiproxy" = true ]; then + extra_args+=" --skip-port-validation" + fi + orig_columns="*" + ghost_columns="*" + order_by="" + if [ -f $tests_path/$test_name/orig_columns ]; then + orig_columns=$(cat $tests_path/$test_name/orig_columns) + fi + if [ -f $tests_path/$test_name/ghost_columns ]; then + ghost_columns=$(cat $tests_path/$test_name/ghost_columns) + fi + if [ -f $tests_path/$test_name/order_by ]; then + order_by="order by $(cat $tests_path/$test_name/order_by)" fi - fi - - if [ -f $tests_path/$test_name/sql_mode ] ; then - gh-ost-test-mysql-master --default-character-set=utf8mb4 test -e "set @@global.sql_mode='$(cat $tests_path/$test_name/sql_mode)'" - gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "set @@global.sql_mode='$(cat $tests_path/$test_name/sql_mode)'" - fi - - gh-ost-test-mysql-master --default-character-set=utf8mb4 test < $tests_path/$test_name/create.sql - test_create_result=$? - - if [ $test_create_result -ne 0 ] ; then - echo - echo "ERROR $test_name create failure. cat $tests_path/$test_name/create.sql:" - cat $tests_path/$test_name/create.sql - return 1 - fi - - extra_args="" - if [ -f $tests_path/$test_name/extra_args ] ; then - extra_args=$(cat $tests_path/$test_name/extra_args) - fi - orig_columns="*" - ghost_columns="*" - order_by="" - if [ -f $tests_path/$test_name/orig_columns ] ; then - orig_columns=$(cat $tests_path/$test_name/orig_columns) - fi - if [ -f $tests_path/$test_name/ghost_columns ] ; then - ghost_columns=$(cat $tests_path/$test_name/ghost_columns) - fi - if [ -f $tests_path/$test_name/order_by ] ; then - order_by="order by $(cat $tests_path/$test_name/order_by)" - fi - # graceful sleep for replica to catch up - echo_dot - sleep 1 - # - cmd="$ghost_binary \ + # graceful sleep for replica to catch up + echo_dot + sleep 1 + # + cmd="$ghost_binary \ --user=gh-ost \ --password=gh-ost \ --host=$replica_host \ @@ -212,118 +234,118 @@ test_single() { --debug \ --stack \ --execute ${extra_args[@]}" - echo_dot - echo $cmd > $exec_command_file - echo_dot - bash $exec_command_file 1> $test_logfile 2>&1 - - execution_result=$? - - if [ -f $tests_path/$test_name/sql_mode ] ; then - gh-ost-test-mysql-master --default-character-set=utf8mb4 test -e "set @@global.sql_mode='${original_sql_mode}'" - gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "set @@global.sql_mode='${original_sql_mode}'" - fi - - if [ -f $tests_path/$test_name/destroy.sql ] ; then - gh-ost-test-mysql-master --default-character-set=utf8mb4 test < $tests_path/$test_name/destroy.sql - fi - - if [ -f $tests_path/$test_name/expect_failure ] ; then - if [ $execution_result -eq 0 ] ; then - echo - echo "ERROR $test_name execution was expected to exit on error but did not. cat $test_logfile" - return 1 + echo_dot + echo $cmd >$exec_command_file + echo_dot + bash $exec_command_file 1>$test_logfile 2>&1 + + execution_result=$? + + if [ -f $tests_path/$test_name/sql_mode ]; then + gh-ost-test-mysql-master --default-character-set=utf8mb4 test -e "set @@global.sql_mode='${original_sql_mode}'" + gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "set @@global.sql_mode='${original_sql_mode}'" fi - if [ -s $tests_path/$test_name/expect_failure ] ; then - # 'expect_failure' file has content. We expect to find this content in the log. - expected_error_message="$(cat $tests_path/$test_name/expect_failure)" - if grep -q "$expected_error_message" $test_logfile ; then - return 0 - fi - echo - echo "ERROR $test_name execution was expected to exit with error message '${expected_error_message}' but did not. cat $test_logfile" - return 1 + + if [ -f $tests_path/$test_name/destroy.sql ]; then + gh-ost-test-mysql-master --default-character-set=utf8mb4 test <$tests_path/$test_name/destroy.sql + fi + + if [ -f $tests_path/$test_name/expect_failure ]; then + if [ $execution_result -eq 0 ]; then + echo + echo "ERROR $test_name execution was expected to exit on error but did not. cat $test_logfile" + return 1 + fi + if [ -s $tests_path/$test_name/expect_failure ]; then + # 'expect_failure' file has content. We expect to find this content in the log. + expected_error_message="$(cat $tests_path/$test_name/expect_failure)" + if grep -q "$expected_error_message" $test_logfile; then + return 0 + fi + echo + echo "ERROR $test_name execution was expected to exit with error message '${expected_error_message}' but did not. cat $test_logfile" + return 1 + fi + # 'expect_failure' file has no content. We generally agree that the failure is correct + return 0 fi - # 'expect_failure' file has no content. We generally agree that the failure is correct - return 0 - fi - - if [ $execution_result -ne 0 ] ; then - echo - echo "ERROR $test_name execution failure. cat $test_logfile:" - cat $test_logfile - return 1 - fi - - gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "show create table _gh_ost_test_gho\G" -ss > $ghost_structure_output_file - - if [ -f $tests_path/$test_name/expect_table_structure ] ; then - expected_table_structure="$(cat $tests_path/$test_name/expect_table_structure)" - if ! grep -q "$expected_table_structure" $ghost_structure_output_file ; then - echo - echo "ERROR $test_name: table structure was expected to include ${expected_table_structure} but did not. cat $ghost_structure_output_file:" - cat $ghost_structure_output_file - return 1 + + if [ $execution_result -ne 0 ]; then + echo + echo "ERROR $test_name execution failure. cat $test_logfile:" + cat $test_logfile + return 1 fi - fi - echo_dot - gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "select ${orig_columns} from gh_ost_test ${order_by}" -ss > $orig_content_output_file - gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "select ${ghost_columns} from _gh_ost_test_gho ${order_by}" -ss > $ghost_content_output_file - orig_checksum=$(cat $orig_content_output_file | md5sum) - ghost_checksum=$(cat $ghost_content_output_file | md5sum) + gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "show create table _gh_ost_test_gho\G" -ss >$ghost_structure_output_file - if [ "$orig_checksum" != "$ghost_checksum" ] ; then - gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "select ${orig_columns} from gh_ost_test" -ss > $orig_content_output_file - gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "select ${ghost_columns} from _gh_ost_test_gho" -ss > $ghost_content_output_file - echo "ERROR $test_name: checksum mismatch" - echo "---" - diff $orig_content_output_file $ghost_content_output_file + if [ -f $tests_path/$test_name/expect_table_structure ]; then + expected_table_structure="$(cat $tests_path/$test_name/expect_table_structure)" + if ! grep -q "$expected_table_structure" $ghost_structure_output_file; then + echo + echo "ERROR $test_name: table structure was expected to include ${expected_table_structure} but did not. cat $ghost_structure_output_file:" + cat $ghost_structure_output_file + return 1 + fi + fi - echo "diff $orig_content_output_file $ghost_content_output_file" + echo_dot + gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "select ${orig_columns} from gh_ost_test ${order_by}" -ss >$orig_content_output_file + gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "select ${ghost_columns} from _gh_ost_test_gho ${order_by}" -ss >$ghost_content_output_file + orig_checksum=$(cat $orig_content_output_file | md5sum) + ghost_checksum=$(cat $ghost_content_output_file | md5sum) - return 1 - fi + if [ "$orig_checksum" != "$ghost_checksum" ]; then + gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "select ${orig_columns} from gh_ost_test" -ss >$orig_content_output_file + gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "select ${ghost_columns} from _gh_ost_test_gho" -ss >$ghost_content_output_file + echo "ERROR $test_name: checksum mismatch" + echo "---" + diff $orig_content_output_file $ghost_content_output_file + + echo "diff $orig_content_output_file $ghost_content_output_file" + + return 1 + fi } build_binary() { - echo "Building" - rm -f $default_ghost_binary - [ "$ghost_binary" == "" ] && ghost_binary="$default_ghost_binary" - if [ -f "$ghost_binary" ] ; then - echo "Using binary: $ghost_binary" - return 0 - fi - - go build -o $ghost_binary go/cmd/gh-ost/main.go - - if [ $? -ne 0 ] ; then - echo "Build failure" - exit 1 - fi + echo "Building" + rm -f $default_ghost_binary + [ "$ghost_binary" == "" ] && ghost_binary="$default_ghost_binary" + if [ -f "$ghost_binary" ]; then + echo "Using binary: $ghost_binary" + return 0 + fi + + go build -o $ghost_binary go/cmd/gh-ost/main.go + + if [ $? -ne 0 ]; then + echo "Build failure" + exit 1 + fi } test_all() { - build_binary - test_dirs=$(find "$tests_path" -mindepth 1 -maxdepth 1 ! -path . -type d | grep "$test_pattern" | sort) - while read -r test_dir; do - test_name=$(basename "$test_dir") - if ! test_single "$test_name" ; then - create_statement=$(gh-ost-test-mysql-replica test -t -e "show create table _gh_ost_test_gho \G") - echo "$create_statement" >> $test_logfile - echo "+ FAIL" - return 1 - else - echo - echo "+ pass" - fi - mysql_version="$(gh-ost-test-mysql-replica -e "select @@version")" - replica_terminology="slave" - if [[ $mysql_version =~ "8.4" ]]; then - replica_terminology="replica" - fi - gh-ost-test-mysql-replica -e "start $replica_terminology" - done <<< "$test_dirs" + build_binary + test_dirs=$(find "$tests_path" -mindepth 1 -maxdepth 1 ! -path . -type d | grep "$test_pattern" | sort) + while read -r test_dir; do + test_name=$(basename "$test_dir") + if ! test_single "$test_name"; then + create_statement=$(gh-ost-test-mysql-replica test -t -e "show create table _gh_ost_test_gho \G") + echo "$create_statement" >>$test_logfile + echo "+ FAIL" + return 1 + else + echo + echo "+ pass" + fi + mysql_version="$(gh-ost-test-mysql-replica -e "select @@version")" + replica_terminology="slave" + if [[ $mysql_version =~ "8.4" ]]; then + replica_terminology="replica" + fi + gh-ost-test-mysql-replica -e "start $replica_terminology" + done <<<"$test_dirs" } verify_master_and_replica diff --git a/script/docker-gh-ost-replica-tests b/script/docker-gh-ost-replica-tests index 4469694d3..7169574a8 100755 --- a/script/docker-gh-ost-replica-tests +++ b/script/docker-gh-ost-replica-tests @@ -5,11 +5,15 @@ # Set the environment var TEST_MYSQL_IMAGE to change the docker image. # # Usage: -# docker-gh-ost-replica-tests up start the containers -# docker-gh-ost-replica-tests down remove the containers -# docker-gh-ost-replica-tests run run replica tests on the containers +# docker-gh-ost-replica-tests up [-t] start the containers +# docker-gh-ost-replica-tests down remove the containers +# docker-gh-ost-replica-tests run [-t] run replica tests on the containers +# +# Flags: +# -t use a toxiproxy for replica connection to simulate dropped connections set -e +toxiproxy=false GH_OST_ROOT=$(git rev-parse --show-toplevel) if [[ ":$PATH:" != *":$GH_OST_ROOT:"* ]]; then @@ -47,6 +51,22 @@ mysql-replica() { fi } +create_toxiproxy() { + curl --fail -X POST http://localhost:8474/proxies \ + -H "Content-Type: application/json" \ + -d '{"name": "mysql_proxy", + "listen": "0.0.0.0:23308", + "upstream": "host.docker.internal:3308"}' + echo + + curl --fail -X POST http://localhost:8474/proxies/mysql_proxy/toxics \ + -H "Content-Type: application/json" \ + -d '{"name": "limit_data_downstream", + "type": "limit_data", + "attributes": {"bytes": 40000}}' + echo +} + setup() { [ -z "$TEST_MYSQL_IMAGE" ] && TEST_MYSQL_IMAGE="mysql:8.0.41" @@ -59,6 +79,22 @@ setup() { MYSQL_SHA2_RSA_KEYS_FLAG="--caching-sha2-password-auto-generate-rsa-keys=ON" fi (TEST_MYSQL_IMAGE="$TEST_MYSQL_IMAGE" MYSQL_SHA2_RSA_KEYS_FLAG="$MYSQL_SHA2_RSA_KEYS_FLAG" envsubst <"$compose_file") >"$compose_file.tmp" + + if [ "$toxiproxy" = true ]; then + echo "Starting toxiproxy container..." + cat <>"$compose_file.tmp" + mysql-toxiproxy: + image: "ghcr.io/shopify/toxiproxy:latest" + container_name: mysql-toxiproxy + ports: + - '8474:8474' + - '23308:23308' + expose: + - '23308' + - '8474' +EOF + fi + docker compose -f "$compose_file.tmp" up -d --wait echo "Waiting for MySQL..." @@ -80,23 +116,36 @@ setup() { mysql-replica -e "start slave;" fi echo "OK" + + if [ "$toxiproxy" = true ]; then + echo "Creating toxiproxy..." + create_toxiproxy + echo "OK" + fi } teardown() { echo "Stopping containers..." docker stop mysql-replica docker stop mysql-primary + docker stop mysql-toxiproxy 2>/dev/null || true echo "Removing containers..." docker rm mysql-replica docker rm mysql-primary + docker rm mysql-toxiproxy 2>/dev/null || true } main() { - if [[ "$1" == "up" ]]; then + local cmd="$1" + local tflag= + if [[ "$2" == "-t" ]]; then + toxiproxy=true + fi + if [[ "$cmd" == "up" ]]; then setup - elif [[ "$1" == "down" ]]; then + elif [[ "$cmd" == "down" ]]; then teardown - elif [[ "$1" == "run" ]]; then + elif [[ "$cmd" == "run" ]]; then shift 1 "$GH_OST_ROOT/localtests/test.sh" -d "$@" fi From afb6e5c2fbc84f5444ffc007e830d41291b23769 Mon Sep 17 00:00:00 2001 From: meiji163 Date: Tue, 7 Oct 2025 15:59:59 -0700 Subject: [PATCH 45/56] fix max retry --- go/logic/streamer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/logic/streamer.go b/go/logic/streamer.go index d222a48f7..17eea181d 100644 --- a/go/logic/streamer.go +++ b/go/logic/streamer.go @@ -213,7 +213,7 @@ func (this *EventsStreamer) StreamEvents(canStopStreaming func() bool) error { } else { successiveFailures = 0 } - if successiveFailures >= this.migrationContext.BinlogSyncerMaxReconnectAttempts { + if this.migrationContext.BinlogSyncerMaxReconnectAttempts > 0 && successiveFailures >= this.migrationContext.BinlogSyncerMaxReconnectAttempts { return fmt.Errorf("%d successive failures in streamer reconnect at coordinates %+v", successiveFailures, this.GetReconnectBinlogCoordinates()) } From c2044ccdd1f93a1dd69fc6e86555e2f7ffd21fac Mon Sep 17 00:00:00 2001 From: meiji163 Date: Tue, 7 Oct 2025 22:05:03 -0700 Subject: [PATCH 46/56] use AddGTID instead of AddSet --- go/binlog/gomysql_reader.go | 4 +--- go/logic/streamer.go | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index 0033ef888..e130ded23 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -182,11 +182,9 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha if err != nil { return err } - interval := gomysql.Interval{Start: event.GNO, Stop: event.GNO + 1} - this.currentCoordinatesMutex.Lock() coords := this.currentCoordinates.(*mysql.GTIDBinlogCoordinates) - coords.GTIDSet.AddSet(gomysql.NewUUIDSet(sid, interval)) + coords.GTIDSet.AddGTID(sid, event.GNO) this.currentCoordinatesMutex.Unlock() case *replication.RotateEvent: if this.migrationContext.UseGTIDs { diff --git a/go/logic/streamer.go b/go/logic/streamer.go index 17eea181d..8cb6acd8e 100644 --- a/go/logic/streamer.go +++ b/go/logic/streamer.go @@ -224,6 +224,7 @@ func (this *EventsStreamer) StreamEvents(canStopStreaming func() bool) error { reconnectCoords = this.initialBinlogCoordinates.Clone() } this.migrationContext.Log.Infof("Reconnecting EventsStreamer... Will resume at %+v", reconnectCoords) + this.binlogReader.Close() if err := this.initBinlogReader(reconnectCoords); err != nil { return err } From 59f258127effaf4e7cfccf22881fa9ce37e2824f Mon Sep 17 00:00:00 2001 From: meiji163 Date: Wed, 8 Oct 2025 10:59:27 -0700 Subject: [PATCH 47/56] remove previous GTID event handling --- go/binlog/gomysql_reader.go | 11 ----------- go/logic/streamer.go | 2 +- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index e130ded23..d418e5582 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -163,17 +163,6 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha } switch event := ev.Event.(type) { - case *replication.PreviousGTIDsEvent: - if !this.migrationContext.UseGTIDs { - continue - } - this.currentCoordinatesMutex.Lock() - coords := this.currentCoordinates.(*mysql.GTIDBinlogCoordinates) - if err := coords.GTIDSet.Update(event.GTIDSets); err != nil { - this.currentCoordinatesMutex.Unlock() - return this.migrationContext.Log.Errorf("Failed to parse GTID set: %v", err) - } - this.currentCoordinatesMutex.Unlock() case *replication.GTIDEvent: if !this.migrationContext.UseGTIDs { continue diff --git a/go/logic/streamer.go b/go/logic/streamer.go index 8cb6acd8e..1ba1593b2 100644 --- a/go/logic/streamer.go +++ b/go/logic/streamer.go @@ -224,7 +224,7 @@ func (this *EventsStreamer) StreamEvents(canStopStreaming func() bool) error { reconnectCoords = this.initialBinlogCoordinates.Clone() } this.migrationContext.Log.Infof("Reconnecting EventsStreamer... Will resume at %+v", reconnectCoords) - this.binlogReader.Close() + _ = this.binlogReader.Close() if err := this.initBinlogReader(reconnectCoords); err != nil { return err } From 0794428d83bd30d8ed50795f5f53b3bb321e581b Mon Sep 17 00:00:00 2001 From: meiji163 Date: Wed, 8 Oct 2025 14:57:03 -0700 Subject: [PATCH 48/56] use AddSet --- go/binlog/gomysql_reader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index d418e5582..0ff1c961b 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -173,7 +173,7 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha } this.currentCoordinatesMutex.Lock() coords := this.currentCoordinates.(*mysql.GTIDBinlogCoordinates) - coords.GTIDSet.AddGTID(sid, event.GNO) + coords.GTIDSet.AddSet(gomysql.NewUUIDSet(sid, gomysql.Interval{Start: event.GNO, Stop: event.GNO + 1})) this.currentCoordinatesMutex.Unlock() case *replication.RotateEvent: if this.migrationContext.UseGTIDs { From de0417aef945886e6786342b362fd8dbf7dd8955 Mon Sep 17 00:00:00 2001 From: meiji163 Date: Wed, 8 Oct 2025 17:45:18 -0700 Subject: [PATCH 49/56] modify GTID coord tracking --- go/binlog/gomysql_reader.go | 28 ++++++++++++++++++++-------- go/logic/streamer.go | 2 +- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index 0ff1c961b..d09e8af87 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -28,7 +28,9 @@ type GoMySQLReader struct { binlogStreamer *replication.BinlogStreamer currentCoordinates mysql.BinlogCoordinates currentCoordinatesMutex *sync.Mutex - // LastTrxCoords is the coordinates of the last transaction read. + // LastTrxCoords tracks the coordinates of the last transaction read. + // It is the GTID set of the transaction, or the coordinates of + // the transaction's XID event if using file coordinates. LastTrxCoords mysql.BinlogCoordinates } @@ -59,12 +61,14 @@ func (this *GoMySQLReader) ConnectBinlogStreamer(coordinates mysql.BinlogCoordin return this.migrationContext.Log.Errorf("Empty coordinates at ConnectBinlogStreamer()") } + this.currentCoordinatesMutex.Lock() + defer this.currentCoordinatesMutex.Unlock() this.currentCoordinates = coordinates - this.migrationContext.Log.Infof("Connecting binlog streamer at %+v", this.currentCoordinates) + this.migrationContext.Log.Infof("Connecting binlog streamer at %+v", coordinates) // Start sync with specified GTID set or binlog file and position if this.migrationContext.UseGTIDs { - coords := this.currentCoordinates.(*mysql.GTIDBinlogCoordinates) + coords := coordinates.(*mysql.GTIDBinlogCoordinates) this.binlogStreamer, err = this.binlogSyncer.StartSyncGTID(coords.GTIDSet) } else { coords := this.currentCoordinates.(*mysql.FileBinlogCoordinates) @@ -73,7 +77,6 @@ func (this *GoMySQLReader) ConnectBinlogStreamer(coordinates mysql.BinlogCoordin Pos: uint32(coords.LogPos)}, ) } - return err } @@ -162,18 +165,25 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha this.currentCoordinatesMutex.Unlock() } + // gomysql.BinlogSyncer keeps track of the streamer's GTID coordinates + // but doesn't expose them, so we have to duplicate the work to track the coords. switch event := ev.Event.(type) { case *replication.GTIDEvent: if !this.migrationContext.UseGTIDs { continue } + if this.LastTrxCoords.IsEmpty() { + continue + } sid, err := uuid.FromBytes(event.SID) if err != nil { return err } this.currentCoordinatesMutex.Lock() + this.currentCoordinates = this.LastTrxCoords.Clone() coords := this.currentCoordinates.(*mysql.GTIDBinlogCoordinates) - coords.GTIDSet.AddSet(gomysql.NewUUIDSet(sid, gomysql.Interval{Start: event.GNO, Stop: event.GNO + 1})) + trxGset := gomysql.NewUUIDSet(sid, gomysql.Interval{Start: event.GNO, Stop: event.GNO + 1}) + coords.GTIDSet.AddSet(trxGset) this.currentCoordinatesMutex.Unlock() case *replication.RotateEvent: if this.migrationContext.UseGTIDs { @@ -185,9 +195,11 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha this.migrationContext.Log.Infof("rotate to next log from %s:%d to %s", coords.LogFile, int64(ev.Header.LogPos), event.NextLogName) this.currentCoordinatesMutex.Unlock() case *replication.XIDEvent: - this.currentCoordinatesMutex.Lock() - this.LastTrxCoords = this.currentCoordinates.Clone() - this.currentCoordinatesMutex.Unlock() + if this.migrationContext.UseGTIDs { + this.LastTrxCoords = &mysql.GTIDBinlogCoordinates{GTIDSet: event.GSet.(*gomysql.MysqlGTIDSet)} + } else { + this.LastTrxCoords = this.currentCoordinates.Clone() + } case *replication.RowsEvent: if err := this.handleRowsEvent(ev, event, entriesChannel); err != nil { return err diff --git a/go/logic/streamer.go b/go/logic/streamer.go index 1ba1593b2..d199825c8 100644 --- a/go/logic/streamer.go +++ b/go/logic/streamer.go @@ -214,7 +214,7 @@ func (this *EventsStreamer) StreamEvents(canStopStreaming func() bool) error { successiveFailures = 0 } if this.migrationContext.BinlogSyncerMaxReconnectAttempts > 0 && successiveFailures >= this.migrationContext.BinlogSyncerMaxReconnectAttempts { - return fmt.Errorf("%d successive failures in streamer reconnect at coordinates %+v", successiveFailures, this.GetReconnectBinlogCoordinates()) + return fmt.Errorf("%d successive failures in streamer reconnect at coordinates %+v", successiveFailures, reconnectCoords) } // Reposition at same coordinates From 38fd3be80d3d85bed5c22b39f8497093c11642d8 Mon Sep 17 00:00:00 2001 From: meiji163 Date: Thu, 9 Oct 2025 10:15:21 -0700 Subject: [PATCH 50/56] fix last trx coords --- go/binlog/gomysql_reader.go | 9 +++------ go/logic/streamer.go | 6 +++--- localtests/test.sh | 2 +- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index d09e8af87..5d23f4114 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -165,22 +165,19 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha this.currentCoordinatesMutex.Unlock() } - // gomysql.BinlogSyncer keeps track of the streamer's GTID coordinates - // but doesn't expose them, so we have to duplicate the work to track the coords. switch event := ev.Event.(type) { case *replication.GTIDEvent: if !this.migrationContext.UseGTIDs { continue } - if this.LastTrxCoords.IsEmpty() { - continue - } sid, err := uuid.FromBytes(event.SID) if err != nil { return err } this.currentCoordinatesMutex.Lock() - this.currentCoordinates = this.LastTrxCoords.Clone() + if this.LastTrxCoords != nil { + this.currentCoordinates = this.LastTrxCoords.Clone() + } coords := this.currentCoordinates.(*mysql.GTIDBinlogCoordinates) trxGset := gomysql.NewUUIDSet(sid, gomysql.Interval{Start: event.GNO, Stop: event.GNO + 1}) coords.GTIDSet.AddSet(trxGset) diff --git a/go/logic/streamer.go b/go/logic/streamer.go index d199825c8..e4de09df7 100644 --- a/go/logic/streamer.go +++ b/go/logic/streamer.go @@ -218,9 +218,9 @@ func (this *EventsStreamer) StreamEvents(canStopStreaming func() bool) error { } // Reposition at same coordinates - reconnectCoords = this.binlogReader.LastTrxCoords.Clone() - if reconnectCoords.IsEmpty() { - // no transactions were handled yet + if this.binlogReader.LastTrxCoords != nil { + reconnectCoords = this.binlogReader.LastTrxCoords.Clone() + } else { reconnectCoords = this.initialBinlogCoordinates.Clone() } this.migrationContext.Log.Infof("Reconnecting EventsStreamer... Will resume at %+v", reconnectCoords) diff --git a/localtests/test.sh b/localtests/test.sh index b96cf9c68..28bb43d13 100755 --- a/localtests/test.sh +++ b/localtests/test.sh @@ -274,7 +274,7 @@ test_single() { fi # - cmd="$ghost_binary \ + cmd="GOTRACEBACK=crash $ghost_binary \ --user=gh-ost \ --password=gh-ost \ --host=$replica_host \ From 5ce8778d3b0c66e2676c3f8643c3e07a43af7ddd Mon Sep 17 00:00:00 2001 From: meiji163 Date: Thu, 9 Oct 2025 14:55:29 -0700 Subject: [PATCH 51/56] re-enable Binlogsyncer retry --- go/binlog/gomysql_reader.go | 15 +++++++-------- go/logic/streamer.go | 16 ++++++++++------ localtests/keyword-column/extra_args | 2 +- script/docker-gh-ost-replica-tests | 2 +- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index 5d23f4114..94b8ca3cb 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -28,9 +28,8 @@ type GoMySQLReader struct { binlogStreamer *replication.BinlogStreamer currentCoordinates mysql.BinlogCoordinates currentCoordinatesMutex *sync.Mutex - // LastTrxCoords tracks the coordinates of the last transaction read. - // It is the GTID set of the transaction, or the coordinates of - // the transaction's XID event if using file coordinates. + // LastTrxCoords are the coordinates of the last transaction completely read. + // If using the file coordinates it is binlog position of the transaction's XID event. LastTrxCoords mysql.BinlogCoordinates } @@ -50,7 +49,7 @@ func NewGoMySQLReader(migrationContext *base.MigrationContext) *GoMySQLReader { TLSConfig: connectionConfig.TLSConfig(), UseDecimal: true, TimestampStringLocation: time.UTC, - DisableRetrySync: true, // we implement our own reconnect. + MaxReconnectAttempts: migrationContext.BinlogSyncerMaxReconnectAttempts, }), } } @@ -90,10 +89,9 @@ func (this *GoMySQLReader) handleRowsEvent(ev *replication.BinlogEvent, rowsEven this.currentCoordinatesMutex.Lock() currentCoords := this.currentCoordinates this.currentCoordinatesMutex.Unlock() - lastTrx := this.LastTrxCoords - if currentCoords.SmallerThan(lastTrx) { - this.migrationContext.Log.Debugf("Skipping handled transaction %+v (last trx is %+v)", currentCoords, lastTrx) + if currentCoords.SmallerThan(this.LastTrxCoords) { + this.migrationContext.Log.Debugf("Skipping handled transaction %+v (last trx is %+v)", currentCoords, this.LastTrxCoords) } dml := ToEventDML(ev.Header.EventType.String()) @@ -151,7 +149,8 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha return err } - // update binlog coords if using file-based coords + // Update binlog coords if using file-based coords. + // GTID coordinates are updated on receiving GTID events. if !this.migrationContext.UseGTIDs { this.currentCoordinatesMutex.Lock() coords := this.currentCoordinates.(*mysql.FileBinlogCoordinates) diff --git a/go/logic/streamer.go b/go/logic/streamer.go index e4de09df7..cfc08964d 100644 --- a/go/logic/streamer.go +++ b/go/logic/streamer.go @@ -191,13 +191,16 @@ func (this *EventsStreamer) StreamEvents(canStopStreaming func() bool) error { } } }() - // The next should block and execute forever, unless there's a serious error + // The next should block and execute forever, unless there's a serious error. var successiveFailures int var reconnectCoords mysql.BinlogCoordinates for { if canStopStreaming() { return nil } + // We will reconnect the binlog streamer at the coordinates + // of the last trx that was read completely from the streamer. + // Since row event application is idempotent, it's OK if we reapply some events. if err := this.binlogReader.StreamEvents(canStopStreaming, this.eventsChannel); err != nil { if canStopStreaming() { return nil @@ -208,11 +211,6 @@ func (this *EventsStreamer) StreamEvents(canStopStreaming func() bool) error { time.Sleep(ReconnectStreamerSleepSeconds * time.Second) // See if there's retry overflow - if this.binlogReader.LastTrxCoords.SmallerThanOrEquals(reconnectCoords) { - successiveFailures += 1 - } else { - successiveFailures = 0 - } if this.migrationContext.BinlogSyncerMaxReconnectAttempts > 0 && successiveFailures >= this.migrationContext.BinlogSyncerMaxReconnectAttempts { return fmt.Errorf("%d successive failures in streamer reconnect at coordinates %+v", successiveFailures, reconnectCoords) } @@ -223,6 +221,12 @@ func (this *EventsStreamer) StreamEvents(canStopStreaming func() bool) error { } else { reconnectCoords = this.initialBinlogCoordinates.Clone() } + if !reconnectCoords.SmallerThan(this.GetCurrentBinlogCoordinates()) { + successiveFailures += 1 + } else { + successiveFailures = 0 + } + this.migrationContext.Log.Infof("Reconnecting EventsStreamer... Will resume at %+v", reconnectCoords) _ = this.binlogReader.Close() if err := this.initBinlogReader(reconnectCoords); err != nil { diff --git a/localtests/keyword-column/extra_args b/localtests/keyword-column/extra_args index 5d73843b0..4b091d601 100644 --- a/localtests/keyword-column/extra_args +++ b/localtests/keyword-column/extra_args @@ -1 +1 @@ ---alter='add column `index` int unsigned' \ +--alter='add column `index` int unsigned' diff --git a/script/docker-gh-ost-replica-tests b/script/docker-gh-ost-replica-tests index 7169574a8..e267d4f89 100755 --- a/script/docker-gh-ost-replica-tests +++ b/script/docker-gh-ost-replica-tests @@ -63,7 +63,7 @@ create_toxiproxy() { -H "Content-Type: application/json" \ -d '{"name": "limit_data_downstream", "type": "limit_data", - "attributes": {"bytes": 40000}}' + "attributes": {"bytes": 1000000}}' echo } From a3684dd083bda03fc5db5b8a06db28abd1cab661 Mon Sep 17 00:00:00 2001 From: meiji163 Date: Thu, 9 Oct 2025 18:56:52 -0700 Subject: [PATCH 52/56] fix localtest arg --- localtests/trivial/extra_args | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/localtests/trivial/extra_args b/localtests/trivial/extra_args index 8b6320aa1..75bbe43a4 100644 --- a/localtests/trivial/extra_args +++ b/localtests/trivial/extra_args @@ -1 +1 @@ ---throttle-query='select false' \ +--throttle-query='select false' From f3ac19cdd03ad8a8a2b9dfc181ecdc96575cc31d Mon Sep 17 00:00:00 2001 From: meiji163 Date: Fri, 10 Oct 2025 09:37:17 -0700 Subject: [PATCH 53/56] remove unneccesary coordinate check --- go/binlog/gomysql_reader.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index 94b8ca3cb..d690a9f65 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -86,14 +86,7 @@ func (this *GoMySQLReader) GetCurrentBinlogCoordinates() mysql.BinlogCoordinates } func (this *GoMySQLReader) handleRowsEvent(ev *replication.BinlogEvent, rowsEvent *replication.RowsEvent, entriesChannel chan<- *BinlogEntry) error { - this.currentCoordinatesMutex.Lock() - currentCoords := this.currentCoordinates - this.currentCoordinatesMutex.Unlock() - - if currentCoords.SmallerThan(this.LastTrxCoords) { - this.migrationContext.Log.Debugf("Skipping handled transaction %+v (last trx is %+v)", currentCoords, this.LastTrxCoords) - } - + currentCoords := this.GetCurrentBinlogCoordinates() dml := ToEventDML(ev.Header.EventType.String()) if dml == NotDML { return fmt.Errorf("Unknown DML type: %s", ev.Header.EventType.String()) From 99626bc3f3c3a0237690049d6a82b9fb8415dd5d Mon Sep 17 00:00:00 2001 From: meiji163 Date: Fri, 10 Oct 2025 09:44:30 -0700 Subject: [PATCH 54/56] rm unused funcs --- go/logic/streamer.go | 11 ---- go/mysql/binlog_gtid.go | 15 ----- script/cibuild-gh-ost-replica-tests | 91 ----------------------------- 3 files changed, 117 deletions(-) delete mode 100755 script/cibuild-gh-ost-replica-tests diff --git a/go/logic/streamer.go b/go/logic/streamer.go index cfc08964d..fd0240ffd 100644 --- a/go/logic/streamer.go +++ b/go/logic/streamer.go @@ -138,17 +138,6 @@ func (this *EventsStreamer) GetCurrentBinlogCoordinates() mysql.BinlogCoordinate return this.binlogReader.GetCurrentBinlogCoordinates() } -func (this *EventsStreamer) GetReconnectBinlogCoordinates() mysql.BinlogCoordinates { - current := this.GetCurrentBinlogCoordinates() - switch coords := current.(type) { - case *mysql.FileBinlogCoordinates: - return &mysql.FileBinlogCoordinates{LogFile: coords.LogFile, LogPos: 4} - case *mysql.GTIDBinlogCoordinates: - return &mysql.GTIDBinlogCoordinates{GTIDSet: coords.GTIDSet} - } - return nil -} - // readCurrentBinlogCoordinates reads master status from hooked server func (this *EventsStreamer) readCurrentBinlogCoordinates() error { binaryLogStatusTerm := mysql.ReplicaTermFor(this.dbVersion, "master status") diff --git a/go/mysql/binlog_gtid.go b/go/mysql/binlog_gtid.go index c02b666c2..d7b86c04f 100644 --- a/go/mysql/binlog_gtid.go +++ b/go/mysql/binlog_gtid.go @@ -6,8 +6,6 @@ package mysql import ( - "errors" - gomysql "github.com/go-mysql-org/go-mysql/mysql" ) @@ -77,19 +75,6 @@ func (this *GTIDBinlogCoordinates) SmallerThanOrEquals(other BinlogCoordinates) return this.Equals(other) || this.SmallerThan(other) } -func (this *GTIDBinlogCoordinates) Update(update interface{}) error { - switch u := update.(type) { - case *gomysql.UUIDSet: - this.GTIDSet.AddSet(u) - this.UUIDSet = u - case *gomysql.MysqlGTIDSet: - this.GTIDSet = u - default: - return errors.New("unsupported update") - } - return nil -} - func (this *GTIDBinlogCoordinates) Clone() BinlogCoordinates { out := >IDBinlogCoordinates{} if this.GTIDSet != nil { diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests deleted file mode 100755 index c4dbfd292..000000000 --- a/script/cibuild-gh-ost-replica-tests +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/bash - -set -e - -whoami - -fetch_ci_env() { - # Clone gh-ost-ci-env - # Only clone if not already running locally at latest commit - remote_commit=$(git ls-remote https://github.com/github/gh-ost-ci-env.git HEAD | cut -f1) - local_commit="unknown" - [ -d "gh-ost-ci-env" ] && local_commit=$(cd gh-ost-ci-env && git log --format="%H" -n 1) - - echo "remote commit is: $remote_commit" - echo "local commit is: $local_commit" - - if [ "$remote_commit" != "$local_commit" ] ; then - rm -rf ./gh-ost-ci-env - git clone https://github.com/github/gh-ost-ci-env.git - fi -} - -test_dbdeployer() { - gh-ost-ci-env/bin/linux/dbdeployer --version -} - -test_mysql_version() { - local mysql_version - mysql_version="$1" - - echo "##### Testing $mysql_version" - - echo "### Setting up sandbox for $mysql_version" - - find sandboxes -name "stop_all" | bash - - mkdir -p sandbox/binary - rm -rf sandbox/binary/* - gh-ost-ci-env/bin/linux/dbdeployer unpack gh-ost-ci-env/mysql-tarballs/"$mysql_version".tar.xz --sandbox-binary ${PWD}/sandbox/binary - - mkdir -p sandboxes - rm -rf sandboxes/* - - local mysql_version_num=${mysql_version#*-} - if echo "$mysql_version_num" | egrep "5[.]5[.]" ; then - gtid="" - else - gtid="--gtid" - fi - gh-ost-ci-env/bin/linux/dbdeployer deploy replication "$mysql_version_num" --nodes 2 --sandbox-binary ${PWD}/sandbox/binary --sandbox-home ${PWD}/sandboxes ${gtid} --my-cnf-options log_slave_updates --my-cnf-options log_bin --my-cnf-options binlog_format=ROW --sandbox-directory rsandbox - - sed '/sandboxes/d' -i gh-ost-ci-env/bin/gh-ost-test-mysql-master - echo 'sandboxes/rsandbox/m "$@"' >> gh-ost-ci-env/bin/gh-ost-test-mysql-master - - sed '/sandboxes/d' -i gh-ost-ci-env/bin/gh-ost-test-mysql-replica - echo 'sandboxes/rsandbox/s1 "$@"' >> gh-ost-ci-env/bin/gh-ost-test-mysql-replica - - export PATH="${PWD}/gh-ost-ci-env/bin/:${PATH}" - - gh-ost-test-mysql-master -uroot -e "create user 'gh-ost'@'%' identified by 'gh-ost'" - gh-ost-test-mysql-master -uroot -e "grant all on *.* to 'gh-ost'@'%'" - - echo "### Running gh-ost tests for $mysql_version" - ./localtests/test.sh -b bin/gh-ost - - find sandboxes -name "stop_all" | bash -} - -main() { - fetch_ci_env - test_dbdeployer - - echo "Building..." - . script/build - - # TEST_MYSQL_VERSION is set by the replica-tests CI job - if [ -z "$TEST_MYSQL_VERSION" ]; then - # Test all versions: - find gh-ost-ci-env/mysql-tarballs/ -name "*.tar.xz" | while read f ; do basename $f ".tar.xz" ; done | sort -r | while read mysql_version ; do - echo "found MySQL version: $mysql_version" - done - find gh-ost-ci-env/mysql-tarballs/ -name "*.tar.xz" | while read f ; do basename $f ".tar.xz" ; done | sort -r | while read mysql_version ; do - test_mysql_version "$mysql_version" - done - else - echo "found MySQL version: $TEST_MYSQL_VERSION" - test_mysql_version "$TEST_MYSQL_VERSION" - fi -} - -main From ed0eba99db79931e0aa427d6c1126876f405f13f Mon Sep 17 00:00:00 2001 From: meiji163 Date: Fri, 10 Oct 2025 10:32:53 -0700 Subject: [PATCH 55/56] add back ReplicaTermFor --- go/logic/inspect.go | 2 +- go/mysql/utils.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index f41f4032f..7a7dc8424 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -384,7 +384,7 @@ func (this *Inspector) applyBinlogFormat() error { // validateBinlogs checks that binary log configuration is good to go func (this *Inspector) validateBinlogs() error { - query := `select @@global.log_bin, @@global.binlog_format` + query := `select /* gh-ost */@@global.log_bin, @@global.binlog_format` var hasBinaryLogs bool if err := this.db.QueryRow(query).Scan(&hasBinaryLogs, &this.migrationContext.OriginalBinlogFormat); err != nil { return err diff --git a/go/mysql/utils.go b/go/mysql/utils.go index 54522df12..7d57afbf1 100644 --- a/go/mysql/utils.go +++ b/go/mysql/utils.go @@ -173,12 +173,12 @@ func GetReplicationBinlogCoordinates(dbVersion string, db *gosql.DB, gtid bool) } } else { readBinlogCoordinates = NewFileBinlogCoordinates( - m.GetString("Master_Log_File"), - m.GetInt64("Read_Master_Log_Pos"), + m.GetString(ReplicaTermFor(dbVersion, "Master_Log_File")), + m.GetInt64(ReplicaTermFor(dbVersion, "Read_Master_Log_Pos")), ) executeBinlogCoordinates = NewFileBinlogCoordinates( - m.GetString("Relay_Master_Log_File"), - m.GetInt64("Exec_Master_Log_Pos"), + m.GetString(ReplicaTermFor(dbVersion, "Relay_Master_Log_File")), + m.GetInt64(ReplicaTermFor(dbVersion, "Exec_Master_Log_Pos")), ) } return nil From 1cd609506fd949ff7cd5473dced85d3fc0f6ece0 Mon Sep 17 00:00:00 2001 From: meiji163 Date: Fri, 10 Oct 2025 10:36:26 -0700 Subject: [PATCH 56/56] Update localtests/test.sh Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- localtests/test.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/localtests/test.sh b/localtests/test.sh index f2e649360..7fe9c2ab2 100755 --- a/localtests/test.sh +++ b/localtests/test.sh @@ -46,9 +46,6 @@ while getopts "b:s:dtg" OPTION; do d) docker=true ;; - t) - toxiproxy=true - ;; g) gtid=true ;;