Merge branch 'master' into adjust-wait-post-repl-restart

.gitignore

@@ -3,3 +3,4 @@
 /libexec/
 /.vendor/
 .idea/
+*.tmp

README.md

@@ -97,7 +97,7 @@ Please see [Coding gh-ost](doc/coding-ghost.md) for a guide to getting started d
 
 `gh-ost` is a Go project; it is built with Go `1.15` and above. To build on your own, use either:
 - [script/build](https://github.com/github/gh-ost/blob/master/script/build) - this is the same build script used by CI hence the authoritative; artifact is `./bin/gh-ost` binary.
-- [build.sh](https://github.com/github/gh-ost/blob/master/build.sh) for building `tar.gz` artifacts in `/tmp/gh-ost`
+- [build.sh](https://github.com/github/gh-ost/blob/master/build.sh) for building `tar.gz` artifacts in `/tmp/gh-ost-release`
 
 Generally speaking, `master` branch is stable, but only [releases](https://github.com/github/gh-ost/releases) are to be used in production.
 

build.sh

@@ -1,6 +1,4 @@
 #!/bin/bash
-#
-#
 
 RELEASE_VERSION=
 buildpath=
@@ -72,11 +70,14 @@ main() {
   build macOS osx darwin amd64
   build macOS osx darwin arm64
 
+  bin_files=$(find $buildpath/gh-ost* -type f -maxdepth 1)
   echo "Binaries found in:"
-  find $buildpath/gh-ost* -type f -maxdepth 1
+  echo "$bin_files"
 
   echo "Checksums:"
-  (cd $buildpath && shasum -a256 gh-ost* 2>/dev/null)
+  (shasum -a256 $bin_files 2>/dev/null)
+
+  echo "Build Success!"
 }
 
 main "$@"

go.mod

@@ -8,17 +8,65 @@ require (
 	github.com/go-sql-driver/mysql v1.8.1
 	github.com/google/uuid v1.6.0
 	github.com/openark/golib v0.0.0-20210531070646-355f37940af8
+	github.com/stretchr/testify v1.9.0
+	github.com/testcontainers/testcontainers-go v0.34.0
 	golang.org/x/net v0.24.0
-	golang.org/x/term v0.19.0
-	golang.org/x/text v0.14.0
+	golang.org/x/sync v0.10.0
+	golang.org/x/term v0.27.0
+	golang.org/x/text v0.21.0
 )
 
 require (
+	dario.cat/mergo v1.0.0 // indirect
 	filippo.io/edwards25519 v1.1.0 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
+	github.com/containerd/containerd v1.7.18 // indirect
+	github.com/containerd/log v0.1.0 // indirect
+	github.com/containerd/platforms v0.2.1 // indirect
+	github.com/cpuguy83/dockercfg v0.3.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/distribution/reference v0.6.0 // indirect
+	github.com/docker/docker v27.1.1+incompatible // indirect
+	github.com/docker/go-connections v0.5.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/klauspost/compress v1.17.4 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/moby/docker-image-spec v1.3.1 // indirect
+	github.com/moby/patternmatcher v0.6.0 // indirect
+	github.com/moby/sys/sequential v0.5.0 // indirect
+	github.com/moby/sys/user v0.1.0 // indirect
+	github.com/moby/term v0.5.0 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.0 // indirect
 	github.com/pingcap/errors v0.11.5-0.20210425183316-da1aaba5fb63 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
+	github.com/shirou/gopsutil/v3 v3.23.12 // indirect
+	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 // indirect
 	github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 // indirect
 	github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/tklauser/go-sysconf v0.3.12 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
+	github.com/yusufpapurcu/wmi v1.2.3 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
+	go.opentelemetry.io/otel v1.24.0 // indirect
+	go.opentelemetry.io/otel/metric v1.24.0 // indirect
+	go.opentelemetry.io/otel/trace v1.24.0 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
-	golang.org/x/sys v0.19.0 // indirect
+	golang.org/x/crypto v0.31.0 // indirect
+	golang.org/x/sys v0.28.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go/base/context.go

@@ -103,6 +103,11 @@ type MigrationContext struct {
 	AzureMySQL               bool
 	AttemptInstantDDL        bool
 
+	// SkipPortValidation allows skipping the port validation in `ValidateConnection`
+	// This is useful when connecting to a MySQL instance where the external port
+	// may not match the internal port.
+	SkipPortValidation bool
+
 	config            ContextConfig
 	configMutex       *sync.Mutex
 	ConfigFile        string

go/base/context_test.go

@@ -11,7 +11,7 @@ import (
 	"time"
 
 	"github.com/openark/golib/log"
-	test "github.com/openark/golib/tests"
+	"github.com/stretchr/testify/require"
 )
 
 func init() {
@@ -22,22 +22,22 @@ func TestGetTableNames(t *testing.T) {
 	{
 		context := NewMigrationContext()
 		context.OriginalTableName = "some_table"
-		test.S(t).ExpectEquals(context.GetOldTableName(), "_some_table_del")
-		test.S(t).ExpectEquals(context.GetGhostTableName(), "_some_table_gho")
-		test.S(t).ExpectEquals(context.GetChangelogTableName(), "_some_table_ghc")
+		require.Equal(t, "_some_table_del", context.GetOldTableName())
+		require.Equal(t, "_some_table_gho", context.GetGhostTableName())
+		require.Equal(t, "_some_table_ghc", context.GetChangelogTableName(), "_some_table_ghc")
 	}
 	{
 		context := NewMigrationContext()
 		context.OriginalTableName = "a123456789012345678901234567890123456789012345678901234567890"
-		test.S(t).ExpectEquals(context.GetOldTableName(), "_a1234567890123456789012345678901234567890123456789012345678_del")
-		test.S(t).ExpectEquals(context.GetGhostTableName(), "_a1234567890123456789012345678901234567890123456789012345678_gho")
-		test.S(t).ExpectEquals(context.GetChangelogTableName(), "_a1234567890123456789012345678901234567890123456789012345678_ghc")
+		require.Equal(t, "_a1234567890123456789012345678901234567890123456789012345678_del", context.GetOldTableName())
+		require.Equal(t, "_a1234567890123456789012345678901234567890123456789012345678_gho", context.GetGhostTableName())
+		require.Equal(t, "_a1234567890123456789012345678901234567890123456789012345678_ghc", context.GetChangelogTableName())
 	}
 	{
 		context := NewMigrationContext()
 		context.OriginalTableName = "a123456789012345678901234567890123456789012345678901234567890123"
 		oldTableName := context.GetOldTableName()
-		test.S(t).ExpectEquals(oldTableName, "_a1234567890123456789012345678901234567890123456789012345678_del")
+		require.Equal(t, "_a1234567890123456789012345678901234567890123456789012345678_del", oldTableName)
 	}
 	{
 		context := NewMigrationContext()
@@ -46,15 +46,15 @@ func TestGetTableNames(t *testing.T) {
 		longForm := "Jan 2, 2006 at 3:04pm (MST)"
 		context.StartTime, _ = time.Parse(longForm, "Feb 3, 2013 at 7:54pm (PST)")
 		oldTableName := context.GetOldTableName()
-		test.S(t).ExpectEquals(oldTableName, "_a1234567890123456789012345678901234567890123_20130203195400_del")
+		require.Equal(t, "_a1234567890123456789012345678901234567890123_20130203195400_del", oldTableName)
 	}
 	{
 		context := NewMigrationContext()
 		context.OriginalTableName = "foo_bar_baz"
 		context.ForceTmpTableName = "tmp"
-		test.S(t).ExpectEquals(context.GetOldTableName(), "_tmp_del")
-		test.S(t).ExpectEquals(context.GetGhostTableName(), "_tmp_gho")
-		test.S(t).ExpectEquals(context.GetChangelogTableName(), "_tmp_ghc")
+		require.Equal(t, "_tmp_del", context.GetOldTableName())
+		require.Equal(t, "_tmp_gho", context.GetGhostTableName())
+		require.Equal(t, "_tmp_ghc", context.GetChangelogTableName())
 	}
 }
 

go/base/load_map_test.go

@@ -9,7 +9,7 @@ import (
 	"testing"
 
 	"github.com/openark/golib/log"
-	test "github.com/openark/golib/tests"
+	"github.com/stretchr/testify/require"
 )
 
 func init() {
@@ -20,39 +20,39 @@ func TestParseLoadMap(t *testing.T) {
 	{
 		loadList := ""
 		m, err := ParseLoadMap(loadList)
-		test.S(t).ExpectNil(err)
-		test.S(t).ExpectEquals(len(m), 0)
+		require.NoError(t, err)
+		require.Len(t, m, 0)
 	}
 	{
 		loadList := "threads_running=20,threads_connected=10"
 		m, err := ParseLoadMap(loadList)
-		test.S(t).ExpectNil(err)
-		test.S(t).ExpectEquals(len(m), 2)
-		test.S(t).ExpectEquals(m["threads_running"], int64(20))
-		test.S(t).ExpectEquals(m["threads_connected"], int64(10))
+		require.NoError(t, err)
+		require.Len(t, m, 2)
+		require.Equal(t, int64(20), m["threads_running"])
+		require.Equal(t, int64(10), m["threads_connected"])
 	}
 	{
 		loadList := "threads_running=20=30,threads_connected=10"
 		_, err := ParseLoadMap(loadList)
-		test.S(t).ExpectNotNil(err)
+		require.Error(t, err)
 	}
 	{
 		loadList := "threads_running=20,threads_connected"
 		_, err := ParseLoadMap(loadList)
-		test.S(t).ExpectNotNil(err)
+		require.Error(t, err)
 	}
 }
 
 func TestString(t *testing.T) {
 	{
 		m, _ := ParseLoadMap("")
 		s := m.String()
-		test.S(t).ExpectEquals(s, "")
+		require.Equal(t, "", s)
 	}
 	{
 		loadList := "threads_running=20,threads_connected=10"
 		m, _ := ParseLoadMap(loadList)
 		s := m.String()
-		test.S(t).ExpectEquals(s, "threads_connected=10,threads_running=20")
+		require.Equal(t, "threads_connected=10,threads_running=20", s)
 	}
 }

go/base/utils.go

@@ -63,18 +63,27 @@ func StringContainsAll(s string, substrings ...string) bool {
 
 func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig, migrationContext *MigrationContext, name string) (string, error) {
 	versionQuery := `select @@global.version`
-	var port, extraPort int
+
 	var version string
 	if err := db.QueryRow(versionQuery).Scan(&version); err != nil {
 		return "", err
 	}
+
+	if migrationContext.SkipPortValidation {
+		return version, nil
+	}
+
+	var extraPort int
+
 	extraPortQuery := `select @@global.extra_port`
 	if err := db.QueryRow(extraPortQuery).Scan(&extraPort); err != nil { //nolint:staticcheck
 		// swallow this error. not all servers support extra_port
 	}
+
 	// AliyunRDS set users port to "NULL", replace it by gh-ost param
 	// GCP set users port to "NULL", replace it by gh-ost param
 	// Azure MySQL set users port to a different value by design, replace it by gh-ost para
+	var port int
 	if migrationContext.AliyunRDS || migrationContext.GoogleCloudPlatform || migrationContext.AzureMySQL {
 		port = connectionConfig.Key.Port
 	} else {

go/base/utils_test.go

@@ -9,7 +9,7 @@ import (
 	"testing"
 
 	"github.com/openark/golib/log"
-	test "github.com/openark/golib/tests"
+	"github.com/stretchr/testify/require"
 )
 
 func init() {
@@ -19,11 +19,11 @@ func init() {
 func TestStringContainsAll(t *testing.T) {
 	s := `insert,delete,update`
 
-	test.S(t).ExpectFalse(StringContainsAll(s))
-	test.S(t).ExpectFalse(StringContainsAll(s, ""))
-	test.S(t).ExpectFalse(StringContainsAll(s, "drop"))
-	test.S(t).ExpectTrue(StringContainsAll(s, "insert"))
-	test.S(t).ExpectFalse(StringContainsAll(s, "insert", "drop"))
-	test.S(t).ExpectTrue(StringContainsAll(s, "insert", ""))
-	test.S(t).ExpectTrue(StringContainsAll(s, "insert", "update", "delete"))
+	require.False(t, StringContainsAll(s))
+	require.False(t, StringContainsAll(s, ""))
+	require.False(t, StringContainsAll(s, "drop"))
+	require.True(t, StringContainsAll(s, "insert"))
+	require.False(t, StringContainsAll(s, "insert", "drop"))
+	require.True(t, StringContainsAll(s, "insert", ""))
+	require.True(t, StringContainsAll(s, "insert", "update", "delete"))
 }

go/cmd/gh-ost/main.go

@@ -107,7 +107,7 @@ func main() {
 	chunkSize := flag.Int64("chunk-size", 1000, "amount of rows to handle in each iteration (allowed range: 10-100,000)")
 	dmlBatchSize := flag.Int64("dml-batch-size", 10, "batch size for DML events to apply in a single transaction (range 1-100)")
 	defaultRetries := flag.Int64("default-retries", 60, "Default number of retries for various operations before panicking")
-	cutOverLockTimeoutSeconds := flag.Int64("cut-over-lock-timeout-seconds", 3, "Max number of seconds to hold locks on tables while attempting to cut-over (retry attempted when lock exceeds timeout)")
+	cutOverLockTimeoutSeconds := flag.Int64("cut-over-lock-timeout-seconds", 3, "Max number of seconds to hold locks on tables while attempting to cut-over (retry attempted when lock exceeds timeout) or attempting instant DDL")
 	niceRatio := flag.Float64("nice-ratio", 0, "force being 'nice', imply sleep time per chunk time; range: [0.0..100.0]. Example values: 0 is aggressive. 1: for every 1ms spent copying rows, sleep additional 1ms (effectively doubling runtime); 0.7: for every 10ms spend in a rowcopy chunk, spend 7ms sleeping immediately after")
 
 	maxLagMillis := flag.Int64("max-lag-millis", 1500, "replication lag at which to throttle operation")