diff --git a/pkg/controller/replication/config.go b/pkg/controller/replication/config.go index cb5253d7e1..5357cfb867 100644 --- a/pkg/controller/replication/config.go +++ b/pkg/controller/replication/config.go @@ -211,9 +211,31 @@ func (r *ReplicationConfigClient) changeMaster(ctx context.Context, mariadb *mar changeMasterOpts = append(changeMasterOpts, opts...) - if err := client.ChangeMaster(ctx, changeMasterOpts...); err != nil { + err = client.ChangeMaster(ctx, changeMasterOpts...) + if err == nil { + return nil + } + // CHANGE MASTER TO can return Error 1201 ("Could not initialize master + // info structure for ''") when a previous switchover left residual + // slave-channel state on this pod (truncated master.info / relay-log.info + // from a partially-completed phase 4-6 sequence). The condition is + // recoverable by clearing slave state via RESET SLAVE ALL and retrying + // once: the residual files are exactly the state we are about to + // overwrite anyway. Field incident: moodle-education-stg-db wedged for + // hours after a previous failed switchover left pod-1 with a 0-byte + // master.info; the operator looped on this error every reconcile until + // the on-disk state was cleared manually. + if !sql.IsMySQLErrorCode(err, sql.ErrCodeMasterInfo) { return fmt.Errorf("error changing master: %v", err) } + if resetErr := client.ResetAllSlaves(ctx); resetErr != nil { + return fmt.Errorf("error changing master (errno %d) and follow-up RESET SLAVE ALL also failed: changeMaster=%v resetSlaveAll=%v", + sql.ErrCodeMasterInfo, err, resetErr) + } + if err := client.ChangeMaster(ctx, changeMasterOpts...); err != nil { + return fmt.Errorf("error changing master after RESET SLAVE ALL recovery from errno %d: %v", + sql.ErrCodeMasterInfo, err) + } return nil } diff --git a/pkg/controller/replication/switchover_test.go b/pkg/controller/replication/switchover_test.go index 604f3abb87..c5497190a3 100644 --- a/pkg/controller/replication/switchover_test.go +++ b/pkg/controller/replication/switchover_test.go @@ -498,3 +498,39 @@ func TestReplicationReconcile_RefreshesRolesBeforeSwitchover(_ *testing.T) { // Integration-test only; documented here for design record. // See pkg/controller/replication/controller.go: Replication.Reconcile dispatch order. } + +// TestChangeMaster_RecoversFromMasterInfoErrno1201 documents the bug: +// +// When a previous switchover completes phase 4 (configureNewPrimary runs +// RESET MASTER / RESET SLAVE ALL on the new primary) but then fails partway +// through phases 5 or 6, the *old* primary can be left with truncated +// master.info / relay-log.info files in its data dir. On the next reconcile +// the operator hits changePrimaryToReplica again and runs ConfigureReplica +// on the old primary, which eventually issues CHANGE MASTER TO with the +// default (empty) connection name. MariaDB rejects this with: +// +// Error 1201 (HY000): Could not initialize master info structure for ''; +// more error messages can be found in the MariaDB error log +// +// because the in-memory master_info structure cannot be rebuilt from the +// residual on-disk state. The operator's switchover state machine then +// loops on this error every reconcile, indefinitely. Field incident: +// moodle-education-stg-db, 2026-05-14. Manual recovery required executing +// `STOP SLAVE; RESET SLAVE ALL;` on the affected pod between operator +// reconciles to clear the slave-channel state. +// +// Fix: when changeMaster returns Error 1201, the operator now calls +// RESET SLAVE ALL on the same client and retries CHANGE MASTER TO once. +// The retry is safe because the residual master.info / relay-log.info +// state is exactly what the in-flight ConfigureReplica is about to +// overwrite anyway; clearing it removes the corruption barrier. +// +// The code path lives in pkg/controller/replication/config.go:changeMaster. +// Because exercising it requires a live SQL connection that emits a +// specific server-side error code, it is covered by integration tests. +// The pure error-classifier (sql.IsMySQLErrorCode) is unit-tested in +// pkg/sql/sql_test.go: TestIsMySQLErrorCode. +func TestChangeMaster_RecoversFromMasterInfoErrno1201(_ *testing.T) { + // Integration-test only; documented here for design record. + // See pkg/controller/replication/config.go: changeMaster (errno 1201 branch). +} diff --git a/pkg/sql/sql.go b/pkg/sql/sql.go index 37b33c7c22..faeb3285c1 100644 --- a/pkg/sql/sql.go +++ b/pkg/sql/sql.go @@ -30,6 +30,34 @@ var ( ErrWaitReplicaTimeout = errors.New("timeout waiting for replica to be synced") ) +// MariaDB server-side error codes the operator inspects to drive recovery +// decisions. Source: MariaDB server error reference. Defined here (rather than +// imported) because the go-sql-driver/mysql package does not export them. +const ( + // ErrCodeMasterInfo is `ER_MASTER_INFO` (1201) — "Could not initialize + // master info structure for ''". Returned by CHANGE + // MASTER TO when the in-memory master_info structure for the named + // replication channel cannot be (re)bound, typically because residual + // on-disk state (`master.info`, `relay-log.info`) is truncated or + // inconsistent from a previous half-completed switchover. Recovery is + // `RESET SLAVE ALL` on the affected connection followed by retry. + ErrCodeMasterInfo uint16 = 1201 +) + +// IsMySQLErrorCode reports whether err is a *mysql.MySQLError with the given +// server-side error code. Returns false for nil error, non-driver errors, or +// driver errors with a different code. +func IsMySQLErrorCode(err error, code uint16) bool { + if err == nil { + return false + } + var mErr *mysql.MySQLError + if !errors.As(err, &mErr) { + return false + } + return mErr.Number == code +} + type Opts struct { Username string Password string diff --git a/pkg/sql/sql_test.go b/pkg/sql/sql_test.go index 8d73d746ba..6b872bb9da 100644 --- a/pkg/sql/sql_test.go +++ b/pkg/sql/sql_test.go @@ -1,8 +1,11 @@ package sql import ( + "errors" + "fmt" "testing" + "github.com/go-sql-driver/mysql" "github.com/google/go-cmp/cmp" mariadbv1alpha1 "github.com/mariadb-operator/mariadb-operator/v26/api/v1alpha1" "k8s.io/utils/ptr" @@ -215,3 +218,72 @@ func TestRequireQuery(t *testing.T) { }) } } + +func TestIsMySQLErrorCode(t *testing.T) { + t.Parallel() + tests := []struct { + name string + err error + code uint16 + want bool + }{ + { + name: "nil error returns false", + err: nil, + code: ErrCodeMasterInfo, + want: false, + }, + { + name: "non-driver error returns false", + err: errors.New("connection refused"), + code: ErrCodeMasterInfo, + want: false, + }, + { + name: "wrapped non-driver error returns false", + err: fmt.Errorf("outer: %w", errors.New("inner")), + code: ErrCodeMasterInfo, + want: false, + }, + { + name: "driver error with matching code returns true", + err: &mysql.MySQLError{Number: 1201, Message: "Could not initialize master info structure for ''"}, + code: ErrCodeMasterInfo, + want: true, + }, + { + name: "driver error with non-matching code returns false", + err: &mysql.MySQLError{Number: 1062, Message: "Duplicate entry"}, + code: ErrCodeMasterInfo, + want: false, + }, + { + name: "driver error wrapped via fmt.Errorf %w with matching code returns true", + err: fmt.Errorf("outer wrapper: %w", &mysql.MySQLError{Number: 1201, Message: "..."}), + code: ErrCodeMasterInfo, + want: true, + }, + { + name: "driver error wrapped multiple levels via fmt.Errorf %w returns true", + err: fmt.Errorf("level 3: %w", + fmt.Errorf("level 2: %w", + &mysql.MySQLError{Number: 1201, Message: "..."})), + code: ErrCodeMasterInfo, + want: true, + }, + { + name: "driver error wrapped via fmt.Errorf %v (loses chain) returns false", + err: fmt.Errorf("outer wrapper: %v", &mysql.MySQLError{Number: 1201, Message: "..."}), + code: ErrCodeMasterInfo, + want: false, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := IsMySQLErrorCode(tc.err, tc.code) + if got != tc.want { + t.Errorf("IsMySQLErrorCode(%v, %d) = %v, want %v", tc.err, tc.code, got, tc.want) + } + }) + } +}