fix: auto-recover from CHANGE MASTER errno 1201 via RESET SLAVE ALL retry

pkg/controller/replication/config.go

@@ -211,9 +211,31 @@ func (r *ReplicationConfigClient) changeMaster(ctx context.Context, mariadb *mar
 
 	changeMasterOpts = append(changeMasterOpts, opts...)
 
-	if err := client.ChangeMaster(ctx, changeMasterOpts...); err != nil {
+	err = client.ChangeMaster(ctx, changeMasterOpts...)
+	if err == nil {
+		return nil
+	}
+	// CHANGE MASTER TO can return Error 1201 ("Could not initialize master
+	// info structure for ''") when a previous switchover left residual
+	// slave-channel state on this pod (truncated master.info / relay-log.info
+	// from a partially-completed phase 4-6 sequence). The condition is
+	// recoverable by clearing slave state via RESET SLAVE ALL and retrying
+	// once: the residual files are exactly the state we are about to
+	// overwrite anyway. Field incident: moodle-education-stg-db wedged for
+	// hours after a previous failed switchover left pod-1 with a 0-byte
+	// master.info; the operator looped on this error every reconcile until
+	// the on-disk state was cleared manually.
+	if !sql.IsMySQLErrorCode(err, sql.ErrCodeMasterInfo) {
 		return fmt.Errorf("error changing master: %v", err)
 	}
+	if resetErr := client.ResetAllSlaves(ctx); resetErr != nil {
+		return fmt.Errorf("error changing master (errno %d) and follow-up RESET SLAVE ALL also failed: changeMaster=%v resetSlaveAll=%v",
+			sql.ErrCodeMasterInfo, err, resetErr)
+	}
+	if err := client.ChangeMaster(ctx, changeMasterOpts...); err != nil {
+		return fmt.Errorf("error changing master after RESET SLAVE ALL recovery from errno %d: %v",
+			sql.ErrCodeMasterInfo, err)
+	}
 	return nil
 }
 

pkg/controller/replication/switchover_test.go

@@ -498,3 +498,39 @@ func TestReplicationReconcile_RefreshesRolesBeforeSwitchover(_ *testing.T) {
 	// Integration-test only; documented here for design record.
 	// See pkg/controller/replication/controller.go: Replication.Reconcile dispatch order.
 }
+
+// TestChangeMaster_RecoversFromMasterInfoErrno1201 documents the bug:
+//
+//	When a previous switchover completes phase 4 (configureNewPrimary runs
+//	RESET MASTER / RESET SLAVE ALL on the new primary) but then fails partway
+//	through phases 5 or 6, the *old* primary can be left with truncated
+//	master.info / relay-log.info files in its data dir. On the next reconcile
+//	the operator hits changePrimaryToReplica again and runs ConfigureReplica
+//	on the old primary, which eventually issues CHANGE MASTER TO with the
+//	default (empty) connection name. MariaDB rejects this with:
+//
+//	    Error 1201 (HY000): Could not initialize master info structure for '';
+//	    more error messages can be found in the MariaDB error log
+//
+//	because the in-memory master_info structure cannot be rebuilt from the
+//	residual on-disk state. The operator's switchover state machine then
+//	loops on this error every reconcile, indefinitely. Field incident:
+//	moodle-education-stg-db, 2026-05-14. Manual recovery required executing
+//	`STOP SLAVE; RESET SLAVE ALL;` on the affected pod between operator
+//	reconciles to clear the slave-channel state.
+//
+//	Fix: when changeMaster returns Error 1201, the operator now calls
+//	RESET SLAVE ALL on the same client and retries CHANGE MASTER TO once.
+//	The retry is safe because the residual master.info / relay-log.info
+//	state is exactly what the in-flight ConfigureReplica is about to
+//	overwrite anyway; clearing it removes the corruption barrier.
+//
+//	The code path lives in pkg/controller/replication/config.go:changeMaster.
+//	Because exercising it requires a live SQL connection that emits a
+//	specific server-side error code, it is covered by integration tests.
+//	The pure error-classifier (sql.IsMySQLErrorCode) is unit-tested in
+//	pkg/sql/sql_test.go: TestIsMySQLErrorCode.
+func TestChangeMaster_RecoversFromMasterInfoErrno1201(_ *testing.T) {
+	// Integration-test only; documented here for design record.
+	// See pkg/controller/replication/config.go: changeMaster (errno 1201 branch).
+}

pkg/sql/sql.go

@@ -30,6 +30,34 @@ var (
 	ErrWaitReplicaTimeout = errors.New("timeout waiting for replica to be synced")
 )
 
+// MariaDB server-side error codes the operator inspects to drive recovery
+// decisions. Source: MariaDB server error reference. Defined here (rather than
+// imported) because the go-sql-driver/mysql package does not export them.
+const (
+	// ErrCodeMasterInfo is `ER_MASTER_INFO` (1201) — "Could not initialize
+	// master info structure for '<connection_name>'". Returned by CHANGE
+	// MASTER TO when the in-memory master_info structure for the named
+	// replication channel cannot be (re)bound, typically because residual
+	// on-disk state (`master.info`, `relay-log.info`) is truncated or
+	// inconsistent from a previous half-completed switchover. Recovery is
+	// `RESET SLAVE ALL` on the affected connection followed by retry.
+	ErrCodeMasterInfo uint16 = 1201
+)
+
+// IsMySQLErrorCode reports whether err is a *mysql.MySQLError with the given
+// server-side error code. Returns false for nil error, non-driver errors, or
+// driver errors with a different code.
+func IsMySQLErrorCode(err error, code uint16) bool {
+	if err == nil {
+		return false
+	}
+	var mErr *mysql.MySQLError
+	if !errors.As(err, &mErr) {
+		return false
+	}
+	return mErr.Number == code
+}
+
 type Opts struct {
 	Username string
 	Password string

pkg/sql/sql_test.go

@@ -1,8 +1,11 @@
 package sql
 
 import (
+	"errors"
+	"fmt"
 	"testing"
 
+	"github.com/go-sql-driver/mysql"
 	"github.com/google/go-cmp/cmp"
 	mariadbv1alpha1 "github.com/mariadb-operator/mariadb-operator/v26/api/v1alpha1"
 	"k8s.io/utils/ptr"
@@ -215,3 +218,72 @@ func TestRequireQuery(t *testing.T) {
 		})
 	}
 }
+
+func TestIsMySQLErrorCode(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name string
+		err  error
+		code uint16
+		want bool
+	}{
+		{
+			name: "nil error returns false",
+			err:  nil,
+			code: ErrCodeMasterInfo,
+			want: false,
+		},
+		{
+			name: "non-driver error returns false",
+			err:  errors.New("connection refused"),
+			code: ErrCodeMasterInfo,
+			want: false,
+		},
+		{
+			name: "wrapped non-driver error returns false",
+			err:  fmt.Errorf("outer: %w", errors.New("inner")),
+			code: ErrCodeMasterInfo,
+			want: false,
+		},
+		{
+			name: "driver error with matching code returns true",
+			err:  &mysql.MySQLError{Number: 1201, Message: "Could not initialize master info structure for ''"},
+			code: ErrCodeMasterInfo,
+			want: true,
+		},
+		{
+			name: "driver error with non-matching code returns false",
+			err:  &mysql.MySQLError{Number: 1062, Message: "Duplicate entry"},
+			code: ErrCodeMasterInfo,
+			want: false,
+		},
+		{
+			name: "driver error wrapped via fmt.Errorf %w with matching code returns true",
+			err:  fmt.Errorf("outer wrapper: %w", &mysql.MySQLError{Number: 1201, Message: "..."}),
+			code: ErrCodeMasterInfo,
+			want: true,
+		},
+		{
+			name: "driver error wrapped multiple levels via fmt.Errorf %w returns true",
+			err: fmt.Errorf("level 3: %w",
+				fmt.Errorf("level 2: %w",
+					&mysql.MySQLError{Number: 1201, Message: "..."})),
+			code: ErrCodeMasterInfo,
+			want: true,
+		},
+		{
+			name: "driver error wrapped via fmt.Errorf %v (loses chain) returns false",
+			err:  fmt.Errorf("outer wrapper: %v", &mysql.MySQLError{Number: 1201, Message: "..."}),
+			code: ErrCodeMasterInfo,
+			want: false,
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			got := IsMySQLErrorCode(tc.err, tc.code)
+			if got != tc.want {
+				t.Errorf("IsMySQLErrorCode(%v, %d) = %v, want %v", tc.err, tc.code, got, tc.want)
+			}
+		})
+	}
+}