diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f2a183c5..e2b02ac0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -93,6 +93,11 @@ jobs: suite: api-tests args: --variable XTRACE:True + - id: rgw-replication-status-test + name: RGW replication status API + suite: rgw-replication-status-test + args: --variable XTRACE:True + - id: single-system-tests name: Single node with encryption suite: single-system-tests @@ -142,6 +147,10 @@ jobs: suite: cephfs-replication-test runner: ubuntu-24.04 + - id: rgw-replication-test + name: Test MicroCeph RGW replication status against a real multisite. + suite: rgw-replication-test + - id: nfs-test name: Test MicroCeph NFS feature suite: nfs-test diff --git a/microceph/api/ops_replication.go b/microceph/api/ops_replication.go index 6043e4e8..c14c43f2 100644 --- a/microceph/api/ops_replication.go +++ b/microceph/api/ops_replication.go @@ -117,6 +117,22 @@ func cmdOpsReplication(s mcTypes.State, r *http.Request, overwriteType types.Rep // If the request is not WorkloadReplicationRequest, set the request type. data.OverwriteRequestType(overwriteType) req = data + case string(types.RgwWorkload): + var data types.RgwReplicationRequest + err := json.NewDecoder(r.Body).Decode(&data) + if err != nil { + logger.Errorf("REPOPS: failed to decode request data: %v", err.Error()) + return mcTypes.InternalError(err) + } + + // carry RgwReplicationRequest in interface object. + err = data.SetAPIObjectID(resource) + if err != nil { + return mcTypes.InternalError(err) + } + // If the request is not WorkloadReplicationRequest, set the request type. + data.OverwriteRequestType(overwriteType) + req = data default: return mcTypes.SmartError(fmt.Errorf("unknown workload %s, resource %s", wl, resource)) } diff --git a/microceph/api/types/replication_rgw.go b/microceph/api/types/replication_rgw.go new file mode 100644 index 00000000..670c3fa3 --- /dev/null +++ b/microceph/api/types/replication_rgw.go @@ -0,0 +1,183 @@ +package types + +import ( + "net/url" + + "github.com/canonical/microceph/microceph/logger" +) + +// ################################## RGW Replication Request ################################## + +// RgwResourceType defines the scope of an RGW replication request. +type RgwResourceType ReplicationResourceType + +const ( + // RgwResourceSite scopes a request to the whole cluster, i.e. to the + // local zone's place in the multisite topology. It doubles as the + // resource segment of the API path for such a request, i.e. + // /1.0/ops/replication/rgw/site, because an empty segment would route + // the request to the workload root endpoint instead, where a GET means + // list rather than status. + RgwResourceSite RgwResourceType = "site" + // RgwResourceBucket scopes a request to a single bucket, whose name + // fills the resource segment instead. + RgwResourceBucket RgwResourceType = "bucket" +) + +// RgwReplicationRequest implements ReplicationRequest for RGW replication. +// +// It carries only the fields the implemented verbs read. Enable, disable, +// configure and promote each need more - a remote name, replication modes, +// endpoint lists, a force flag - and each brings its own when it lands. The +// request body is decoded leniently, so a later field is an ordinary additive +// change rather than a break. +type RgwReplicationRequest struct { + Bucket string `json:"bucket" yaml:"bucket"` + ResourceType RgwResourceType `json:"resource_type" yaml:"resource_type"` + RequestType ReplicationRequestType `json:"request_type" yaml:"request_type"` +} + +// GetWorkloadType provides the workload name for replication request +func (req RgwReplicationRequest) GetWorkloadType() CephWorkloadType { + return RgwWorkload +} + +// GetAPIObjectID provides the API object id i.e. /replication/rgw/ +// +// An empty id is what routes a request to the workload root endpoint, so it is +// returned for exactly the cluster wide verbs. Every other verb is site or +// bucket scoped and must carry a resource segment, or the daemon would answer +// it with the wrong event entirely. +func (req RgwReplicationRequest) GetAPIObjectID() string { + switch req.RequestType { + case WorkloadReplicationRequest, ListReplicationRequest, PromoteReplicationRequest, DemoteReplicationRequest: + return "" + } + + if len(req.Bucket) != 0 { + resource := url.QueryEscape(req.Bucket) + logger.Debugf("REPAPI: Resource: %s", resource) + return resource + } + + return string(RgwResourceSite) +} + +// SetAPIObjectID populates the request from the API object id i.e. +// /replication/rgw/ +// +// The site sentinel carries no data. A bucket name does, but the request +// body's resource type stays authoritative, so a bucket named "site" is never +// mistaken for the sentinel. +func (req *RgwReplicationRequest) SetAPIObjectID(id string) error { + // unescape object string + object, err := url.PathUnescape(id) + if err != nil { + return err + } + + if req.ResourceType == RgwResourceBucket { + req.Bucket = object + } + + return nil +} + +// GetAPIRequestType provides the REST method for the request +func (req RgwReplicationRequest) GetAPIRequestType() string { + return GetAPIRequestTypeGeneric(req.RequestType) +} + +// GetWorkloadRequestType provides the event used as the FSM trigger. +func (req RgwReplicationRequest) GetWorkloadRequestType() string { + return GetWorkloadRequestTypeGeneric(req.RequestType) +} + +// OverwriteRequestType sets the RequestType param to provided value. +func (req *RgwReplicationRequest) OverwriteRequestType(overwriteRequestType ReplicationRequestType) { + if len(overwriteRequestType) != 0 { + req.RequestType = overwriteRequestType + } +} + +// ################################## RGW Replication Response ################################## + +// RgwReplicationSyncState is one sync stream's verdict, rendered for an +// operator. The inconclusive outcomes are deliberately distinct from each +// other and from being behind: a peer or local status that could not be +// read is not a claim about how far behind this zone is, and a peer that +// is not a configured source has no stream to make claims about at all. +type RgwReplicationSyncState string + +const ( + // RgwSyncStateCaughtUp means every shard has been compared against the + // peer's log head and none of them is behind. + RgwSyncStateCaughtUp RgwReplicationSyncState = "caught-up" + // RgwSyncStateBehind means at least one shard is behind, or is still + // doing its first full copy. + RgwSyncStateBehind RgwReplicationSyncState = "behind" + // RgwSyncStatePeerUnavailable means the peer's log head could not be + // read, so no comparison was possible. + RgwSyncStatePeerUnavailable RgwReplicationSyncState = "peer-unavailable" + // RgwSyncStateLocalUnavailable means this zone's own sync markers could + // not be read locally, so no comparison was even attempted. + RgwSyncStateLocalUnavailable RgwReplicationSyncState = "local-unavailable" + // RgwSyncStatePeriodMismatch means this zone is on an older realm + // period, so its markers cannot be compared with the peer's log. + RgwSyncStatePeriodMismatch RgwReplicationSyncState = "period-mismatch" + // RgwSyncStateNotSource means the local zone is not configured to sync + // data from this peer at all - sync_from_all is off and the peer is + // not named in sync_from - so there is no stream to report progress on. + RgwSyncStateNotSource RgwReplicationSyncState = "not-a-source" + // RgwSyncStateMaster means the stream does not apply: this zone is the + // metadata master and syncs from no one. + RgwSyncStateMaster RgwReplicationSyncState = "master" +) + +// RgwReplicationZoneBrief describes one member of the local zonegroup. +type RgwReplicationZoneBrief struct { + Name string `json:"name" yaml:"name"` + ID string `json:"id" yaml:"id"` + Endpoints []string `json:"endpoints" yaml:"endpoints"` + IsMaster bool `json:"is_master" yaml:"is_master"` + IsLocal bool `json:"is_local" yaml:"is_local"` +} + +// RgwReplicationSyncBrief reports how far the local zone has got syncing one +// stream: its metadata from the master, or its data from one source zone. +// +// SyncStatus is what radosgw-admin reports about the stream itself ("sync" +// once running, "init" before it starts), while State is the comparison +// against the peer's log. BehindShards and FullSyncShards are only meaningful +// when that comparison actually ran, i.e. when State is caught-up or behind. +// +// Note that a caught-up data stream means this zone has noticed every bucket +// the source logged activity for. It does not prove every object inside those +// buckets arrived, nor that no shard is retrying a failed object: reading that +// costs one radosgw-admin call per shard, which is too slow for a status +// command. Use `radosgw-admin bucket sync status` for per-object certainty. +type RgwReplicationSyncBrief struct { + SourceZone string `json:"source_zone" yaml:"source_zone"` + RemoteName string `json:"remote" yaml:"remote"` + State RgwReplicationSyncState `json:"state" yaml:"state"` + SyncStatus string `json:"sync_status" yaml:"sync_status"` + ShardCount int `json:"shard_count" yaml:"shard_count"` + BehindShards []int `json:"behind_shards" yaml:"behind_shards"` + FullSyncShards int `json:"full_sync_shards" yaml:"full_sync_shards"` +} + +// RgwReplicationResponseStatus is the site scoped status of RGW replication: +// the local zone's place in the multisite topology, plus one sync brief per +// stream flowing into it. +type RgwReplicationResponseStatus struct { + Realm string `json:"realm" yaml:"realm"` + RealmEpoch int `json:"realm_epoch" yaml:"realm_epoch"` + CurrentPeriod string `json:"current_period" yaml:"current_period"` + ZoneGroup string `json:"zonegroup" yaml:"zonegroup"` + Zone string `json:"zone" yaml:"zone"` + IsMasterZone bool `json:"is_master_zone" yaml:"is_master_zone"` + MasterZone string `json:"master_zone" yaml:"master_zone"` + Zones []RgwReplicationZoneBrief `json:"zones" yaml:"zones"` + MetadataSync RgwReplicationSyncBrief `json:"metadata_sync" yaml:"metadata_sync"` + DataSync []RgwReplicationSyncBrief `json:"data_sync" yaml:"data_sync"` +} diff --git a/microceph/api/types/replication_rgw_test.go b/microceph/api/types/replication_rgw_test.go new file mode 100644 index 00000000..9316c7cb --- /dev/null +++ b/microceph/api/types/replication_rgw_test.go @@ -0,0 +1,97 @@ +package types + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// The API object id is not cosmetic: an empty one routes a request to the +// workload root endpoint, which can only ever fire the cluster wide events. +// A site scoped verb that returns an empty id would silently be answered as a +// list, so each verb's id is pinned here. + +func TestRgwGetAPIObjectIDSiteScopedVerbs(t *testing.T) { + for _, requestType := range []ReplicationRequestType{ + StatusReplicationRequest, + EnableReplicationRequest, + DisableReplicationRequest, + ConfigureReplicationRequest, + } { + req := RgwReplicationRequest{RequestType: requestType, ResourceType: RgwResourceSite} + assert.Equal(t, "site", req.GetAPIObjectID(), "request type %s", requestType) + } +} + +func TestRgwGetAPIObjectIDClusterWideVerbs(t *testing.T) { + for _, requestType := range []ReplicationRequestType{ + ListReplicationRequest, + PromoteReplicationRequest, + DemoteReplicationRequest, + WorkloadReplicationRequest, + } { + req := RgwReplicationRequest{RequestType: requestType} + assert.Empty(t, req.GetAPIObjectID(), "request type %s", requestType) + } +} + +func TestRgwGetAPIObjectIDBucketScoped(t *testing.T) { + req := RgwReplicationRequest{ + RequestType: StatusReplicationRequest, + ResourceType: RgwResourceBucket, + Bucket: "my-bucket.photos", + } + + assert.Equal(t, "my-bucket.photos", req.GetAPIObjectID()) +} + +func TestRgwSetAPIObjectIDSiteSentinelCarriesNoData(t *testing.T) { + req := RgwReplicationRequest{ResourceType: RgwResourceSite} + + err := req.SetAPIObjectID("site") + assert.NoError(t, err) + assert.Empty(t, req.Bucket) + assert.Equal(t, RgwResourceSite, req.ResourceType) +} + +func TestRgwSetAPIObjectIDBucket(t *testing.T) { + req := RgwReplicationRequest{ResourceType: RgwResourceBucket} + + err := req.SetAPIObjectID("my-bucket.photos") + assert.NoError(t, err) + assert.Equal(t, "my-bucket.photos", req.Bucket) +} + +// A bucket that happens to be named "site" is scoped by the request body, not +// by the path segment it shares with the sentinel. +func TestRgwSetAPIObjectIDBucketNamedSite(t *testing.T) { + req := RgwReplicationRequest{ResourceType: RgwResourceBucket} + + err := req.SetAPIObjectID("site") + assert.NoError(t, err) + assert.Equal(t, "site", req.Bucket) + assert.Equal(t, RgwResourceBucket, req.ResourceType) +} + +func TestRgwRequestTypeAccessors(t *testing.T) { + req := RgwReplicationRequest{RequestType: StatusReplicationRequest} + + assert.Equal(t, RgwWorkload, req.GetWorkloadType()) + assert.Equal(t, "GET", req.GetAPIRequestType()) + assert.Equal(t, "status_replication", req.GetWorkloadRequestType()) +} + +func TestRgwOverwriteRequestType(t *testing.T) { + // Promote and demote share one endpoint and one HTTP verb, so the route + // cannot say which of the two arrived. The server passes the empty + // sentinel to mean "trust the body", and the guard has to leave the + // client's value alone. + req := RgwReplicationRequest{RequestType: PromoteReplicationRequest} + req.OverwriteRequestType(WorkloadReplicationRequest) + assert.Equal(t, PromoteReplicationRequest, req.RequestType) + + // Every other endpoint maps to exactly one verb, so the server's value + // wins over whatever the client encoded. + req.OverwriteRequestType(EnableReplicationRequest) + assert.Equal(t, EnableReplicationRequest, req.RequestType) +} diff --git a/microceph/ceph/replication.go b/microceph/ceph/replication.go index bd5dcff2..8aa2bb5b 100644 --- a/microceph/ceph/replication.go +++ b/microceph/ceph/replication.go @@ -43,10 +43,10 @@ type ReplicationHandlerInterface interface { } func GetReplicationHandler(name string) ReplicationHandlerInterface { - // Add RGW and CephFs Replication handlers here. table := map[string]ReplicationHandlerInterface{ "rbd": &RbdReplicationHandler{}, "cephfs": &CephfsReplicationHandler{}, + "rgw": &RgwReplicationHandler{}, } rh, ok := table[name] diff --git a/microceph/ceph/replication_rgw.go b/microceph/ceph/replication_rgw.go new file mode 100644 index 00000000..27559d4a --- /dev/null +++ b/microceph/ceph/replication_rgw.go @@ -0,0 +1,537 @@ +package ceph + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/database" + "github.com/canonical/microceph/microceph/interfaces" + "github.com/canonical/microceph/microceph/logger" +) + +// RgwReplicationHandler implements ReplicationHandlerInterface for RGW +// multisite replication. +// +// Every fact it reports about replication is derived live from RADOS, and +// none of it is persisted: a secondary cluster keeps no replication state of +// its own, so a stored answer would be empty exactly where an operator most +// needs one. +// +// The database is read for one thing only, which remotes are imported, since +// that is how a peer's own sync logs are reached. +type RgwReplicationHandler struct { + // Prefill objects: always populated before any handler is called. + Realm RgwRealm + ZoneGroup RgwZoneGroup + Zone RgwZone + // Request Info + Request types.RgwReplicationRequest + + // Only populated during status requests. + // The realm period, read only when the local zonegroup is not the + // realm's master zonegroup: it is the one read that can name the + // metadata master across zonegroups (see masterZoneName). + Period RgwPeriod + // Metadata sync markers, left zero valued on the metadata master. + MetadataSync RgwMetadataSyncStatus + // MetadataSyncUnavailable records that the local metadata sync status + // read failed, leaving MetadataSync a placeholder rather than an + // answer. Check it before reading MetadataSync anywhere. + MetadataSyncUnavailable bool + // Data sync markers for each source zone, keyed by source zone name. + DataSync map[string]RgwDataSyncStatus + // DataSyncUnavailable records the source zones whose local data sync + // status read failed; their DataSync entries are placeholders too. + DataSyncUnavailable map[string]bool +} + +// PreFill populates the handler struct with the local multisite topology. +// +// The empty cluster and client pair on every read below targets this cluster; +// a non-empty pair would append --cluster and --id and answer about a peer +// instead. +// +// A cluster wide request arrives with no resource fields set at all, so +// nothing here may assume the request carries a zone, bucket or remote. +func (rh *RgwReplicationHandler) PreFill(ctx context.Context, request types.ReplicationRequest) error { + var err error + req := request.(types.RgwReplicationRequest) + rh.Request = req + + // The read wrappers turn a failing radosgw-admin call into a zero value + // with a nil error, so a gateway that is not configured for multisite + // (or not running at all) reads as ordinary empty state here. + rh.Realm, err = GetRgwRealm("", "") + if err != nil { + return err + } + + // Without a realm there is no topology to describe, and every remaining + // read would come back empty anyway. + if len(rh.Realm.Name) == 0 { + return nil + } + + rh.ZoneGroup, err = GetRgwZoneGroup("", "") + if err != nil { + return err + } + + rh.Zone, err = GetRgwZone("", "") + if err != nil { + return err + } + + // Sync markers are only needed to answer a status request (the CephFS + // precedent), and reading them costs one radosgw-admin call per source. + if req.RequestType == types.StatusReplicationRequest { + // The metadata master's name can only be resolved through the + // realm period when it lives outside the local zonegroup. + if !rh.ZoneGroup.IsMaster { + rh.Period, err = GetRgwPeriod("", "") + if err != nil { + return err + } + } + + err = rh.preFillSyncStatus() + if err != nil { + return err + } + } + + return nil +} + +// preFillSyncStatus reads this zone's own sync markers: its metadata progress, +// and its data progress against every source zone it is configured to pull +// from. The metadata read is purely local, but `data sync status` first +// fetches the source zone's datalog shard info over HTTP: a source that +// refuses connections fails the read immediately (surfaced per stream +// below), while a blackholed one blocks it for radosgw-admin's full 300s +// curl timeout. +// +// A read whose command failed outright is recorded per stream rather than +// failing the whole request, mirroring how an unreadable peer log is already +// handled: one broken stream must not blank out the healthy ones. A malformed +// or self-contradictory response still fails the request, since it means the +// gateway answered and the answer cannot be trusted. +func (rh *RgwReplicationHandler) preFillSyncStatus() error { + // The metadata master syncs from no one and reports an empty "init" + // status. Reading it would only invite that emptiness to be mistaken for + // a stalled secondary. + if !rh.isMasterZone() { + status, err := GetRgwMetadataSyncStatus("", "") + if err != nil { + if !errors.Is(err, ErrRgwSyncStatusUnreadable) { + return err + } + logger.Warnf("REPRGW: %v", err) + rh.MetadataSyncUnavailable = true + } else { + rh.MetadataSync = status + } + } + + rh.DataSync = map[string]RgwDataSyncStatus{} + rh.DataSyncUnavailable = map[string]bool{} + for _, zone := range rh.dataSyncSourceZones() { + status, err := GetRgwDataSyncStatus(zone.Name, "", "") + if err != nil { + if !errors.Is(err, ErrRgwSyncStatusUnreadable) { + return err + } + logger.Warnf("REPRGW: %v", err) + rh.DataSyncUnavailable[zone.Name] = true + continue + } + + rh.DataSync[zone.Name] = status + } + + return nil +} + +// GetResourceState fetches the replication state of the local RGW zone. +func (rh *RgwReplicationHandler) GetResourceState() (ReplicationState, error) { + // No realm means a plain single site gateway, or no gateway at all. + if len(rh.Realm.Name) == 0 { + return StateDisabledReplication, nil + } + + // A realm whose zonegroup no longer lists the local zone is a zone that + // has been removed from the topology: nothing replicates to or from it. + if !rh.isZoneGroupMember() { + return StateDisabledReplication, nil + } + + return StateEnabledReplication, nil +} + +// EnableHandler is not implemented for the rgw workload yet. +func (rh *RgwReplicationHandler) EnableHandler(ctx context.Context, args ...any) error { + logger.Debugf("REPRGW: Enable handler, Req %v", rh.Request) + return fmt.Errorf("%s not implemented for rgw", types.EnableReplicationRequest) +} + +// DisableHandler is not implemented for the rgw workload yet. +func (rh *RgwReplicationHandler) DisableHandler(ctx context.Context, args ...any) error { + logger.Debugf("REPRGW: Disable handler, Req %v", rh.Request) + return fmt.Errorf("%s not implemented for rgw", types.DisableReplicationRequest) +} + +// ConfigureHandler is not implemented for the rgw workload yet. +func (rh *RgwReplicationHandler) ConfigureHandler(ctx context.Context, args ...any) error { + logger.Debugf("REPRGW: Configure handler, Req %v", rh.Request) + return fmt.Errorf("%s not implemented for rgw", types.ConfigureReplicationRequest) +} + +// ListHandler is not implemented for the rgw workload yet. +func (rh *RgwReplicationHandler) ListHandler(ctx context.Context, args ...any) error { + logger.Debugf("REPRGW: List handler, Req %v", rh.Request) + return fmt.Errorf("%s not implemented for rgw", types.ListReplicationRequest) +} + +// PromoteHandler is not implemented for the rgw workload yet. +func (rh *RgwReplicationHandler) PromoteHandler(ctx context.Context, args ...any) error { + logger.Debugf("REPRGW: Promote handler, Req %v", rh.Request) + return fmt.Errorf("%s not implemented for rgw", types.PromoteReplicationRequest) +} + +// DemoteHandler is not implemented for the rgw workload yet. +func (rh *RgwReplicationHandler) DemoteHandler(ctx context.Context, args ...any) error { + logger.Debugf("REPRGW: Demote handler, Req %v", rh.Request) + return fmt.Errorf("%s not implemented for rgw", types.DemoteReplicationRequest) +} + +// StatusHandler reports the local zone's place in the multisite topology and +// how far it has got syncing from each of its peers. +func (rh *RgwReplicationHandler) StatusHandler(ctx context.Context, args ...any) error { + logger.Debugf("REPRGW: Status handler, Req %v", rh.Request) + + if rh.Request.ResourceType == types.RgwResourceBucket { + return fmt.Errorf("bucket scoped %s is not implemented for rgw", types.StatusReplicationRequest) + } + + st := args[repArgState].(interfaces.CephState) + remotes, err := getRgwRemotesByZone(ctx, st) + if err != nil { + return err + } + + response := types.RgwReplicationResponseStatus{ + Realm: rh.Realm.Name, + RealmEpoch: rh.Realm.Epoch, + CurrentPeriod: rh.Realm.CurrentPeriod, + ZoneGroup: rh.ZoneGroup.Name, + Zone: rh.Zone.Name, + IsMasterZone: rh.isMasterZone(), + MasterZone: rh.masterZoneName(), + Zones: rh.zoneBriefs(), + MetadataSync: rh.metadataSyncBrief(remotes), + DataSync: rh.dataSyncBriefs(remotes), + } + + // Marshal to json string + data, err := json.Marshal(response) + if err != nil { + err := fmt.Errorf("failed to marshal resource status: %w", err) + logger.Error(err.Error()) + return err + } + + // pass response for API + *args[repArgResponse].(*string) = string(data) + return nil +} + +// metadataSyncBrief compares this zone's metadata markers against the master's +// metadata log. The master itself syncs from no one, so it reports that +// instead of a comparison it cannot make. +func (rh *RgwReplicationHandler) metadataSyncBrief(remotes map[string]types.RemoteRecord) types.RgwReplicationSyncBrief { + masterZone := rh.masterZoneName() + if rh.isMasterZone() { + return types.RgwReplicationSyncBrief{ + SourceZone: masterZone, + State: types.RgwSyncStateMaster, + BehindShards: []int{}, + } + } + + if rh.MetadataSyncUnavailable { + // The local read failed outright, so there is no comparison to + // make: neither caught up nor behind, and not the peer's fault. + remote := remotes[masterZone] + return summariseRgwSyncVerdict(masterZone, remote.Name, rh.MetadataSync.Info, RgwSyncVerdict{LocalUnavailable: true}) + } + + remote, ok := remotes[masterZone] + if !ok { + // Without a remote for the master cluster its metadata log cannot + // be read at all, which is not the same as being caught up. + logger.Warnf("REPRGW: no remote is imported for master zone %q, its metadata log cannot be read", masterZone) + return summariseRgwSyncVerdict(masterZone, "", rh.MetadataSync.Info, RgwSyncVerdict{PeerLogUnavailable: true}) + } + + masterLog, err := GetRgwMdlogStatus(remote.Name, remote.LocalName) + if err != nil { + logger.Warnf("REPRGW: failed to read the metadata log of remote %s: %v", remote.Name, err) + masterLog = nil + } + + verdict := ComputeRgwMetadataSyncVerdict(rh.MetadataSync, masterLog, rh.Realm.CurrentPeriod) + return summariseRgwSyncVerdict(masterZone, remote.Name, rh.MetadataSync.Info, verdict) +} + +// dataSyncBriefs compares this zone's data markers against each source zone's +// data log, one brief per peer zone in the zonegroup. A peer the local zone +// is not configured to sync from still gets a brief, an explicit not-a-source +// one, so the response's peer coverage stays predictable. +func (rh *RgwReplicationHandler) dataSyncBriefs(remotes map[string]types.RemoteRecord) []types.RgwReplicationSyncBrief { + peers := rh.peerZones() + briefs := make([]types.RgwReplicationSyncBrief, 0, len(peers)) + + for _, zone := range peers { + if !rh.isDataSyncSource(zone) { + // No stream from this peer is configured, so there is no + // progress to fabricate a verdict about. This outranks the + // other outcomes: even "unavailable" would be an answer about + // a stream that does not exist. + briefs = append(briefs, summariseRgwSyncVerdict(zone.Name, "", RgwSyncInfo{}, RgwSyncVerdict{NotSource: true})) + continue + } + + local := rh.DataSync[zone.Name] + + if rh.DataSyncUnavailable[zone.Name] { + // The local read for this stream failed outright: nothing was + // measured, which must not fall through to a real verdict. + remote := remotes[zone.Name] + briefs = append(briefs, summariseRgwSyncVerdict(zone.Name, remote.Name, local.Info, RgwSyncVerdict{LocalUnavailable: true})) + continue + } + + remote, ok := remotes[zone.Name] + if !ok { + // The source's data log lives in the source's own cluster, so + // without a remote for it there is nothing to compare against. + // Reading the local log here instead would compare this zone + // with itself and always report caught up. + logger.Warnf("REPRGW: no remote is imported for source zone %q, its data log cannot be read", zone.Name) + briefs = append(briefs, summariseRgwSyncVerdict(zone.Name, "", local.Info, RgwSyncVerdict{PeerLogUnavailable: true})) + continue + } + + sourceLog, err := GetRgwDatalogStatus(remote.Name, remote.LocalName) + if err != nil { + logger.Warnf("REPRGW: failed to read the data log of remote %s: %v", remote.Name, err) + sourceLog = nil + } + + verdict := ComputeRgwDataSyncVerdict(local, sourceLog) + briefs = append(briefs, summariseRgwSyncVerdict(zone.Name, remote.Name, local.Info, verdict)) + } + + return briefs +} + +// zoneBriefs describes every member of the local zonegroup. +func (rh *RgwReplicationHandler) zoneBriefs() []types.RgwReplicationZoneBrief { + briefs := make([]types.RgwReplicationZoneBrief, 0, len(rh.ZoneGroup.Zones)) + for _, zone := range rh.ZoneGroup.Zones { + briefs = append(briefs, types.RgwReplicationZoneBrief{ + Name: zone.Name, + ID: zone.ID, + Endpoints: zone.Endpoints, + IsMaster: zone.ID == rh.ZoneGroup.MasterZone, + IsLocal: zone.ID == rh.Zone.ID, + }) + } + + return briefs +} + +// isMasterZone reports whether the local zone is the realm's metadata +// master: the master zone of the realm's master zonegroup. Being master of +// a non-master zonegroup is not enough, since such a zone still syncs its +// metadata from the realm's master. Mirrors RGWSI_Zone::is_meta_master in +// Ceph's src/rgw/services/svc_zone.cc. The zonegroup names its master by +// id, never by name. +func (rh *RgwReplicationHandler) isMasterZone() bool { + return rh.ZoneGroup.IsMaster && len(rh.Zone.ID) != 0 && rh.ZoneGroup.MasterZone == rh.Zone.ID +} + +// isZoneGroupMember reports whether the local zone belongs to the zonegroup. +func (rh *RgwReplicationHandler) isZoneGroupMember() bool { + if len(rh.Zone.ID) == 0 { + return false + } + + for _, zone := range rh.ZoneGroup.Zones { + if zone.ID == rh.Zone.ID { + return true + } + } + + return false +} + +// masterZoneName resolves the realm's metadata master - the master zone of +// the realm's master zonegroup - to a zone name. When the local zonegroup +// is the master zonegroup the answer is one of its own members; otherwise +// the master lives in a zonegroup a plain zonegroup get can never see, and +// the realm period, which carries every zonegroup, answers instead. +func (rh *RgwReplicationHandler) masterZoneName() string { + if rh.ZoneGroup.IsMaster { + for _, zone := range rh.ZoneGroup.Zones { + if zone.ID == rh.ZoneGroup.MasterZone { + return zone.Name + } + } + + return "" + } + + for _, zonegroup := range rh.Period.PeriodMap.ZoneGroups { + if zonegroup.ID != rh.Period.MasterZonegroup { + continue + } + + for _, zone := range zonegroup.Zones { + if zone.ID == rh.Period.MasterZone { + return zone.Name + } + } + } + + return "" +} + +// peerZones lists the zonegroup members other than the local zone. +func (rh *RgwReplicationHandler) peerZones() []RgwZoneGroupZone { + peers := make([]RgwZoneGroupZone, 0, len(rh.ZoneGroup.Zones)) + for _, zone := range rh.ZoneGroup.Zones { + if zone.ID == rh.Zone.ID { + continue + } + + peers = append(peers, zone) + } + + return peers +} + +// localZoneGroupZone finds the local zone's own entry in the zonegroup, +// which is where its sync_from configuration lives. +func (rh *RgwReplicationHandler) localZoneGroupZone() (RgwZoneGroupZone, bool) { + for _, zone := range rh.ZoneGroup.Zones { + if zone.ID == rh.Zone.ID { + return zone, true + } + } + + return RgwZoneGroupZone{}, false +} + +// isDataSyncSource reports whether the local zone pulls data from the given +// peer: every peer when sync_from_all is set, and only the zones named in +// sync_from otherwise. radosgw-admin's own sync status applies exactly this +// check (RGWZone::syncs_from) before reporting a data stream. +func (rh *RgwReplicationHandler) isDataSyncSource(peer RgwZoneGroupZone) bool { + local, ok := rh.localZoneGroupZone() + if !ok { + // An orphaned zone is already reported as disabled replication; + // hiding every peer here would dress that up as topology instead. + return true + } + + return local.SyncFromAll || slices.Contains(local.SyncFrom, peer.Name) +} + +// dataSyncSourceZones lists the peers the local zone actually pulls data +// from. A peer outside this set has no sync stream to report on. +func (rh *RgwReplicationHandler) dataSyncSourceZones() []RgwZoneGroupZone { + peers := rh.peerZones() + sources := make([]RgwZoneGroupZone, 0, len(peers)) + for _, zone := range peers { + if !rh.isDataSyncSource(zone) { + continue + } + + sources = append(sources, zone) + } + + return sources +} + +// getRgwRemotesByZone indexes the imported remotes by the zone name each one +// reaches. +// +// A peer's sync logs live in the peer's own cluster, so reading them needs the +// conf and keyring that `remote import` renders. Multisite names a peer's zone +// after the remote record it was created from, which is what makes this lookup +// possible without storing anything: a remote named siteb reaches the cluster +// hosting the zone named siteb. A zone with no matching remote is reported as +// unreadable rather than guessed at. +func getRgwRemotesByZone(ctx context.Context, st interfaces.CephState) (map[string]types.RemoteRecord, error) { + records, err := database.GetRemoteDb(ctx, st.ClusterState(), "") + if err != nil { + return nil, fmt.Errorf("failed to fetch the imported remotes: %w", err) + } + + remotes := make(map[string]types.RemoteRecord, len(records)) + for _, record := range records { + remotes[record.Name] = record + } + + return remotes, nil +} + +// summariseRgwSyncVerdict renders one sync verdict for an operator. +// +// The short circuited outcomes stay distinct from each other and from being +// behind. A peer whose log could not be read, a local status that could not +// be read and a period that could not be compared are not claims about how +// far behind this zone is, and a stream that is not configured at all makes +// no claim about progress either. Shard counts are only carried when the +// comparison actually ran, because a verdict that short circuited leaves +// them at zero, which would otherwise read as fully synced. +func summariseRgwSyncVerdict(sourceZone string, remoteName string, info RgwSyncInfo, verdict RgwSyncVerdict) types.RgwReplicationSyncBrief { + brief := types.RgwReplicationSyncBrief{ + SourceZone: sourceZone, + RemoteName: remoteName, + SyncStatus: info.Status, + ShardCount: info.NumShards, + BehindShards: []int{}, + } + + switch { + case verdict.NotSource: + brief.State = types.RgwSyncStateNotSource + return brief + case verdict.LocalUnavailable: + brief.State = types.RgwSyncStateLocalUnavailable + return brief + case verdict.PeriodMismatch: + brief.State = types.RgwSyncStatePeriodMismatch + return brief + case verdict.PeerLogUnavailable: + brief.State = types.RgwSyncStatePeerUnavailable + return brief + case verdict.CaughtUp: + brief.State = types.RgwSyncStateCaughtUp + default: + brief.State = types.RgwSyncStateBehind + } + + if len(verdict.BehindShards) != 0 { + brief.BehindShards = verdict.BehindShards + } + brief.FullSyncShards = verdict.FullSyncShards + + return brief +} diff --git a/microceph/ceph/replication_rgw_test.go b/microceph/ceph/replication_rgw_test.go new file mode 100644 index 00000000..e0ff44e8 --- /dev/null +++ b/microceph/ceph/replication_rgw_test.go @@ -0,0 +1,791 @@ +package ceph + +// Tests for the RGW replication handler. The prefill and status paths run +// against a mocked command runner and the captured JSON in test_assets/; the +// topology and verdict rendering helpers are tested with inline data. + +import ( + "context" + "encoding/json" + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/common" + "github.com/canonical/microceph/microceph/database" + "github.com/canonical/microceph/microceph/interfaces" + "github.com/canonical/microceph/microceph/mocks" + "github.com/canonical/microceph/microceph/tests" + mcTypes "github.com/canonical/microcluster/v3/microcluster/types" +) + +const ( + siteAZoneID = "7b7a8b32-3e1e-4bab-9965-e756fbe29aa7" + siteBZoneID = "58a9f4ec-c0b7-415d-93a5-8eb1c03818ae" + siteCZoneID = "9c151f92-d92b-4c28-a5a1-4f0f4dd2ea11" + + microcephZoneGroupID = "67be86c9-2912-4ce2-835d-9bdf91915363" + euZoneGroupID = "d6d1a03a-40c5-4f4a-9ce6-3b4c2f04a1de" +) + +// siteBZoneGet is a `zone get` response trimmed to the fields the handler +// reads, standing in for the secondary side of the captured two site pair. +const siteBZoneGet = `{"id": "58a9f4ec-c0b7-415d-93a5-8eb1c03818ae", "name": "siteb"}` + +// siteCZoneGet is the local zone of a second, non-master zonegroup in the +// same realm the captured fixtures describe. +const siteCZoneGet = `{"id": "9c151f92-d92b-4c28-a5a1-4f0f4dd2ea11", "name": "sitec"}` + +// euZoneGroupGet is a `zonegroup get` response for that second zonegroup: +// not the realm's master, with sitec as its own master and only member. +const euZoneGroupGet = `{ + "id": "d6d1a03a-40c5-4f4a-9ce6-3b4c2f04a1de", + "name": "eu", + "is_master": false, + "master_zone": "9c151f92-d92b-4c28-a5a1-4f0f4dd2ea11", + "zones": [{"id": "9c151f92-d92b-4c28-a5a1-4f0f4dd2ea11", "name": "sitec", "endpoints": ["http://10.85.33.10:80"]}], + "realm_id": "cf90947b-b444-488d-abd3-779c3c6062d7" +}` + +// euPeriodGet is the realm period as sitec sees it: both zonegroups, with +// the master zonegroup being the captured microceph one holding sitea. +const euPeriodGet = `{ + "master_zonegroup": "67be86c9-2912-4ce2-835d-9bdf91915363", + "master_zone": "7b7a8b32-3e1e-4bab-9965-e756fbe29aa7", + "period_map": {"zonegroups": [ + { + "id": "67be86c9-2912-4ce2-835d-9bdf91915363", + "name": "microceph", + "is_master": true, + "master_zone": "7b7a8b32-3e1e-4bab-9965-e756fbe29aa7", + "zones": [ + {"id": "7b7a8b32-3e1e-4bab-9965-e756fbe29aa7", "name": "sitea"}, + {"id": "58a9f4ec-c0b7-415d-93a5-8eb1c03818ae", "name": "siteb"} + ] + }, + { + "id": "d6d1a03a-40c5-4f4a-9ce6-3b4c2f04a1de", + "name": "eu", + "is_master": false, + "master_zone": "9c151f92-d92b-4c28-a5a1-4f0f4dd2ea11", + "zones": [{"id": "9c151f92-d92b-4c28-a5a1-4f0f4dd2ea11", "name": "sitec"}] + } + ]} +}` + +type RgwReplicationSuite struct { + tests.BaseSuite + getRemoteDb func(ctx context.Context, s mcTypes.State, name string) (types.RemoteRecords, error) +} + +func TestRgwReplication(t *testing.T) { + suite.Run(t, new(RgwReplicationSuite)) +} + +func (s *RgwReplicationSuite) SetupTest() { + s.BaseSuite.SetupTest() + s.getRemoteDb = database.GetRemoteDb +} + +func (s *RgwReplicationSuite) TearDownTest() { + database.GetRemoteDb = s.getRemoteDb +} + +// setRemotes points the remotes table at the provided records. +func (s *RgwReplicationSuite) setRemotes(records ...types.RemoteRecord) { + database.GetRemoteDb = func(ctx context.Context, st mcTypes.State, name string) (types.RemoteRecords, error) { + return records, nil + } +} + +// masterHandler is the local cluster as the captured fixtures describe it: +// zone sitea, which is the zonegroup's master, with siteb as its peer. +func masterHandler() *RgwReplicationHandler { + return &RgwReplicationHandler{ + Realm: RgwRealm{Name: "microceph", CurrentPeriod: "period-1", Epoch: 2}, + ZoneGroup: RgwZoneGroup{ + Name: "microceph", + IsMaster: true, + MasterZone: siteAZoneID, + Zones: []RgwZoneGroupZone{ + {ID: siteBZoneID, Name: "siteb", Endpoints: []string{"http://10.85.32.128:80"}, SyncFromAll: true}, + {ID: siteAZoneID, Name: "sitea", Endpoints: []string{"http://10.85.32.250:80"}, SyncFromAll: true}, + }, + }, + Zone: RgwZone{ID: siteAZoneID, Name: "sitea"}, + } +} + +// secondaryHandler is the same topology seen from siteb. +func secondaryHandler() *RgwReplicationHandler { + rh := masterHandler() + rh.Zone = RgwZone{ID: siteBZoneID, Name: "siteb"} + return rh +} + +// threeZoneHandler is masterHandler plus a third zone sitec, with the local +// zone sitea restricted to pulling data from siteb only. +func threeZoneHandler() *RgwReplicationHandler { + rh := masterHandler() + rh.ZoneGroup.Zones = []RgwZoneGroupZone{ + {ID: siteBZoneID, Name: "siteb", SyncFromAll: true}, + {ID: siteAZoneID, Name: "sitea", SyncFromAll: false, SyncFrom: []string{"siteb"}}, + {ID: siteCZoneID, Name: "sitec", SyncFromAll: true}, + } + return rh +} + +// ############################## PreFill ############################## + +func (s *RgwReplicationSuite) TestPreFillMasterZone() { + r := mocks.NewRunner(s.T()) + s.expectTopologyReads(r, "./test_assets/rgw_zone_get.json", "") + + // A master syncs from no one, so its own metadata markers are never + // read; only the data markers for its peer zone are. + dataSync, _ := os.ReadFile("./test_assets/rgw_data_sync_status.json") + r.On("RunCommand", []interface{}{ + "radosgw-admin", "data", "sync", "status", "--source-zone", "siteb"}...).Return(string(dataSync), nil).Once() + common.ProcessExec = r + + rh := &RgwReplicationHandler{} + err := rh.PreFill(context.Background(), types.RgwReplicationRequest{ + RequestType: types.StatusReplicationRequest, + ResourceType: types.RgwResourceSite, + }) + + assert.NoError(s.T(), err) + assert.Equal(s.T(), "microceph", rh.Realm.Name) + assert.Equal(s.T(), "sitea", rh.Zone.Name) + assert.True(s.T(), rh.isMasterZone()) + assert.Empty(s.T(), rh.MetadataSync.Info.Status) + assert.Equal(s.T(), 128, rh.DataSync["siteb"].Info.NumShards) +} + +func (s *RgwReplicationSuite) TestPreFillSecondaryZone() { + r := mocks.NewRunner(s.T()) + s.expectTopologyReads(r, "", siteBZoneGet) + + metaSync, _ := os.ReadFile("./test_assets/rgw_metadata_sync_status_secondary.json") + r.On("RunCommand", []interface{}{ + "radosgw-admin", "metadata", "sync", "status"}...).Return(string(metaSync), nil).Once() + dataSync, _ := os.ReadFile("./test_assets/rgw_data_sync_status.json") + r.On("RunCommand", []interface{}{ + "radosgw-admin", "data", "sync", "status", "--source-zone", "sitea"}...).Return(string(dataSync), nil).Once() + common.ProcessExec = r + + rh := &RgwReplicationHandler{} + err := rh.PreFill(context.Background(), types.RgwReplicationRequest{ + RequestType: types.StatusReplicationRequest, + ResourceType: types.RgwResourceSite, + }) + + assert.NoError(s.T(), err) + assert.False(s.T(), rh.isMasterZone()) + assert.Equal(s.T(), 64, rh.MetadataSync.Info.NumShards) + assert.Contains(s.T(), rh.DataSync, "sitea") +} + +// A local metadata sync status command that cannot run is a per stream +// outage: the rest of the prefill must survive it and the failure must be +// recorded rather than left as a convincing zero value. +func (s *RgwReplicationSuite) TestPreFillMetadataSyncUnavailable() { + r := mocks.NewRunner(s.T()) + s.expectTopologyReads(r, "", siteBZoneGet) + r.On("RunCommand", []interface{}{ + "radosgw-admin", "metadata", "sync", "status"}...).Return("", fmt.Errorf("exit status 5")).Once() + dataSync, _ := os.ReadFile("./test_assets/rgw_data_sync_status.json") + r.On("RunCommand", []interface{}{ + "radosgw-admin", "data", "sync", "status", "--source-zone", "sitea"}...).Return(string(dataSync), nil).Once() + common.ProcessExec = r + + rh := &RgwReplicationHandler{} + err := rh.PreFill(context.Background(), types.RgwReplicationRequest{ + RequestType: types.StatusReplicationRequest, + ResourceType: types.RgwResourceSite, + }) + + assert.NoError(s.T(), err) + assert.True(s.T(), rh.MetadataSyncUnavailable) + assert.Empty(s.T(), rh.MetadataSync.Info.Status) + assert.Equal(s.T(), 128, rh.DataSync["sitea"].Info.NumShards) +} + +// The same outage on one data stream must leave the metadata stream and the +// map bookkeeping intact. +func (s *RgwReplicationSuite) TestPreFillDataSyncUnavailable() { + r := mocks.NewRunner(s.T()) + s.expectTopologyReads(r, "", siteBZoneGet) + metaSync, _ := os.ReadFile("./test_assets/rgw_metadata_sync_status_secondary.json") + r.On("RunCommand", []interface{}{ + "radosgw-admin", "metadata", "sync", "status"}...).Return(string(metaSync), nil).Once() + r.On("RunCommand", []interface{}{ + "radosgw-admin", "data", "sync", "status", "--source-zone", "sitea"}...).Return("", fmt.Errorf("exit status 5")).Once() + common.ProcessExec = r + + rh := &RgwReplicationHandler{} + err := rh.PreFill(context.Background(), types.RgwReplicationRequest{ + RequestType: types.StatusReplicationRequest, + ResourceType: types.RgwResourceSite, + }) + + assert.NoError(s.T(), err) + assert.True(s.T(), rh.DataSyncUnavailable["sitea"]) + assert.NotContains(s.T(), rh.DataSync, "sitea") + assert.Equal(s.T(), 64, rh.MetadataSync.Info.NumShards) +} + +// A gateway that answers with a self-contradictory sync status is corrupt +// data rather than an outage, and must keep failing the whole request. +func (s *RgwReplicationSuite) TestPreFillMalformedSyncStatusStillFails() { + r := mocks.NewRunner(s.T()) + s.expectTopologyReads(r, "", siteBZoneGet) + invalid := `{"sync_status":{"info":{"status":"sync","num_shards":2},"markers":[{"key":5,"val":{"state":1,"marker":""}}]}}` + r.On("RunCommand", []interface{}{ + "radosgw-admin", "metadata", "sync", "status"}...).Return(invalid, nil).Once() + common.ProcessExec = r + + rh := &RgwReplicationHandler{} + err := rh.PreFill(context.Background(), types.RgwReplicationRequest{ + RequestType: types.StatusReplicationRequest, + ResourceType: types.RgwResourceSite, + }) + + assert.Error(s.T(), err) +} + +// A zone that is master of its own, non-master zonegroup is the exact +// topology the realm-wide master check exists for: it must still read its +// own metadata sync markers, and the metadata master it names lives in a +// zonegroup only the realm period can see. +func (s *RgwReplicationSuite) TestPreFillMasterOfNonMasterZoneGroup() { + r := mocks.NewRunner(s.T()) + realm, _ := os.ReadFile("./test_assets/rgw_realm_get.json") + r.On("RunCommand", []interface{}{ + "radosgw-admin", "realm", "get"}...).Return(string(realm), nil).Once() + r.On("RunCommand", []interface{}{ + "radosgw-admin", "zonegroup", "get"}...).Return(euZoneGroupGet, nil).Once() + r.On("RunCommand", []interface{}{ + "radosgw-admin", "zone", "get"}...).Return(siteCZoneGet, nil).Once() + r.On("RunCommand", []interface{}{ + "radosgw-admin", "period", "get"}...).Return(euPeriodGet, nil).Once() + metaSync, _ := os.ReadFile("./test_assets/rgw_metadata_sync_status_secondary.json") + r.On("RunCommand", []interface{}{ + "radosgw-admin", "metadata", "sync", "status"}...).Return(string(metaSync), nil).Once() + common.ProcessExec = r + + rh := &RgwReplicationHandler{} + err := rh.PreFill(context.Background(), types.RgwReplicationRequest{ + RequestType: types.StatusReplicationRequest, + ResourceType: types.RgwResourceSite, + }) + + assert.NoError(s.T(), err) + assert.False(s.T(), rh.isMasterZone()) + assert.Equal(s.T(), 64, rh.MetadataSync.Info.NumShards) + assert.Equal(s.T(), "sitea", rh.masterZoneName()) +} + +// A gateway with no realm stops the prefill dead: there is no topology to +// read, and every further call would come back empty anyway. +func (s *RgwReplicationSuite) TestPreFillUnconfigured() { + r := mocks.NewRunner(s.T()) + r.On("RunCommand", []interface{}{ + "radosgw-admin", "realm", "get"}...).Return("", fmt.Errorf("failed to load realm: (2) No such file or directory")).Once() + common.ProcessExec = r + + rh := &RgwReplicationHandler{} + err := rh.PreFill(context.Background(), types.RgwReplicationRequest{ + RequestType: types.StatusReplicationRequest, + ResourceType: types.RgwResourceSite, + }) + + assert.NoError(s.T(), err) + assert.Empty(s.T(), rh.Realm.Name) + + state, err := rh.GetResourceState() + assert.NoError(s.T(), err) + assert.Equal(s.T(), StateDisabledReplication, state) +} + +// A cluster wide request carries no resource fields at all, and the prefill +// must survive that: it is what every future list and promote arrives as. +func (s *RgwReplicationSuite) TestPreFillToleratesZeroValueRequest() { + r := mocks.NewRunner(s.T()) + s.expectTopologyReads(r, "./test_assets/rgw_zone_get.json", "") + common.ProcessExec = r + + rh := &RgwReplicationHandler{} + err := rh.PreFill(context.Background(), types.RgwReplicationRequest{}) + + assert.NoError(s.T(), err) + // Sync markers are only read for a status request. + assert.Empty(s.T(), rh.DataSync) +} + +// expectTopologyReads queues the realm, zonegroup and zone reads every prefill +// starts with. Pass either a zone fixture path or an inline zone response. +func (s *RgwReplicationSuite) expectTopologyReads(r *mocks.Runner, zoneFixture string, zoneResponse string) { + realm, _ := os.ReadFile("./test_assets/rgw_realm_get.json") + r.On("RunCommand", []interface{}{ + "radosgw-admin", "realm", "get"}...).Return(string(realm), nil).Once() + + zonegroup, _ := os.ReadFile("./test_assets/rgw_zonegroup_get.json") + r.On("RunCommand", []interface{}{ + "radosgw-admin", "zonegroup", "get"}...).Return(string(zonegroup), nil).Once() + + if len(zoneFixture) != 0 { + zone, _ := os.ReadFile(zoneFixture) + zoneResponse = string(zone) + } + r.On("RunCommand", []interface{}{ + "radosgw-admin", "zone", "get"}...).Return(zoneResponse, nil).Once() +} + +// ############################## GetResourceState ############################## + +func (s *RgwReplicationSuite) TestGetResourceStateEnabled() { + state, err := masterHandler().GetResourceState() + assert.NoError(s.T(), err) + assert.Equal(s.T(), StateEnabledReplication, state) +} + +// A zone removed from the zonegroup keeps its realm and its own configuration, +// but nothing replicates to or from it any more. +func (s *RgwReplicationSuite) TestGetResourceStateZoneNotInZoneGroup() { + rh := masterHandler() + rh.Zone = RgwZone{ID: "orphaned-zone-id", Name: "sitec"} + + state, err := rh.GetResourceState() + assert.NoError(s.T(), err) + assert.Equal(s.T(), StateDisabledReplication, state) +} + +func (s *RgwReplicationSuite) TestGetResourceStateNoZone() { + rh := masterHandler() + rh.Zone = RgwZone{} + + state, err := rh.GetResourceState() + assert.NoError(s.T(), err) + assert.Equal(s.T(), StateDisabledReplication, state) +} + +// ############################## Topology helpers ############################## + +func (s *RgwReplicationSuite) TestTopologyHelpers() { + rh := masterHandler() + + assert.True(s.T(), rh.isMasterZone()) + assert.True(s.T(), rh.isZoneGroupMember()) + assert.Equal(s.T(), "sitea", rh.masterZoneName()) + + peers := rh.peerZones() + assert.Len(s.T(), peers, 1) + assert.Equal(s.T(), "siteb", peers[0].Name) +} + +// A zone can be the master of its own zonegroup while the realm's metadata +// master lives in a different zonegroup. Such a zone still syncs metadata +// and must not present itself as a master. +func (s *RgwReplicationSuite) TestIsMasterZoneOfNonMasterZoneGroup() { + rh := masterHandler() + rh.ZoneGroup.IsMaster = false + + assert.False(s.T(), rh.isMasterZone()) +} + +// Data sync is directional: with sync_from_all off, only the peers named in +// sync_from are sources, exactly as RGWZone::syncs_from decides it. +func (s *RgwReplicationSuite) TestDataSyncSourceZones() { + rh := threeZoneHandler() + + assert.True(s.T(), rh.isDataSyncSource(RgwZoneGroupZone{ID: siteBZoneID, Name: "siteb"})) + assert.False(s.T(), rh.isDataSyncSource(RgwZoneGroupZone{ID: siteCZoneID, Name: "sitec"})) + + sources := rh.dataSyncSourceZones() + assert.Len(s.T(), sources, 1) + assert.Equal(s.T(), "siteb", sources[0].Name) +} + +// A non-source peer must not be queried at all: the strict mock proves no +// data sync status command runs for sitec. +func (s *RgwReplicationSuite) TestPreFillSkipsNonSourcePeers() { + r := mocks.NewRunner(s.T()) + dataSync, _ := os.ReadFile("./test_assets/rgw_data_sync_status.json") + r.On("RunCommand", []interface{}{ + "radosgw-admin", "data", "sync", "status", "--source-zone", "siteb"}...).Return(string(dataSync), nil).Once() + common.ProcessExec = r + + rh := threeZoneHandler() + err := rh.preFillSyncStatus() + + assert.NoError(s.T(), err) + assert.Contains(s.T(), rh.DataSync, "siteb") + assert.NotContains(s.T(), rh.DataSync, "sitec") +} + +// In a non-master zonegroup the metadata master's name comes from the realm +// period, since the local zonegroup listing cannot contain it. +func (s *RgwReplicationSuite) TestMasterZoneNameAcrossZoneGroups() { + rh := masterHandler() + rh.ZoneGroup.IsMaster = false + rh.Period = RgwPeriod{ + MasterZonegroup: euZoneGroupID, + MasterZone: siteCZoneID, + PeriodMap: RgwPeriodMap{ + ZoneGroups: []RgwZoneGroup{ + rh.ZoneGroup, + { + ID: euZoneGroupID, + Name: "eu", + IsMaster: true, + MasterZone: siteCZoneID, + Zones: []RgwZoneGroupZone{{ID: siteCZoneID, Name: "sitec"}}, + }, + }, + }, + } + + assert.Equal(s.T(), "sitec", rh.masterZoneName()) +} + +// Without a period the cross-zonegroup master cannot be named at all, which +// must read as empty rather than as the local zonegroup's own master. +func (s *RgwReplicationSuite) TestMasterZoneNameNonMasterZoneGroupNoPeriod() { + rh := masterHandler() + rh.ZoneGroup.IsMaster = false + + assert.Empty(s.T(), rh.masterZoneName()) +} + +func (s *RgwReplicationSuite) TestZoneBriefs() { + briefs := masterHandler().zoneBriefs() + + assert.Len(s.T(), briefs, 2) + assert.Equal(s.T(), "siteb", briefs[0].Name) + assert.False(s.T(), briefs[0].IsMaster) + assert.False(s.T(), briefs[0].IsLocal) + assert.Equal(s.T(), "sitea", briefs[1].Name) + assert.True(s.T(), briefs[1].IsMaster) + assert.True(s.T(), briefs[1].IsLocal) +} + +// ############################## Verdict rendering ############################## + +func (s *RgwReplicationSuite) TestSummariseRgwSyncVerdictCaughtUp() { + brief := summariseRgwSyncVerdict("sitea", "sitea", RgwSyncInfo{Status: "sync", NumShards: 64}, RgwSyncVerdict{CaughtUp: true}) + + assert.Equal(s.T(), types.RgwSyncStateCaughtUp, brief.State) + assert.Equal(s.T(), 64, brief.ShardCount) + assert.Empty(s.T(), brief.BehindShards) + assert.Equal(s.T(), 0, brief.FullSyncShards) +} + +func (s *RgwReplicationSuite) TestSummariseRgwSyncVerdictBehind() { + verdict := RgwSyncVerdict{BehindShards: []int{3, 7}, FullSyncShards: 2} + brief := summariseRgwSyncVerdict("sitea", "sitea", RgwSyncInfo{Status: "sync", NumShards: 64}, verdict) + + assert.Equal(s.T(), types.RgwSyncStateBehind, brief.State) + assert.Equal(s.T(), []int{3, 7}, brief.BehindShards) + assert.Equal(s.T(), 2, brief.FullSyncShards) +} + +// An unreadable peer is not a claim about how far behind this zone is, and the +// shard counts a short circuited verdict leaves at zero must not travel with +// it: zero behind shards would otherwise read as caught up. +func (s *RgwReplicationSuite) TestSummariseRgwSyncVerdictPeerUnavailable() { + brief := summariseRgwSyncVerdict("sitea", "", RgwSyncInfo{Status: "sync", NumShards: 64}, RgwSyncVerdict{PeerLogUnavailable: true}) + + assert.Equal(s.T(), types.RgwSyncStatePeerUnavailable, brief.State) + assert.Empty(s.T(), brief.BehindShards) + assert.Equal(s.T(), 0, brief.FullSyncShards) +} + +// A stream that is not configured carries no counts, no sync state and no +// remote: it is a topology fact, not a measurement. +func (s *RgwReplicationSuite) TestSummariseRgwSyncVerdictNotSource() { + brief := summariseRgwSyncVerdict("sitec", "", RgwSyncInfo{}, RgwSyncVerdict{NotSource: true}) + + assert.Equal(s.T(), types.RgwSyncStateNotSource, brief.State) + assert.Empty(s.T(), brief.SyncStatus) + assert.Equal(s.T(), 0, brief.ShardCount) + assert.Empty(s.T(), brief.BehindShards) + assert.Equal(s.T(), 0, brief.FullSyncShards) +} + +// A local status that was never read carries no shard counts or sync state +// worth showing, and must not fall through to a real verdict. +func (s *RgwReplicationSuite) TestSummariseRgwSyncVerdictLocalUnavailable() { + brief := summariseRgwSyncVerdict("sitea", "sitea", RgwSyncInfo{}, RgwSyncVerdict{LocalUnavailable: true}) + + assert.Equal(s.T(), types.RgwSyncStateLocalUnavailable, brief.State) + assert.Empty(s.T(), brief.SyncStatus) + assert.Equal(s.T(), 0, brief.ShardCount) + assert.Empty(s.T(), brief.BehindShards) + assert.Equal(s.T(), 0, brief.FullSyncShards) +} + +func (s *RgwReplicationSuite) TestSummariseRgwSyncVerdictPeriodMismatch() { + verdict := RgwSyncVerdict{PeriodMismatch: true} + brief := summariseRgwSyncVerdict("sitea", "sitea", RgwSyncInfo{Status: "sync", NumShards: 64}, verdict) + + assert.Equal(s.T(), types.RgwSyncStatePeriodMismatch, brief.State) +} + +// ############################## Sync briefs ############################## + +func (s *RgwReplicationSuite) TestMetadataSyncBriefOnMaster() { + brief := masterHandler().metadataSyncBrief(map[string]types.RemoteRecord{}) + + assert.Equal(s.T(), types.RgwSyncStateMaster, brief.State) + assert.Equal(s.T(), "sitea", brief.SourceZone) +} + +// Without a remote for the master cluster its metadata log cannot be read, and +// reading the local one instead would compare this zone with itself. +func (s *RgwReplicationSuite) TestMetadataSyncBriefWithoutRemote() { + rh := secondaryHandler() + rh.MetadataSync = RgwMetadataSyncStatus{Info: RgwSyncInfo{Status: "sync", NumShards: 64}} + + brief := rh.metadataSyncBrief(map[string]types.RemoteRecord{}) + + assert.Equal(s.T(), types.RgwSyncStatePeerUnavailable, brief.State) + assert.Equal(s.T(), "sitea", brief.SourceZone) + assert.Empty(s.T(), brief.RemoteName) +} + +func (s *RgwReplicationSuite) TestMetadataSyncBriefWithRemote() { + r := mocks.NewRunner(s.T()) + mdlog, _ := os.ReadFile("./test_assets/rgw_mdlog_status.json") + r.On("RunCommand", []interface{}{ + "radosgw-admin", "mdlog", "status", "--cluster", "sitea", "--id", "siteb"}...).Return(string(mdlog), nil).Once() + common.ProcessExec = r + + rh := secondaryHandler() + rh.MetadataSync = RgwMetadataSyncStatus{ + Info: RgwSyncInfo{Status: "sync", NumShards: 4, Period: "period-1"}, + Markers: []RgwMetadataSyncShard{ + {Key: 0, Val: RgwMetadataSyncMarker{State: RgwMetadataSyncStateIncremental}}, + {Key: 1, Val: RgwMetadataSyncMarker{State: RgwMetadataSyncStateIncremental}}, + {Key: 2, Val: RgwMetadataSyncMarker{State: RgwMetadataSyncStateIncremental}}, + {Key: 3, Val: RgwMetadataSyncMarker{State: RgwMetadataSyncStateIncremental, Marker: "1_1784681399.801225_678.1"}}, + }, + } + + brief := rh.metadataSyncBrief(map[string]types.RemoteRecord{ + "sitea": {Name: "sitea", LocalName: "siteb"}, + }) + + // Shard 2's log head is ahead of an empty local marker; shard 3 matches. + assert.Equal(s.T(), types.RgwSyncStateBehind, brief.State) + assert.Equal(s.T(), []int{2}, brief.BehindShards) + assert.Equal(s.T(), "sitea", brief.RemoteName) +} + +// A zone left behind on an older period cannot have its markers compared with +// the master's log at all, so it must not read as either caught up or behind. +// This exercises the handler's own wiring of the realm period into the +// comparison, not just the comparison itself. +func (s *RgwReplicationSuite) TestMetadataSyncBriefPeriodMismatch() { + r := mocks.NewRunner(s.T()) + mdlog, _ := os.ReadFile("./test_assets/rgw_mdlog_status.json") + r.On("RunCommand", []interface{}{ + "radosgw-admin", "mdlog", "status", "--cluster", "sitea", "--id", "siteb"}...).Return(string(mdlog), nil).Once() + common.ProcessExec = r + + rh := secondaryHandler() + rh.MetadataSync = RgwMetadataSyncStatus{ + Info: RgwSyncInfo{Status: "sync", NumShards: 4, Period: "an-older-period"}, + Markers: []RgwMetadataSyncShard{ + {Key: 0, Val: RgwMetadataSyncMarker{State: RgwMetadataSyncStateIncremental}}, + }, + } + + brief := rh.metadataSyncBrief(map[string]types.RemoteRecord{ + "sitea": {Name: "sitea", LocalName: "siteb"}, + }) + + assert.Equal(s.T(), types.RgwSyncStatePeriodMismatch, brief.State) + assert.Empty(s.T(), brief.BehindShards) + assert.Equal(s.T(), 0, brief.FullSyncShards) +} + +// A failed local metadata read renders as local-unavailable without ever +// touching the peer: no command may run here. +func (s *RgwReplicationSuite) TestMetadataSyncBriefLocalUnavailable() { + common.ProcessExec = mocks.NewRunner(s.T()) + + rh := secondaryHandler() + rh.MetadataSyncUnavailable = true + + brief := rh.metadataSyncBrief(map[string]types.RemoteRecord{ + "sitea": {Name: "sitea", LocalName: "siteb"}, + }) + + assert.Equal(s.T(), types.RgwSyncStateLocalUnavailable, brief.State) + assert.Equal(s.T(), "sitea", brief.SourceZone) + assert.Equal(s.T(), "sitea", brief.RemoteName) + assert.Empty(s.T(), brief.SyncStatus) + assert.Equal(s.T(), 0, brief.ShardCount) +} + +// The same rendering per data stream. +func (s *RgwReplicationSuite) TestDataSyncBriefsLocalUnavailable() { + common.ProcessExec = mocks.NewRunner(s.T()) + + rh := masterHandler() + rh.DataSync = map[string]RgwDataSyncStatus{} + rh.DataSyncUnavailable = map[string]bool{"siteb": true} + + briefs := rh.dataSyncBriefs(map[string]types.RemoteRecord{ + "siteb": {Name: "siteb", LocalName: "sitea"}, + }) + + assert.Len(s.T(), briefs, 1) + assert.Equal(s.T(), types.RgwSyncStateLocalUnavailable, briefs[0].State) + assert.Equal(s.T(), "siteb", briefs[0].SourceZone) + assert.Equal(s.T(), "siteb", briefs[0].RemoteName) +} + +// Every zonegroup peer still gets exactly one brief; a non-source peer's is +// the explicit not-a-source one, and it outranks every other outcome, even +// a recorded local read failure for the same zone. +func (s *RgwReplicationSuite) TestDataSyncBriefsNotSource() { + common.ProcessExec = mocks.NewRunner(s.T()) + + rh := threeZoneHandler() + rh.DataSync = map[string]RgwDataSyncStatus{ + "siteb": {Info: RgwSyncInfo{Status: "sync", NumShards: 128}}, + } + rh.DataSyncUnavailable = map[string]bool{"sitec": true} + + briefs := rh.dataSyncBriefs(map[string]types.RemoteRecord{}) + + assert.Len(s.T(), briefs, 2) + assert.Equal(s.T(), "siteb", briefs[0].SourceZone) + assert.Equal(s.T(), types.RgwSyncStatePeerUnavailable, briefs[0].State) + assert.Equal(s.T(), "sitec", briefs[1].SourceZone) + assert.Equal(s.T(), types.RgwSyncStateNotSource, briefs[1].State) + assert.Empty(s.T(), briefs[1].RemoteName) + assert.Empty(s.T(), briefs[1].SyncStatus) +} + +func (s *RgwReplicationSuite) TestDataSyncBriefsWithoutRemote() { + rh := masterHandler() + rh.DataSync = map[string]RgwDataSyncStatus{ + "siteb": {Info: RgwSyncInfo{Status: "sync", NumShards: 128}}, + } + + briefs := rh.dataSyncBriefs(map[string]types.RemoteRecord{}) + + assert.Len(s.T(), briefs, 1) + assert.Equal(s.T(), "siteb", briefs[0].SourceZone) + assert.Equal(s.T(), types.RgwSyncStatePeerUnavailable, briefs[0].State) +} + +func (s *RgwReplicationSuite) TestDataSyncBriefsWithRemote() { + r := mocks.NewRunner(s.T()) + datalog, _ := os.ReadFile("./test_assets/rgw_datalog_status.json") + r.On("RunCommand", []interface{}{ + "radosgw-admin", "datalog", "status", "--cluster", "siteb", "--id", "sitea"}...).Return(string(datalog), nil).Once() + common.ProcessExec = r + + rh := masterHandler() + rh.DataSync = map[string]RgwDataSyncStatus{ + "siteb": { + Info: RgwSyncInfo{Status: "sync", NumShards: 3}, + Markers: []RgwDataSyncShard{ + {Key: 0, Val: RgwDataSyncMarker{Status: "incremental-sync"}}, + {Key: 1, Val: RgwDataSyncMarker{Status: "incremental-sync"}}, + {Key: 2, Val: RgwDataSyncMarker{Status: "incremental-sync", Marker: "00000000000000000000:00000000000000000512"}}, + }, + }, + } + + briefs := rh.dataSyncBriefs(map[string]types.RemoteRecord{ + "siteb": {Name: "siteb", LocalName: "sitea"}, + }) + + assert.Len(s.T(), briefs, 1) + assert.Equal(s.T(), types.RgwSyncStateCaughtUp, briefs[0].State) + assert.Equal(s.T(), "siteb", briefs[0].RemoteName) +} + +// ############################## StatusHandler ############################## + +func (s *RgwReplicationSuite) TestStatusHandler() { + r := mocks.NewRunner(s.T()) + datalog, _ := os.ReadFile("./test_assets/rgw_datalog_status.json") + r.On("RunCommand", []interface{}{ + "radosgw-admin", "datalog", "status", "--cluster", "siteb", "--id", "sitea"}...).Return(string(datalog), nil).Once() + common.ProcessExec = r + s.setRemotes(types.RemoteRecord{Name: "siteb", LocalName: "sitea"}) + + rh := masterHandler() + rh.Request = types.RgwReplicationRequest{ + RequestType: types.StatusReplicationRequest, + ResourceType: types.RgwResourceSite, + } + rh.DataSync = map[string]RgwDataSyncStatus{ + "siteb": {Info: RgwSyncInfo{Status: "sync", NumShards: 3}}, + } + + var resp string + err := rh.StatusHandler(context.Background(), rh, &resp, interfaces.CephState{}) + assert.NoError(s.T(), err) + + var status types.RgwReplicationResponseStatus + err = json.Unmarshal([]byte(resp), &status) + assert.NoError(s.T(), err) + + assert.Equal(s.T(), "microceph", status.Realm) + assert.Equal(s.T(), 2, status.RealmEpoch) + assert.Equal(s.T(), "sitea", status.Zone) + assert.True(s.T(), status.IsMasterZone) + assert.Equal(s.T(), "sitea", status.MasterZone) + assert.Len(s.T(), status.Zones, 2) + assert.Equal(s.T(), types.RgwSyncStateMaster, status.MetadataSync.State) + assert.Len(s.T(), status.DataSync, 1) + assert.Equal(s.T(), "siteb", status.DataSync[0].SourceZone) +} + +// Bucket scoped status is a later rung of the ladder; until then it says so +// rather than silently answering with the site wide view. +func (s *RgwReplicationSuite) TestStatusHandlerRejectsBucketScope() { + rh := masterHandler() + rh.Request = types.RgwReplicationRequest{ + RequestType: types.StatusReplicationRequest, + ResourceType: types.RgwResourceBucket, + Bucket: "photos", + } + + var resp string + err := rh.StatusHandler(context.Background(), rh, &resp, interfaces.CephState{}) + assert.ErrorContains(s.T(), err, "not implemented for rgw") +} + +// ############################## Unimplemented verbs ############################## + +func (s *RgwReplicationSuite) TestUnimplementedVerbs() { + rh := masterHandler() + ctx := context.Background() + + assert.ErrorContains(s.T(), rh.EnableHandler(ctx), "not implemented for rgw") + assert.ErrorContains(s.T(), rh.DisableHandler(ctx), "not implemented for rgw") + assert.ErrorContains(s.T(), rh.ConfigureHandler(ctx), "not implemented for rgw") + assert.ErrorContains(s.T(), rh.ListHandler(ctx), "not implemented for rgw") + assert.ErrorContains(s.T(), rh.PromoteHandler(ctx), "not implemented for rgw") + assert.ErrorContains(s.T(), rh.DemoteHandler(ctx), "not implemented for rgw") +} + +// The handler must be reachable through the workload registry, or the API +// answers every rgw request with "no replication handler". +func (s *RgwReplicationSuite) TestHandlerIsRegistered() { + rh := GetReplicationHandler(string(types.RgwWorkload)) + assert.NotNil(s.T(), rh) + assert.IsType(s.T(), &RgwReplicationHandler{}, rh) +} diff --git a/microceph/ceph/rgw_multisite.go b/microceph/ceph/rgw_multisite.go index a6a9ad5f..a58aee5f 100644 --- a/microceph/ceph/rgw_multisite.go +++ b/microceph/ceph/rgw_multisite.go @@ -2,12 +2,19 @@ package ceph import ( "encoding/json" + "errors" "fmt" "github.com/canonical/microceph/microceph/common" "github.com/canonical/microceph/microceph/logger" ) +// ErrRgwSyncStatusUnreadable marks a sync status read whose radosgw-admin +// command failed outright. Callers use it to tell "the command could not +// run at all" apart from a malformed or self-contradictory response, which +// keeps propagating as an ordinary error. +var ErrRgwSyncStatusUnreadable = errors.New("rgw sync status could not be read") + // radosgwAdminRun runs radosgw-admin with the given arguments. func radosgwAdminRun(args ...string) (string, error) { return common.ProcessExec.RunCommand("radosgw-admin", args...) @@ -34,6 +41,29 @@ type RgwZoneGroupZone struct { Name string `json:"name"` Endpoints []string `json:"endpoints"` ReadOnly bool `json:"read_only"` + // SyncFromAll and SyncFrom say which peers this zone pulls data from: + // every peer when SyncFromAll is set, and only the zones named in + // SyncFrom otherwise. Mirrors RGWZone::syncs_from in Ceph's + // src/rgw/rgw_zone_types.h. + SyncFromAll bool `json:"sync_from_all"` + SyncFrom []string `json:"sync_from"` +} + +// UnmarshalJSON decodes a zonegroup zone entry with sync_from_all +// defaulting to true when the field is absent, as radosgw-admin's own +// decoder does (RGWZone::decode_json in Ceph's src/rgw/rgw_zone.cc). A +// plain bool would read an absent field as false and invert the topology. +func (z *RgwZoneGroupZone) UnmarshalJSON(data []byte) error { + type rgwZoneGroupZoneAlias RgwZoneGroupZone + decoded := rgwZoneGroupZoneAlias{SyncFromAll: true} + + err := json.Unmarshal(data, &decoded) + if err != nil { + return err + } + + *z = RgwZoneGroupZone(decoded) + return nil } // RgwZoneGroup is the subset of `zonegroup get` output we use. @@ -120,6 +150,45 @@ func GetRgwZone(cluster string, client string) (RgwZone, error) { return response, nil } +// RgwPeriodMap is the subset of the period's zonegroup directory RGW +// replication uses: every zonegroup in the realm, each with its own zone +// list, not just the one the local zone belongs to. +type RgwPeriodMap struct { + ZoneGroups []RgwZoneGroup `json:"zonegroups"` +} + +// RgwPeriod is the subset of `period get` output RGW replication uses. +// MasterZone is the realm's metadata master - the master zone of the +// realm's master zonegroup - which differs from the local zonegroup's own +// master_zone whenever the local zonegroup is not the realm's master. +type RgwPeriod struct { + MasterZonegroup string `json:"master_zonegroup"` + MasterZone string `json:"master_zone"` + PeriodMap RgwPeriodMap `json:"period_map"` +} + +// GetRgwPeriod fetches the realm's current period, the one topology read +// that describes every zonegroup in the realm rather than only the local +// one. A non-empty cluster/client pair targets a remote cluster. A failing +// command yields a zero value and a nil error, so an unconfigured gateway +// reads as ordinary empty state. +func GetRgwPeriod(cluster string, client string) (RgwPeriod, error) { + response := RgwPeriod{} + + output, err := radosgwAdminRunRemote(cluster, client, "period", "get") + if err != nil { + logger.Warnf("REPRGW: failed period get operation: %v", err) + return response, nil + } + + err = json.Unmarshal([]byte(output), &response) + if err != nil { + return response, fmt.Errorf("cannot unmarshal period get output: %w", err) + } + + return response, nil +} + // RgwSyncInfo is the info block shared by `metadata sync status` and // `data sync status` output. Period and RealmEpoch are metadata-only. type RgwSyncInfo struct { @@ -233,15 +302,15 @@ func validateRgwDataSyncShards(numShards int, markers []RgwDataSyncShard) error // GetRgwMetadataSyncStatus fetches this zone's own metadata sync markers - // local progress only, no peer contact. A non-empty cluster/client pair -// targets a remote cluster. A failing command yields a zero value and a -// nil error. +// targets a remote cluster. A failing command returns an error wrapping +// ErrRgwSyncStatusUnreadable rather than a zero value: a zero value would +// later compare as behind, which is a claim this read never made. func GetRgwMetadataSyncStatus(cluster string, client string) (RgwMetadataSyncStatus, error) { envelope := rgwMetadataSyncEnvelope{} output, err := radosgwAdminRunRemote(cluster, client, "metadata", "sync", "status") if err != nil { - logger.Warnf("REPRGW: failed metadata sync status operation: %v", err) - return RgwMetadataSyncStatus{}, nil + return RgwMetadataSyncStatus{}, fmt.Errorf("%w: failed metadata sync status operation: %w", ErrRgwSyncStatusUnreadable, err) } err = json.Unmarshal([]byte(output), &envelope) @@ -260,14 +329,14 @@ func GetRgwMetadataSyncStatus(cluster string, client string) (RgwMetadataSyncSta // GetRgwDataSyncStatus fetches this zone's own data sync markers for one // source zone - local progress only, no contact with the source. A // non-empty cluster/client pair targets a remote cluster. A failing -// command yields a zero value and a nil error. +// command returns an error wrapping ErrRgwSyncStatusUnreadable rather than +// a zero value, for the same reason as GetRgwMetadataSyncStatus. func GetRgwDataSyncStatus(sourceZone string, cluster string, client string) (RgwDataSyncStatus, error) { envelope := rgwDataSyncEnvelope{} output, err := radosgwAdminRunRemote(cluster, client, "data", "sync", "status", "--source-zone", sourceZone) if err != nil { - logger.Warnf("REPRGW: failed data sync status operation for source(%s): %v", sourceZone, err) - return RgwDataSyncStatus{}, nil + return RgwDataSyncStatus{}, fmt.Errorf("%w: failed data sync status operation for source %q: %w", ErrRgwSyncStatusUnreadable, sourceZone, err) } err = json.Unmarshal([]byte(output), &envelope) @@ -357,8 +426,15 @@ func GetRgwDatalogStatus(cluster string, client string) ([]RgwLogShard, error) { // neither is caught up. Skip this call entirely for a master rather than // reading the resulting false as behind. // -// PeriodMismatch and PeerLogUnavailable both mean the comparison could not -// be made at all, so neither is a claim about how far behind the zone is. +// PeriodMismatch, PeerLogUnavailable and LocalUnavailable all mean the +// comparison could not be made at all - the first two because the peer's +// side could not be used, the last because this zone's own markers were +// never read - so none of them is a claim about how far behind the zone +// is. LocalUnavailable is never set by the compute functions, which are +// only called with a local status that was actually read; the handler sets +// it in their place when the local read failed. NotSource likewise never +// comes from the compute functions: it says no stream from the peer is +// configured at all, so there was nothing to compute. // // Known gaps: a shard the peer does not report is logged and skipped // rather than counted, as upstream also does; log trimming can briefly @@ -371,6 +447,8 @@ type RgwSyncVerdict struct { FullSyncShards int PeriodMismatch bool PeerLogUnavailable bool + LocalUnavailable bool + NotSource bool } // ComputeRgwMetadataSyncVerdict compares a secondary's metadata markers diff --git a/microceph/ceph/rgw_multisite_test.go b/microceph/ceph/rgw_multisite_test.go index 19810866..279794ed 100644 --- a/microceph/ceph/rgw_multisite_test.go +++ b/microceph/ceph/rgw_multisite_test.go @@ -5,6 +5,7 @@ package ceph // the pure verdict and validation helpers are tested with inline data. import ( + "encoding/json" "fmt" "os" "testing" @@ -92,6 +93,29 @@ func (s *RgwMultisiteSuite) TestGetRgwZoneGroup() { assert.Contains(s.T(), names, "sitea") assert.Contains(s.T(), names, "siteb") assert.NotEmpty(s.T(), zonegroup.Zones[0].Endpoints) + assert.True(s.T(), zonegroup.Zones[0].SyncFromAll) + assert.Empty(s.T(), zonegroup.Zones[0].SyncFrom) +} + +// sync_from_all defaults to true when absent, matching radosgw-admin's own +// decoder; a plain zero value would silently invert the topology. +func (s *RgwMultisiteSuite) TestRgwZoneGroupZoneUnmarshalDefaults() { + zone := RgwZoneGroupZone{} + err := json.Unmarshal([]byte(`{"id": "z1", "name": "sitea"}`), &zone) + assert.NoError(s.T(), err) + assert.True(s.T(), zone.SyncFromAll) + assert.Empty(s.T(), zone.SyncFrom) + + zone = RgwZoneGroupZone{} + err = json.Unmarshal([]byte(`{"id": "z1", "name": "sitea", "sync_from_all": false, "sync_from": ["siteb"]}`), &zone) + assert.NoError(s.T(), err) + assert.False(s.T(), zone.SyncFromAll) + assert.Equal(s.T(), []string{"siteb"}, zone.SyncFrom) + + zone = RgwZoneGroupZone{} + err = json.Unmarshal([]byte(`{"id": "z1", "name": "sitea", "sync_from_all": true}`), &zone) + assert.NoError(s.T(), err) + assert.True(s.T(), zone.SyncFromAll) } func (s *RgwMultisiteSuite) TestGetRgwZone() { @@ -110,6 +134,51 @@ func (s *RgwMultisiteSuite) TestGetRgwZone() { assert.NotEmpty(s.T(), zone.SystemKey.SecretKey) } +func (s *RgwMultisiteSuite) TestGetRgwPeriod() { + r := mocks.NewRunner(s.T()) + + output, _ := os.ReadFile("./test_assets/rgw_period_get.json") + r.On("RunCommand", []interface{}{ + "radosgw-admin", "period", "get"}...).Return(string(output), nil).Once() + common.ProcessExec = r + + period, err := GetRgwPeriod("", "") + assert.NoError(s.T(), err) + assert.Equal(s.T(), "67be86c9-2912-4ce2-835d-9bdf91915363", period.MasterZonegroup) + assert.Equal(s.T(), "7b7a8b32-3e1e-4bab-9965-e756fbe29aa7", period.MasterZone) + assert.Len(s.T(), period.PeriodMap.ZoneGroups, 1) + assert.Equal(s.T(), "microceph", period.PeriodMap.ZoneGroups[0].Name) + assert.Len(s.T(), period.PeriodMap.ZoneGroups[0].Zones, 2) +} + +func (s *RgwMultisiteSuite) TestGetRgwPeriodRemote() { + r := mocks.NewRunner(s.T()) + + output, _ := os.ReadFile("./test_assets/rgw_period_get.json") + r.On("RunCommand", []interface{}{ + "radosgw-admin", "period", "get", "--cluster", "siteb", "--id", "sitea"}...).Return(string(output), nil).Once() + common.ProcessExec = r + + period, err := GetRgwPeriod("siteb", "sitea") + assert.NoError(s.T(), err) + assert.Equal(s.T(), "7b7a8b32-3e1e-4bab-9965-e756fbe29aa7", period.MasterZone) +} + +func (s *RgwMultisiteSuite) TestGetRgwPeriodUnconfigured() { + r := mocks.NewRunner(s.T()) + + // A realm-less gateway fails period get; the wrapper swallows the exec + // error into a zero-value period like the other topology reads. + r.On("RunCommand", []interface{}{ + "radosgw-admin", "period", "get"}...).Return("", fmt.Errorf("failed to load realm: (2) No such file or directory")).Once() + common.ProcessExec = r + + period, err := GetRgwPeriod("", "") + assert.NoError(s.T(), err) + assert.Empty(s.T(), period.MasterZone) + assert.Empty(s.T(), period.PeriodMap.ZoneGroups) +} + func (s *RgwMultisiteSuite) TestGetRgwMetadataSyncStatusSecondary() { r := mocks.NewRunner(s.T()) @@ -139,6 +208,31 @@ func (s *RgwMultisiteSuite) TestGetRgwMetadataSyncStatusInvalidResponse() { _, err := GetRgwMetadataSyncStatus("", "") assert.Error(s.T(), err) + assert.NotErrorIs(s.T(), err, ErrRgwSyncStatusUnreadable) +} + +// A command that cannot run at all is a different failure from a malformed +// response, and the sentinel is what lets callers tell them apart. +func (s *RgwMultisiteSuite) TestGetRgwMetadataSyncStatusCommandFailure() { + r := mocks.NewRunner(s.T()) + + r.On("RunCommand", []interface{}{ + "radosgw-admin", "metadata", "sync", "status"}...).Return("", fmt.Errorf("exit status 5")).Once() + common.ProcessExec = r + + _, err := GetRgwMetadataSyncStatus("", "") + assert.ErrorIs(s.T(), err, ErrRgwSyncStatusUnreadable) +} + +func (s *RgwMultisiteSuite) TestGetRgwDataSyncStatusCommandFailure() { + r := mocks.NewRunner(s.T()) + + r.On("RunCommand", []interface{}{ + "radosgw-admin", "data", "sync", "status", "--source-zone", "sitea"}...).Return("", fmt.Errorf("exit status 5")).Once() + common.ProcessExec = r + + _, err := GetRgwDataSyncStatus("sitea", "", "") + assert.ErrorIs(s.T(), err, ErrRgwSyncStatusUnreadable) } func (s *RgwMultisiteSuite) TestGetRgwMetadataSyncStatusMaster() { @@ -184,6 +278,7 @@ func (s *RgwMultisiteSuite) TestGetRgwDataSyncStatusInvalidResponse() { _, err := GetRgwDataSyncStatus("sitea", "", "") assert.Error(s.T(), err) + assert.NotErrorIs(s.T(), err, ErrRgwSyncStatusUnreadable) } func (s *RgwMultisiteSuite) TestGetRgwMdlogStatus() { diff --git a/microceph/ceph/test_assets/rgw_period_get.json b/microceph/ceph/test_assets/rgw_period_get.json new file mode 100644 index 00000000..6b331230 --- /dev/null +++ b/microceph/ceph/test_assets/rgw_period_get.json @@ -0,0 +1,178 @@ +{ + "id": "9b9a5a4f-ecb4-42a1-b2ff-b31fc0ef5b1b", + "epoch": 2, + "predecessor_uuid": "e2b0a5a1-8e5f-4c3a-9a2e-1f6d0c4b7a90", + "sync_status": [], + "period_map": { + "id": "9b9a5a4f-ecb4-42a1-b2ff-b31fc0ef5b1b", + "zonegroups": [ + { + "id": "67be86c9-2912-4ce2-835d-9bdf91915363", + "name": "microceph", + "api_name": "microceph", + "is_master": true, + "endpoints": [ + "http://10.85.32.250:80" + ], + "hostnames": [], + "hostnames_s3website": [], + "master_zone": "7b7a8b32-3e1e-4bab-9965-e756fbe29aa7", + "zones": [ + { + "id": "58a9f4ec-c0b7-415d-93a5-8eb1c03818ae", + "name": "siteb", + "endpoints": [ + "http://10.85.32.128:80" + ], + "log_meta": false, + "log_data": true, + "bucket_index_max_shards": 11, + "read_only": false, + "tier_type": "", + "sync_from_all": true, + "sync_from": [], + "redirect_zone": "", + "supported_features": [ + "compress-encrypted", + "notification_v2", + "resharding" + ] + }, + { + "id": "7b7a8b32-3e1e-4bab-9965-e756fbe29aa7", + "name": "sitea", + "endpoints": [ + "http://10.85.32.250:80" + ], + "log_meta": false, + "log_data": true, + "bucket_index_max_shards": 11, + "read_only": false, + "tier_type": "", + "sync_from_all": true, + "sync_from": [], + "redirect_zone": "", + "supported_features": [ + "compress-encrypted", + "notification_v2", + "resharding" + ] + } + ], + "placement_targets": [ + { + "name": "default-placement", + "tags": [], + "storage_classes": [ + "STANDARD" + ] + } + ], + "default_placement": "default-placement", + "realm_id": "cf90947b-b444-488d-abd3-779c3c6062d7", + "sync_policy": { + "groups": [ + { + "id": "default", + "data_flow": { + "symmetrical": [ + { + "id": "sitea-siteb", + "zones": [ + "58a9f4ec-c0b7-415d-93a5-8eb1c03818ae", + "7b7a8b32-3e1e-4bab-9965-e756fbe29aa7" + ] + } + ] + }, + "pipes": [ + { + "id": "all", + "source": { + "bucket": "*", + "zones": [ + "*" + ] + }, + "dest": { + "bucket": "*", + "zones": [ + "*" + ] + }, + "params": { + "source": { + "filter": { + "tags": [] + } + }, + "dest": {}, + "priority": 0, + "mode": "system", + "user": "" + } + } + ], + "status": "enabled" + } + ] + }, + "enabled_features": [ + "notification_v2", + "resharding" + ] + } + ], + "short_zone_ids": [ + { + "key": "58a9f4ec-c0b7-415d-93a5-8eb1c03818ae", + "val": 1720993486 + }, + { + "key": "7b7a8b32-3e1e-4bab-9965-e756fbe29aa7", + "val": 4230083189 + } + ] + }, + "master_zonegroup": "67be86c9-2912-4ce2-835d-9bdf91915363", + "master_zone": "7b7a8b32-3e1e-4bab-9965-e756fbe29aa7", + "period_config": { + "bucket_quota": { + "enabled": false, + "check_on_raw": false, + "max_size": -1, + "max_size_kb": 0, + "max_objects": -1 + }, + "user_quota": { + "enabled": false, + "check_on_raw": false, + "max_size": -1, + "max_size_kb": 0, + "max_objects": -1 + }, + "user_ratelimit": { + "max_read_ops": 0, + "max_write_ops": 0, + "max_read_bytes": 0, + "max_write_bytes": 0, + "enabled": false + }, + "bucket_ratelimit": { + "max_read_ops": 0, + "max_write_ops": 0, + "max_read_bytes": 0, + "max_write_bytes": 0, + "enabled": false + }, + "anonymous_ratelimit": { + "max_read_ops": 0, + "max_write_ops": 0, + "max_read_bytes": 0, + "max_write_bytes": 0, + "enabled": false + } + }, + "realm_id": "cf90947b-b444-488d-abd3-779c3c6062d7", + "realm_epoch": 2 +} diff --git a/tests/robot/README.md b/tests/robot/README.md index 3b57a8f3..86f5a654 100644 --- a/tests/robot/README.md +++ b/tests/robot/README.md @@ -71,16 +71,17 @@ Each directory under `tests/robot/` is a suite: ``` api-tests nfs-test -availability-zone-tests nfs-multinode-test -cephadm-adopt-test rbd-replication-test -cephfs-replication-test single-system-tests -cluster-tests static-checks -dsl-functional-tests test-maintenance-modes -loop-file-tests test-sequential-mon-host-refresh -messenger-v2-tests unit-tests -multi-node-tests upgrade-reef-tests +availability-zone-tests rbd-replication-test +cephadm-adopt-test rgw-replication-status-test +cephfs-replication-test rgw-replication-test +cluster-tests single-system-tests +dsl-functional-tests static-checks +loop-file-tests test-maintenance-modes +messenger-v2-tests test-sequential-mon-host-refresh +multi-node-tests unit-tests multi-node-tests-with-custom-microceph-ip - wal-db-tests + upgrade-reef-tests +nfs-multinode-test wal-db-tests wiping-test ``` diff --git a/tests/robot/resources/microceph_harness.py b/tests/robot/resources/microceph_harness.py index b7c00a13..69cd7ad8 100644 --- a/tests/robot/resources/microceph_harness.py +++ b/tests/robot/resources/microceph_harness.py @@ -28,6 +28,7 @@ rbd_primary_image_count, rbd_synced_image_count, ) +from rgw_replication import parse_rgw_replication_status, rgw_data_sync_states from snap_services import enabled_active_services from streaming_process import run_streaming_process @@ -1718,6 +1719,109 @@ def microceph_api_get(self, path): ) return res.stdout + def get_rgw_replication_status(self): + """GETs the site-scoped RGW replication status from the control socket + on the outer VM and returns the parsed status document. + + The ops/replication endpoints decode a JSON request body even on GET, + so one is always sent; the envelope's string-encoded metadata is + decoded by the pure parser in rgw_replication.py. + """ + body = '{"resource_type": "site", "request_type": ""}' + res = self.run_in_vm_and_check( + f"sudo curl -s --unix-socket {MICROCEPH_CONTROL_SOCKET}" + f" -X GET -H 'Content-Type: application/json' -d '{body}'" + " http://localhost/1.0/ops/replication/rgw/site", + 60, + ) + return parse_rgw_replication_status(res.stdout) + + def get_rgw_replication_status_in_container(self, container): + """GETs the site-scoped RGW replication status from the control socket + inside an inner container and returns the parsed status document. The + ops/replication endpoints decode a JSON request body even on GET, so + one is always sent.""" + res = self.exec_in_container( + container, "curl", "-s", "--unix-socket", MICROCEPH_CONTROL_SOCKET, + "-X", "GET", "-H", "Content-Type: application/json", + "-d", '{"resource_type": "site", "request_type": ""}', + "http://localhost/1.0/ops/replication/rgw/site", + timeout=60, check=True, + ) + return parse_rgw_replication_status(res.stdout) + + def _observe_rgw_sync_state(self, container, source_zone=None): + """Returns the current metadata sync state - or, with *source_zone*, that + source's data sync state - from the status API in *container*. Returns + None when the status cannot be read yet, so pollers keep probing while + a daemon restarts instead of aborting.""" + try: + status = self.get_rgw_replication_status_in_container(container) + except AssertionError: + return None + if source_zone is None: + return (status.get("metadata_sync") or {}).get("state") + return rgw_data_sync_states(status).get(source_zone) + + def wait_for_rgw_metadata_sync_state(self, container, expected, attempts=60, interval=15): + """Polls the status API in *container* until metadata_sync.state equals + *expected*. Initial sync of even an empty realm takes a while: every + shard must finish its first full sync before caught-up is possible.""" + last = {"state": None} + + def probe(): + last["state"] = self._observe_rgw_sync_state(container) + return last["state"] == expected + + self._poll_until( + probe, attempts, interval, + lambda: f"metadata sync on {container} never reached {expected!r} (last seen: {last['state']!r})", + ) + + def wait_for_rgw_data_sync_state(self, container, source_zone, expected, attempts=60, interval=15): + """Polls the status API in *container* until the data sync brief for + *source_zone* reports *expected*.""" + last = {"state": None} + + def probe(): + last["state"] = self._observe_rgw_sync_state(container, source_zone) + return last["state"] == expected + + self._poll_until( + probe, attempts, interval, + lambda: f"data sync from {source_zone} on {container} never reached {expected!r} (last seen: {last['state']!r})", + ) + + def wait_for_rgw_endpoint(self, container, url, attempts=30, interval=5): + """Polls until the RGW endpoint at *url* answers HTTP from inside + *container* - radosgw takes a few seconds to listen after enable, and + a realm pull against a not-yet-listening master would fail.""" + def probe(): + res = self.exec_in_container( + container, "curl", "-s", "-o", "/dev/null", url, timeout=15, + ) + return res.rc == 0 + + self._poll_until( + probe, attempts, interval, + f"rgw endpoint {url} never answered from {container}", + ) + + def wait_for_rgw_user_in_container(self, container, uid, attempts=40, interval=15): + """Polls until radosgw-admin in *container* can see user *uid*, i.e. the + user's metadata has replicated to that site.""" + def probe(): + res = self.exec_in_container( + container, "microceph.radosgw-admin", "user", "info", f"--uid={uid}", + timeout=60, + ) + return res.rc == 0 + + self._poll_until( + probe, attempts, interval, + f"user {uid!r} never replicated to {container}", + ) + def microceph_api_put(self, path, body, timeout=120): """PUTs a JSON body to a path on the MicroCeph control socket on the outer VM. diff --git a/tests/robot/resources/microceph_harness.resource b/tests/robot/resources/microceph_harness.resource index 4a3a5e8a..3b07ada2 100644 --- a/tests/robot/resources/microceph_harness.resource +++ b/tests/robot/resources/microceph_harness.resource @@ -9,6 +9,7 @@ Library microceph_harness.py Library streaming_process.py Library snap_services.py Library cephfs_replication.py +Library rgw_replication.py *** Variables *** ${SNAP_PATH} ${EMPTY} diff --git a/tests/robot/resources/rgw_replication.py b/tests/robot/resources/rgw_replication.py new file mode 100644 index 00000000..f2758917 --- /dev/null +++ b/tests/robot/resources/rgw_replication.py @@ -0,0 +1,42 @@ +"""Robot Framework library: parsing of RGW replication API output. + +Pure helpers (no Robot context needed) that keep RGW-specific JSON parsing +out of the shared harness, mirroring the cephfs_replication.py / +rbd_replication.py pattern. The fetch keyword (Get Rgw Replication Status) +lives in the harness and calls parse_rgw_replication_status to turn the raw +control-socket response into the status document. +""" + +import json + + +def parse_rgw_replication_status(raw): + """Returns the RGW replication status document from a raw API response. + + The ops/replication endpoints wrap the handler's JSON document in the + microcluster envelope as a string, i.e. ``{"metadata": "{\\"realm\\": ...}"}``, + so the metadata is decoded a second time when it arrives as a string. + Raises AssertionError when the response carries no status document, so a + failed request never reads as an empty status. + """ + try: + envelope = json.loads(raw) + except (ValueError, TypeError): + raise AssertionError(f"replication status response is not JSON: {raw!r}") + + metadata = envelope.get("metadata") if isinstance(envelope, dict) else None + if isinstance(metadata, str): + try: + metadata = json.loads(metadata) + except ValueError: + raise AssertionError(f"replication status metadata is not JSON: {metadata!r}") + if not isinstance(metadata, dict): + raise AssertionError(f"replication status response has no document: {raw!r}") + return metadata + + +def rgw_data_sync_states(status): + """Returns {source_zone: state} for every data sync brief in a parsed + status document, so a suite can assert per-peer states by name instead of + relying on list order.""" + return {brief["source_zone"]: brief["state"] for brief in (status.get("data_sync") or [])} diff --git a/tests/robot/resources/test_harness_helpers.py b/tests/robot/resources/test_harness_helpers.py index f7f82b82..670a0fea 100644 --- a/tests/robot/resources/test_harness_helpers.py +++ b/tests/robot/resources/test_harness_helpers.py @@ -22,6 +22,7 @@ rbd_primary_image_count, rbd_synced_image_count, ) +from rgw_replication import parse_rgw_replication_status, rgw_data_sync_states from streaming_process import run_streaming_process @@ -1264,3 +1265,48 @@ def fake_observe(member): msg = str(exc.value) assert "never became absent" in msg assert "unparseable" in msg + + +# --------------------------------------------------------------------------- +# rgw_replication.py pure helpers +# --------------------------------------------------------------------------- + + +def test_parse_rgw_replication_status_string_metadata(): + # The ops API wraps the handler's JSON document as a string in the + # microcluster envelope; both decodes must happen. + doc = {"realm": "verify", "zone": "us-east", "data_sync": []} + raw = json.dumps({"type": "sync", "status": "Success", "metadata": json.dumps(doc)}) + assert parse_rgw_replication_status(raw) == doc + + +def test_parse_rgw_replication_status_dict_metadata(): + # Tolerate a future server that stops string-encoding the document. + doc = {"realm": "verify"} + raw = json.dumps({"metadata": doc}) + assert parse_rgw_replication_status(raw) == doc + + +def test_parse_rgw_replication_status_rejects_no_document(): + # An error envelope or non-JSON body must fail, never read as empty status. + with pytest.raises(AssertionError): + parse_rgw_replication_status(json.dumps({"error": "boom", "metadata": ""})) + with pytest.raises(AssertionError): + parse_rgw_replication_status("not json at all") + with pytest.raises(AssertionError): + parse_rgw_replication_status(json.dumps({"metadata": "not json either"})) + + +def test_rgw_data_sync_states(): + status = { + "data_sync": [ + {"source_zone": "us-west", "state": "local-unavailable"}, + {"source_zone": "us-archive", "state": "not-a-source"}, + ] + } + assert rgw_data_sync_states(status) == { + "us-west": "local-unavailable", + "us-archive": "not-a-source", + } + assert rgw_data_sync_states({}) == {} + assert rgw_data_sync_states({"data_sync": None}) == {} diff --git a/tests/robot/rgw-replication-status-test/rgw_replication_status.robot b/tests/robot/rgw-replication-status-test/rgw_replication_status.robot new file mode 100644 index 00000000..f7c7e3c7 --- /dev/null +++ b/tests/robot/rgw-replication-status-test/rgw_replication_status.robot @@ -0,0 +1,81 @@ +*** Settings *** +Documentation rgw-replication-status-test +... Exercises the /1.0/ops/replication/rgw/site status API on a single node by +... fabricating a two-zonegroup multisite topology with radosgw-admin: no second +... cluster, no imported remotes, no running sync. This covers the API routing, +... the FSM, PreFill's radosgw-admin reads and the brief rendering - everything +... except an actual peer comparison, which needs a second cluster. +... +... Every fabricated endpoint is a connection-refused address (127.0.0.1:9) on +... purpose: sync status reads against them fail instantly and surface as +... local-unavailable, whereas an unroutable address would block each read for +... radosgw-admin's full 300s curl timeout. +Resource ../resources/microceph_harness.resource +Suite Setup RGW Replication Status Suite Setup +Suite Teardown Teardown MicroCeph Environment +Test Tags single-node rgw replication api lxd integration + +*** Variables *** +${DEAD_ENDPOINT} http://127.0.0.1:9 + +*** Keywords *** +RGW Replication Status Suite Setup + Launch Outer Test VM vm_name=microceph-rgw-rep-vm + Copy Scripts To VM + Copy Snap To VM + Free Runner Disk + Install And Bootstrap MicroCeph + Run In VM And Check sudo microceph disk add loop,2G,3 300 + Create Fabricated Multisite Topology + +Create Fabricated Multisite Topology + [Documentation] One realm, two zonegroups, one cluster. Zonegroup us (the + ... realm's master) holds the local zone us-east plus the fabricated peers + ... us-west (a configured sync source) and us-archive (excluded from + ... us-east's sync_from). Zonegroup eu (non-master) holds eu-central and + ... eu-west for the cross-zonegroup metadata master case. + Run In VM And Check sudo microceph.radosgw-admin realm create --rgw-realm=verify --default 60 + Run In VM And Check sudo microceph.radosgw-admin zonegroup create --rgw-zonegroup=us --endpoints=${DEAD_ENDPOINT} --master --default 60 + Run In VM And Check sudo microceph.radosgw-admin zone create --rgw-zonegroup=us --rgw-zone=us-east --endpoints=${DEAD_ENDPOINT} --master --default 60 + Run In VM And Check sudo microceph.radosgw-admin zone create --rgw-zonegroup=us --rgw-zone=us-west --endpoints=${DEAD_ENDPOINT} 60 + Run In VM And Check sudo microceph.radosgw-admin zone create --rgw-zonegroup=us --rgw-zone=us-archive --endpoints=${DEAD_ENDPOINT} 60 + Run In VM And Check sudo microceph.radosgw-admin zone modify --rgw-zonegroup=us --rgw-zone=us-east --sync-from-all=false --sync-from=us-west 60 + Run In VM And Check sudo microceph.radosgw-admin zonegroup create --rgw-zonegroup=eu --endpoints=${DEAD_ENDPOINT} 60 + Run In VM And Check sudo microceph.radosgw-admin zone create --rgw-zonegroup=eu --rgw-zone=eu-central --endpoints=${DEAD_ENDPOINT} --master 60 + Run In VM And Check sudo microceph.radosgw-admin zone create --rgw-zonegroup=eu --rgw-zone=eu-west --endpoints=${DEAD_ENDPOINT} 60 + Run In VM And Check sudo microceph.radosgw-admin period update --commit 120 + +*** Test Cases *** +Site Status On The Metadata Master + [Documentation] us-east is the master zone of the realm's master zonegroup: + ... metadata sync reports master, the configured source us-west surfaces its + ... failed local sync read as local-unavailable, and us-archive - not named + ... in us-east's sync_from - is reported not-a-source without being queried. + ${status}= Get Rgw Replication Status + Should Be Equal ${status['realm']} verify + Should Be Equal ${status['zonegroup']} us + Should Be Equal ${status['zone']} us-east + Should Be True ${status['is_master_zone']} + Should Be Equal ${status['master_zone']} us-east + Length Should Be ${status['zones']} ${3} + Should Be Equal ${status['metadata_sync']['state']} master + ${data_states}= Rgw Data Sync States ${status} + Should Be Equal ${data_states['us-west']} local-unavailable + Should Be Equal ${data_states['us-archive']} not-a-source + +Site Status In A Non Master Zonegroup + [Documentation] With the cluster defaults switched to eu/eu-central - the + ... master of a NON-master zonegroup - is_master_zone must be false and the + ... metadata master must resolve across zonegroups to us-east via the realm + ... period. The never-started metadata sync and the eu-west data stream both + ... surface their failed local reads as local-unavailable. + Run In VM And Check sudo microceph.radosgw-admin zonegroup default --rgw-zonegroup=eu 60 + Run In VM And Check sudo microceph.radosgw-admin zone default --rgw-zone=eu-central 60 + ${status}= Get Rgw Replication Status + Should Be Equal ${status['zonegroup']} eu + Should Be Equal ${status['zone']} eu-central + Should Not Be True ${status['is_master_zone']} + Should Be Equal ${status['master_zone']} us-east + Should Be Equal ${status['metadata_sync']['state']} local-unavailable + ${data_states}= Rgw Data Sync States ${status} + Should Be Equal ${data_states['eu-west']} local-unavailable diff --git a/tests/robot/rgw-replication-test/rgw_replication_tests.robot b/tests/robot/rgw-replication-test/rgw_replication_tests.robot new file mode 100644 index 00000000..4ece84e5 --- /dev/null +++ b/tests/robot/rgw-replication-test/rgw_replication_tests.robot @@ -0,0 +1,115 @@ +*** Settings *** +Documentation rgw-replication-test +... Tests the RGW replication status API against a real multisite: two 2-node +... sites (sitea=wrk0/1, siteb=wrk2/3) with remotes exchanged, the realm +... configured manually with radosgw-admin (MicroCeph cannot enable RGW +... multisite itself yet), rgw running on both sides and sync live. This is +... the coverage the single-node rgw-replication-status-test cannot reach: +... peer log reads through imported remotes and real caught-up verdicts. +... +... The tests are an ordered ladder: multisite runs BEFORE any remotes are +... imported, so the same live streams first render peer-unavailable (local +... markers readable, peer log unreachable), then flip to caught-up once the +... remotes land. +Resource ../resources/microceph_harness.resource +Resource ../resources/replication.resource +Suite Setup RGW Replication Suite Setup +Suite Teardown Teardown MicroCeph Environment +Test Tags multi-node rgw replication remote lxd slow integration + +*** Variables *** +${REALM} microceph +${ZONEGROUP} microceph +# Fixed system-user keys for inter-zone sync auth; test-only credentials. +${SYNC_ACCESS} rgwreptestaccesskey1 +${SYNC_SECRET} rgwreptestsecretkey1 + +*** Keywords *** +RGW Replication Suite Setup + Provision Multinode VM microceph-rgwrep-vm ${OUTER_VM_DISK} public + Bootstrap Two Sites + Configure Rgw Multisite Manually + +Configure Rgw Multisite Manually + [Documentation] The standard squid manual multisite procedure with sitea as + ... the master zone: realm/zonegroup/zone plus the system user on sitea, then + ... realm pull and the secondary zone on siteb. Multisite is configured before + ... each site's rgw first starts, so no implicit "default" zone is ever + ... created. Zone names deliberately match the imported remote names - that + ... pairing is how the status API reaches a peer's sync logs. + Log To Console [rgw] Configuring multisite manually (sitea master, siteb secondary)... + ${sitea_ip}= Get Node Ip node-wrk0 + ${siteb_ip}= Get Node Ip node-wrk2 + Run In Container node-wrk0 microceph.radosgw-admin realm create --rgw-realm=${REALM} --default 60 + Run In Container node-wrk0 microceph.radosgw-admin zonegroup create --rgw-zonegroup=${ZONEGROUP} --endpoints=http://${sitea_ip}:80 --rgw-realm=${REALM} --master --default 60 + Run In Container node-wrk0 microceph.radosgw-admin zone create --rgw-zonegroup=${ZONEGROUP} --rgw-zone=sitea --endpoints=http://${sitea_ip}:80 --access-key=${SYNC_ACCESS} --secret=${SYNC_SECRET} --master --default 60 + Run In Container node-wrk0 microceph.radosgw-admin user create --uid=sync --display-name=sync --access-key=${SYNC_ACCESS} --secret=${SYNC_SECRET} --system 60 + Run In Container node-wrk0 microceph.radosgw-admin period update --commit 120 + Run In Container node-wrk0 microceph enable rgw 120 + Wait For Rgw Endpoint node-wrk2 http://${sitea_ip}:80 + Run In Container node-wrk2 microceph.radosgw-admin realm pull --url=http://${sitea_ip}:80 --access-key=${SYNC_ACCESS} --secret=${SYNC_SECRET} --default 120 + Run In Container node-wrk2 microceph.radosgw-admin zone create --rgw-zonegroup=${ZONEGROUP} --rgw-zone=siteb --endpoints=http://${siteb_ip}:80 --access-key=${SYNC_ACCESS} --secret=${SYNC_SECRET} --default 60 + Run In Container node-wrk2 microceph.radosgw-admin period update --commit 120 + Run In Container node-wrk2 microceph enable rgw 120 + Wait For Rgw Endpoint node-wrk0 http://${siteb_ip}:80 + +*** Test Cases *** +Status Without Imported Remotes + [Documentation] Multisite is live but no remotes are imported yet, so the + ... local sync markers are readable while every peer log is not: each + ... stream must render peer-unavailable with an empty remote - not + ... caught-up, behind, or local-unavailable. + Wait For Rgw Data Sync State node-wrk0 siteb peer-unavailable + ${status}= Get Rgw Replication Status In Container node-wrk0 + Should Be Equal ${status['metadata_sync']['state']} master + ${data_states}= Rgw Data Sync States ${status} + Should Be Equal ${data_states['siteb']} peer-unavailable + Wait For Rgw Metadata Sync State node-wrk2 peer-unavailable + ${status}= Get Rgw Replication Status In Container node-wrk2 + Should Be Equal ${status['metadata_sync']['remote']} ${EMPTY} + ${data_states}= Rgw Data Sync States ${status} + Should Be Equal ${data_states['sitea']} peer-unavailable + +Importing Remotes Enables Peer Comparisons + [Documentation] Exchanges tokens so each site can read the other's sync + ... logs; the following tests assert the same streams now compare for real. + Exchange Remote Site Tokens + Verify Remote Authentication On All Nodes + +Master Site Status + [Documentation] sitea is the metadata master: it syncs metadata from no one + ... and pulls data from siteb through the imported remote. caught-up here is + ... a real verdict - local markers compared against siteb's live datalog. + Wait For Rgw Data Sync State node-wrk0 siteb caught-up + ${status}= Get Rgw Replication Status In Container node-wrk0 + Should Be Equal ${status['realm']} ${REALM} + Should Be Equal ${status['zonegroup']} ${ZONEGROUP} + Should Be Equal ${status['zone']} sitea + Should Be True ${status['is_master_zone']} + Should Be Equal ${status['master_zone']} sitea + Length Should Be ${status['zones']} ${2} + Should Be Equal ${status['metadata_sync']['state']} master + ${data_states}= Rgw Data Sync States ${status} + Should Be Equal ${data_states['siteb']} caught-up + +Secondary Site Status + [Documentation] siteb syncs metadata from sitea and data from sitea, both + ... through the imported sitea remote, and must name sitea as the master. + Wait For Rgw Metadata Sync State node-wrk2 caught-up + Wait For Rgw Data Sync State node-wrk2 sitea caught-up + ${status}= Get Rgw Replication Status In Container node-wrk2 + Should Be Equal ${status['zone']} siteb + Should Not Be True ${status['is_master_zone']} + Should Be Equal ${status['master_zone']} sitea + Should Be Equal ${status['metadata_sync']['state']} caught-up + Should Be Equal ${status['metadata_sync']['remote']} sitea + ${data_states}= Rgw Data Sync States ${status} + Should Be Equal ${data_states['sitea']} caught-up + +Metadata Change Replicates And Status Recovers + [Documentation] A user created on the master must appear on the secondary via + ... live metadata sync, and the status must return to caught-up afterwards - + ... proving the verdict tracks a real stream rather than a vacuous one. + Run In Container node-wrk0 microceph.radosgw-admin user create --uid=repl-check --display-name=repl-check 60 + Wait For Rgw User In Container node-wrk2 repl-check + Wait For Rgw Metadata Sync State node-wrk2 caught-up