diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..8bd5c6ce3 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,9 @@ +* text=auto eol=lf +*.png -text +*.jpg -text +*.jpeg -text +*.gif -text +*.webp -text +*.pdf -text +*.zip -text +backend/crater -text diff --git a/.gitignore b/.gitignore index 83c85888e..e6c7d6bb7 100644 --- a/.gitignore +++ b/.gitignore @@ -79,3 +79,47 @@ kubeconfig* # Claude Code .claude/ +storage/etc/config.yaml +.gocache/ +.gomodcache/ + +# Local build/test scratch +.gocache*/ +.gotmp*/ +backend/crater + +# Python cache +__pycache__/ +*.pyc +*.pyo + +# Temporary experiment artifacts +tmp_* +rewrite_section3*.py +storage-governance-datasets.zip +storage-governance-qwen25-7b.zip + +# Editor/tool backup files with random numeric suffixes +*.go.[0-9]* + +# Local workspace-only directories and assets +/.claude/ +/.gocache-test/ +/.gopath/ +/_external/ +/datasets/ +/models/ +/models_compare/ +/offline-tools/ +/offline-tools.zip +/graduation-thesis-latex/ + +# Local document/build logs +/texput.log + +# Local documentation drafts +/docs/zh-CN/framework-architecture-and-workflow.md +/docs/zh-CN/storage-governance-requirements/ +/docs/zh-CN/storage-management.md +/docs/zh-CN/storage-metadata-index-policy-engine.md +/docs/zh-CN/storage-tiering-policy-satisfier.md diff --git a/.vscode/launch.json b/.vscode/launch.json index 4bc15ced3..7e466d4f2 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -12,9 +12,11 @@ "program": "${workspaceFolder}/backend/cmd/crater/main.go", "cwd": "${workspaceFolder}/backend", "env": { + "GOCACHE": "${workspaceFolder}/backend/.gocache", + "GOTOOLCHAIN": "go1.25.4", "KUBECONFIG": "${workspaceFolder}/backend/kubeconfig", "NO_PROXY": "k8s.cluster.master" } } ] -} \ No newline at end of file +} diff --git a/backend/.vscode/launch.json b/backend/.vscode/launch.json index 113ff42db..539f66417 100644 --- a/backend/.vscode/launch.json +++ b/backend/.vscode/launch.json @@ -16,6 +16,8 @@ "${workspaceFolder}/etc/debug-config.yaml" ], "env": { + "GOCACHE": "${workspaceFolder}/.gocache", + "GOTOOLCHAIN": "go1.25.4", "KUBECONFIG": "${env:HOME}/.kube/config", "NO_PROXY": "k8s.cluster.master" } @@ -32,6 +34,8 @@ "${workspaceFolder}/etc/debug-config-actgpu.yaml" ], "env": { + "GOCACHE": "${workspaceFolder}/.gocache", + "GOTOOLCHAIN": "go1.25.4", "KUBECONFIG": "${workspaceFolder}/kubeconfig_act" } }, @@ -47,6 +51,8 @@ "${workspaceFolder}/etc/debug-config-little.yaml" ], "env": { + "GOCACHE": "${workspaceFolder}/.gocache", + "GOTOOLCHAIN": "go1.25.4", "KUBECONFIG": "${workspaceFolder}/kubeconfig_little" } }, @@ -62,8 +68,10 @@ "${workspaceFolder}/etc/debug-config-ali.yaml" ], "env": { + "GOCACHE": "${workspaceFolder}/.gocache", + "GOTOOLCHAIN": "go1.25.4", "KUBECONFIG": "${workspaceFolder}/kubeconfig_ali" } } ] -} \ No newline at end of file +} diff --git a/backend/cmd/crater/helper/config.go b/backend/cmd/crater/helper/config.go index 7b548f9d1..8868f4036 100644 --- a/backend/cmd/crater/helper/config.go +++ b/backend/cmd/crater/helper/config.go @@ -106,6 +106,7 @@ func (ci *ConfigInitializer) SetupManagerDependencies(registerConfig *handler.Re registerConfig.CronJobManager = cronjob.NewCronJobManager( registerConfig.Client, registerConfig.KubeClient, + registerConfig.KubeConfig, registerConfig.PrometheusClient, registerConfig.GpuAnalysisService, registerConfig.BillingService, diff --git a/backend/cmd/gorm-gen/models/migrate.go b/backend/cmd/gorm-gen/models/migrate.go index f14140496..97dd94f85 100644 --- a/backend/cmd/gorm-gen/models/migrate.go +++ b/backend/cmd/gorm-gen/models/migrate.go @@ -1573,6 +1573,9 @@ func main() { }, }, modelDownloadSubmissionMigration(), + storageGovernanceMigration(), + storageGovernanceAutomationCleanupMigration(), + storageUsageManualRefreshMigration(), }) m.InitSchema(func(tx *gorm.DB) error { @@ -1609,6 +1612,9 @@ func main() { &model.PrequeueConfig{}, &model.QueueQuotaLimit{}, &model.UserBanRecord{}, + &model.UserSpaceSize{}, + &model.TenantUsageHistory{}, + &model.StorageDecisionRecord{}, ) if err != nil { return err @@ -1735,6 +1741,169 @@ func main() { } } +func storageGovernanceMigration() *gormigrate.Migration { + type userStorageColumns struct { + SpaceQuota int64 `gorm:"type:bigint;default:-1"` + OriginalSpaceQuota *int64 `gorm:"type:bigint;default:null"` + JobsFrozen bool `gorm:"type:boolean;default:false"` + ShrinkStage *string `gorm:"type:varchar(64);default:null"` + ShrinkStageUpdatedAt *time.Time `gorm:"default:null"` + } + + return &gormigrate.Migration{ + ID: "202608020001", + Migrate: func(tx *gorm.DB) error { + if err := tx.AutoMigrate( + &model.UserSpaceSize{}, + &model.TenantUsageHistory{}, + &model.StorageDecisionRecord{}, + ); err != nil { + return err + } + + migrator := tx.Table("users").Migrator() + for _, field := range []string{ + "SpaceQuota", + "OriginalSpaceQuota", + "JobsFrozen", + "ShrinkStage", + "ShrinkStageUpdatedAt", + } { + if migrator.HasColumn(&userStorageColumns{}, field) { + continue + } + if err := migrator.AddColumn(&userStorageColumns{}, field); err != nil { + return err + } + } + + jobs := []model.CronJobConfig{ + { + Name: "update-user-space-size", + Type: model.CronJobTypePatrolFunc, + Spec: "*/30 * * * *", + Status: model.CronJobConfigStatusIdle, + Config: datatypes.JSON(`{}`), + EntryID: -1, + }, + } + for i := range jobs { + if err := tx.Where("name = ?", jobs[i].Name).FirstOrCreate(&jobs[i]).Error; err != nil { + return err + } + } + + return tx.Table("cron_job_configs"). + Where("name IN ?", []string{ + "refresh-public-storage-index-baseline", + "refresh-user-storage-index-daily", + "analyze-storage-alerts", + "auto-shrink-storage-expansions", + }). + Delete(nil).Error + }, + Rollback: func(tx *gorm.DB) error { + if err := tx.Table("cron_job_configs"). + Where("name IN ?", []string{ + "update-user-space-size", + }). + Delete(nil).Error; err != nil { + return err + } + + migrator := tx.Table("users").Migrator() + for _, field := range []string{ + "ShrinkStageUpdatedAt", + "ShrinkStage", + "JobsFrozen", + "OriginalSpaceQuota", + "SpaceQuota", + } { + if migrator.HasColumn(&userStorageColumns{}, field) { + if err := migrator.DropColumn(&userStorageColumns{}, field); err != nil { + return err + } + } + } + + return tx.Migrator().DropTable( + &model.StorageDecisionRecord{}, + &model.TenantUsageHistory{}, + &model.UserSpaceSize{}, + ) + }, + } +} + +func storageGovernanceAutomationCleanupMigration() *gormigrate.Migration { + jobNames := []string{ + "update-user-space-size", + "analyze-storage-alerts", + "auto-shrink-storage-expansions", + } + + return &gormigrate.Migration{ + ID: "202608080001", + Migrate: func(tx *gorm.DB) error { + return tx.Table("cron_job_configs").Where("name IN ?", jobNames).Delete(nil).Error + }, + Rollback: func(tx *gorm.DB) error { + jobs := []model.CronJobConfig{ + { + Name: "update-user-space-size", + Type: model.CronJobTypePatrolFunc, + Spec: "*/30 * * * *", + Status: model.CronJobConfigStatusIdle, + Config: datatypes.JSON(`{}`), + EntryID: -1, + }, + { + Name: "analyze-storage-alerts", + Type: model.CronJobTypePatrolFunc, + Spec: "*/30 * * * *", + Status: model.CronJobConfigStatusSuspended, + Config: datatypes.JSON(`{}`), + EntryID: -1, + }, + { + Name: "auto-shrink-storage-expansions", + Type: model.CronJobTypePatrolFunc, + Spec: "0 * * * *", + Status: model.CronJobConfigStatusSuspended, + Config: datatypes.JSON(`{}`), + EntryID: -1, + }, + } + for i := range jobs { + if err := tx.Where("name = ?", jobs[i].Name).FirstOrCreate(&jobs[i]).Error; err != nil { + return err + } + } + return nil + }, + } +} + +func storageUsageManualRefreshMigration() *gormigrate.Migration { + return &gormigrate.Migration{ + ID: "202608090001", + Migrate: func(tx *gorm.DB) error { + return tx.Table("cron_job_configs").Where("name = ?", "update-user-space-size").Delete(nil).Error + }, + Rollback: func(tx *gorm.DB) error { + job := model.CronJobConfig{ + Name: "update-user-space-size", + Type: model.CronJobTypePatrolFunc, + Spec: "*/30 * * * *", + Status: model.CronJobConfigStatusIdle, + Config: datatypes.JSON(`{}`), + EntryID: -1, + } + return tx.Where("name = ?", job.Name).FirstOrCreate(&job).Error + }, + } +} + type jobListIndex struct { create string drop string diff --git a/backend/cmd/storage-server/main.go b/backend/cmd/storage-server/main.go index 9c2bcaada..4001a84f3 100644 --- a/backend/cmd/storage-server/main.go +++ b/backend/cmd/storage-server/main.go @@ -11,6 +11,7 @@ import ( "github.com/raids-lab/crater/dao/query" "github.com/raids-lab/crater/internal/storage" "github.com/raids-lab/crater/pkg/config" + "github.com/raids-lab/crater/pkg/storagequota" ) var ( @@ -20,6 +21,11 @@ var ( BuildTime string ) +const ( + storageModeFull = "full" + storageModeQuotaAgent = "quota-agent" +) + func initVersionInfo() { if AppVersion == "" { AppVersion = "dev-local" @@ -51,6 +57,7 @@ func normalizePort(port string) string { return ":" + port } +//nolint:gocyclo // Startup validates mode-specific dependencies in one linear flow. func main() { initVersionInfo() @@ -63,8 +70,21 @@ func main() { } } - _ = config.GetConfig() - query.SetDefault(query.GetDB()) + mode := strings.ToLower(firstNonEmptyEnv("CRATER_STORAGE_MODE")) + if mode == "" { + mode = storageModeFull + } + if mode != storageModeFull && mode != storageModeQuotaAgent { + klog.Fatalf("unsupported storage-server mode %q; expected full or quota-agent", mode) + } + hasInternalCredential := strings.TrimSpace(os.Getenv(storagequota.InternalTokenEnv)) != "" || + strings.TrimSpace(os.Getenv(storagequota.InternalSecretEnv)) != "" + if mode == storageModeFull || !hasInternalCredential { + _ = config.GetConfig() + } + if mode == storageModeFull { + query.SetDefault(query.GetDB()) + } port := firstNonEmptyEnv("CRATER_STORAGE_PORT", "PORT") if port == "" { @@ -79,14 +99,18 @@ func main() { klog.Fatalf("failed to create storage root directory %s: %v", rootDir, err) } storage.SetRootDir(rootDir) - go storage.StartCheckSpace() r := gin.Default() - storage.RegisterRoutes(r) + if mode == storageModeQuotaAgent { + storage.RegisterQuotaRoutes(r) + } else { + go storage.StartCheckSpace() + storage.RegisterRoutes(r) + } addr := normalizePort(port) - klog.Infof("storage-server starting on %s (version=%s, commit=%s, buildType=%s, buildTime=%s)", - addr, AppVersion, CommitSHA, BuildType, BuildTime) + klog.Infof("storage-server starting on %s (mode=%s, version=%s, commit=%s, buildType=%s, buildTime=%s)", + addr, mode, AppVersion, CommitSHA, BuildType, BuildTime) if err := r.Run(addr); err != nil { klog.Fatalf("failed to run storage-server: %v", err) } diff --git a/backend/dao/model/storage_decision.go b/backend/dao/model/storage_decision.go new file mode 100644 index 000000000..a8fba911b --- /dev/null +++ b/backend/dao/model/storage_decision.go @@ -0,0 +1,62 @@ +package model + +import ( + "time" + + "gorm.io/datatypes" + "gorm.io/gorm" +) + +type StorageDecisionStatus string + +const ( + StorageDecisionStatusPending StorageDecisionStatus = "pending" + StorageDecisionStatusRunning StorageDecisionStatus = "running" + StorageDecisionStatusDone StorageDecisionStatus = "done" + StorageDecisionStatusError StorageDecisionStatus = "error" +) + +type StorageDecisionSource string + +const ( + StorageDecisionSourceManual StorageDecisionSource = "manual" + StorageDecisionSourcePatrol StorageDecisionSource = "patrol" + StorageDecisionSourceReplay StorageDecisionSource = "replay" +) + +// StorageDecisionRecord stores the full decision trail of a storage governance run. +// It keeps the input snapshot, the raw LLM proposal, and the final decision after +// applying platform safety constraints so the result can be audited and replayed. +type StorageDecisionRecord struct { + gorm.Model + JobID string `gorm:"type:varchar(64);not null;uniqueIndex;comment:决策任务ID" json:"jobId"` + UserID uint `gorm:"index;comment:用户ID" json:"userId"` + Username string `gorm:"type:varchar(64);not null;index;comment:用户名" json:"username"` + Source StorageDecisionSource `gorm:"type:varchar(32);not null;index;comment:触发来源" json:"source"` + Status StorageDecisionStatus `gorm:"type:varchar(32);not null;index;default:pending;comment:决策状态" json:"status"` + TriggerReason string `gorm:"type:text;comment:触发原因" json:"triggerReason"` + + Snapshot datatypes.JSON `gorm:"type:jsonb;comment:决策时的输入快照" json:"snapshot"` + RawDecision datatypes.JSON `gorm:"type:jsonb;comment:LLM原始决策" json:"rawDecision"` + FinalDecision datatypes.JSON `gorm:"type:jsonb;comment:约束校验后的最终决策" json:"finalDecision"` + ConstraintResult datatypes.JSON `gorm:"type:jsonb;comment:安全约束评估结果" json:"constraintResult"` + + RawAllowExpand bool `gorm:"not null;default:false;comment:LLM原始是否允许扩容" json:"rawAllowExpand"` + RawExpandBytes int64 `gorm:"not null;default:0;comment:LLM原始建议扩容量" json:"rawExpandBytes"` + RawFreezeNewJobs bool `gorm:"not null;default:false;comment:LLM原始是否冻结新作业" json:"rawFreezeNewJobs"` + FinalAllowExpand bool `gorm:"not null;default:false;comment:最终是否允许扩容" json:"finalAllowExpand"` + FinalExpandBytes int64 `gorm:"not null;default:0;comment:最终扩容量" json:"finalExpandBytes"` + FinalFreezeNewJobs bool `gorm:"not null;default:false;comment:最终是否冻结新作业" json:"finalFreezeNewJobs"` + ConstraintAdjusted bool `gorm:"not null;default:false;comment:安全约束是否调整了决策" json:"constraintAdjusted"` + ConstraintBlocked bool `gorm:"not null;default:false;comment:安全约束是否阻断了扩容" json:"constraintBlocked"` + AppliedAction string `gorm:"type:varchar(64);comment:最终动作摘要" json:"appliedAction"` + ErrorMessage string `gorm:"type:text;comment:错误信息" json:"errorMessage"` + StartedAt *time.Time `gorm:"comment:开始时间" json:"startedAt"` + FinishedAt *time.Time `gorm:"comment:结束时间" json:"finishedAt"` + LatencyMs int64 `gorm:"not null;default:0;comment:决策耗时毫秒" json:"latencyMs"` + ConstraintVersion string `gorm:"type:varchar(64);comment:约束策略版本" json:"constraintVersion"` +} + +func (StorageDecisionRecord) TableName() string { + return "storage_decision_records" +} diff --git a/backend/dao/model/system_config.go b/backend/dao/model/system_config.go index 3224e005c..4860deca0 100644 --- a/backend/dao/model/system_config.go +++ b/backend/dao/model/system_config.go @@ -1,16 +1,14 @@ -// 请将此文件保存为 dao/model/system_config.go - package model -// SystemConfig 用于存储系统级别的键值对配置 +// SystemConfig stores system-wide key-value configuration. type SystemConfig struct { - Key string `gorm:"primarykey;size:100;comment:配置项的键"` - Value string `gorm:"type:text;comment:配置项的值"` + Key string `gorm:"primarykey;size:100;comment:配置项键"` + Value string `gorm:"type:text;comment:配置项值"` } const ( - // LLM 相关配置键 - ConfigKeyLLMBaseURL = "LLM_API_BASE_URL" // 例如: https://api.openai.com/v1 + // Generic LLM configuration keys. + ConfigKeyLLMBaseURL = "LLM_API_BASE_URL" // e.g. https://api.openai.com/v1 ConfigKeyLLMAPIKey = "LLM_API_KEY" // #nosec G101 ConfigKeyLLMModelName = "LLM_MODEL_NAME" @@ -40,13 +38,25 @@ const ( ConfigKeyModelDownloadBandwidth = "POD_BANDWIDTH_MODEL_DOWNLOAD" ConfigKeyJobIngressBandwidth = "POD_BANDWIDTH_JOB_INGRESS" ConfigKeyJobEgressBandwidth = "POD_BANDWIDTH_JOB_EGRESS" + + // Storage decision keys. + ConfigKeyStorageDecisionMode = "STORAGE_DECISION_MODE" + ConfigKeyStorageDecisionConfigSource = "STORAGE_DECISION_CONFIG_SOURCE" + ConfigKeyStorageDirectModelBaseURL = "STORAGE_DIRECT_MODEL_BASE_URL" + ConfigKeyStorageDirectModelAPIKey = "STORAGE_DIRECT_MODEL_API_KEY" // #nosec G101 + ConfigKeyStorageDirectModelName = "STORAGE_DIRECT_MODEL_NAME" ) -// DefaultConfigKeys 定义了系统启动时必须存在的键 +// DefaultConfigKeys defines keys that must exist after startup. var DefaultConfigKeys = []string{ ConfigKeyLLMBaseURL, ConfigKeyLLMAPIKey, ConfigKeyLLMModelName, + ConfigKeyStorageDecisionMode, + ConfigKeyStorageDecisionConfigSource, + ConfigKeyStorageDirectModelBaseURL, + ConfigKeyStorageDirectModelAPIKey, + ConfigKeyStorageDirectModelName, ConfigKeyEnableGpuAnalysis, ConfigKeyEnableBillingFeature, ConfigKeyEnableBillingActive, diff --git a/backend/dao/model/user.go b/backend/dao/model/user.go index e6fba8b49..dfacc7a12 100644 --- a/backend/dao/model/user.go +++ b/backend/dao/model/user.go @@ -59,17 +59,22 @@ func (r UserBanRestrictions) Any() bool { // User is the basic entity of the system type User struct { gorm.Model - Name string `gorm:"uniqueIndex;type:varchar(64);not null;comment:用户名"` - Nickname string `gorm:"type:varchar(64);comment:昵称"` - Password *string `gorm:"type:varchar(256);comment:密码"` - Role Role `gorm:"index:role;not null;comment:用户在平台的角色 (guest, user, admin)"` - Status Status `gorm:"index:status;not null;comment:用户状态 (pending, active, inactive)"` - Space string `gorm:"uniqueIndex;type:varchar(256);not null;comment:用户空间绝对路径"` - ImageQuota int64 `gorm:"type:bigint;default:-1;comment:用户在镜像仓库的配额"` - ExtraBalance int64 `gorm:"type:bigint;not null;default:0;comment:用户额外点数余额(内部微点, 充值/奖励)"` - LastEmailVerifiedAt *time.Time `gorm:"comment:最后一次邮箱验证时间"` - BannedTimestamp *time.Time `gorm:"index;comment:用户封禁截止时间，晚于当前时间表示封禁中"` - BanRestrictions datatypes.JSONType[UserBanRestrictions] `gorm:"type:jsonb;not null;default:'{}';comment:最近一次封禁配置的限制内容，仅在封禁截止时间有效时生效"` + Name string `gorm:"uniqueIndex;type:varchar(64);not null;comment:用户名"` + Nickname string `gorm:"type:varchar(64);comment:昵称"` + Password *string `gorm:"type:varchar(256);comment:密码"` + Role Role `gorm:"index:role;not null;comment:用户在平台的角色 (guest, user, admin)"` + Status Status `gorm:"index:status;not null;comment:用户状态 (pending, active, inactive)"` + Space string `gorm:"uniqueIndex;type:varchar(256);not null;comment:用户空间绝对路径"` + ImageQuota int64 `gorm:"type:bigint;default:-1;comment:用户在镜像仓库的配额"` + SpaceQuota int64 `gorm:"type:bigint;default:-1;comment:用户存储空间配额"` + OriginalSpaceQuota *int64 `gorm:"type:bigint;default:null;comment:临时扩容前的用户存储空间配额"` + JobsFrozen bool `gorm:"type:boolean;default:false;comment:是否因存储配额冻结作业"` + ShrinkStage *string `gorm:"type:varchar(64);default:null;comment:存储配额缩容阶段"` + ShrinkStageUpdatedAt *time.Time `gorm:"default:null;comment:存储配额缩容阶段更新时间"` + ExtraBalance int64 `gorm:"type:bigint;not null;default:0;comment:用户额外点数余额(内部微点, 充值/奖励)"` + LastEmailVerifiedAt *time.Time `gorm:"comment:最后一次邮箱验证时间"` + BannedTimestamp *time.Time `gorm:"index;comment:用户封禁截止时间，晚于当前时间表示封禁中"` + BanRestrictions datatypes.JSONType[UserBanRestrictions] `gorm:"type:jsonb;not null;default:'{}';comment:最近一次封禁配置的限制内容，仅在封禁截止时间有效时生效"` Attributes datatypes.JSONType[UserAttribute] `gorm:"comment:用户的额外属性 (昵称、邮箱、电话、头像等)"` UserAccounts []UserAccount diff --git a/backend/dao/model/user_space_size.go b/backend/dao/model/user_space_size.go new file mode 100644 index 000000000..b6c469bc8 --- /dev/null +++ b/backend/dao/model/user_space_size.go @@ -0,0 +1,26 @@ +package model + +import ( + "time" +) + +// UserSpaceSize 用户空间大小模型 +type UserSpaceSize struct { + ID uint `gorm:"primaryKey" json:"id"` + UserID uint `gorm:"index" json:"user_id"` + User User `gorm:"foreignKey:UserID" json:"user"` + Username string `gorm:"size:64;not null;uniqueIndex" json:"username"` + Size int64 `json:"size"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +// TenantUsageHistory 租户存储使用历史 +type TenantUsageHistory struct { + ID uint `gorm:"primaryKey" json:"id"` + TenantID uint `gorm:"index" json:"tenant_id"` + UsageBytes int64 `json:"usage_bytes"` + RecordedAt time.Time `json:"recorded_at"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} diff --git a/backend/dao/query/users.gen.go b/backend/dao/query/users.gen.go index e8acc7dab..517455ba3 100644 --- a/backend/dao/query/users.gen.go +++ b/backend/dao/query/users.gen.go @@ -39,6 +39,11 @@ func newUser(db *gorm.DB, opts ...gen.DOOption) user { _user.Status = field.NewUint8(tableName, "status") _user.Space = field.NewString(tableName, "space") _user.ImageQuota = field.NewInt64(tableName, "image_quota") + _user.SpaceQuota = field.NewInt64(tableName, "space_quota") + _user.OriginalSpaceQuota = field.NewInt64(tableName, "original_space_quota") + _user.JobsFrozen = field.NewBool(tableName, "jobs_frozen") + _user.ShrinkStage = field.NewString(tableName, "shrink_stage") + _user.ShrinkStageUpdatedAt = field.NewTime(tableName, "shrink_stage_updated_at") _user.ExtraBalance = field.NewInt64(tableName, "extra_balance") _user.LastEmailVerifiedAt = field.NewTime(tableName, "last_email_verified_at") _user.BannedTimestamp = field.NewTime(tableName, "banned_timestamp") @@ -64,24 +69,29 @@ func newUser(db *gorm.DB, opts ...gen.DOOption) user { type user struct { userDo userDo - ALL field.Asterisk - ID field.Uint - CreatedAt field.Time - UpdatedAt field.Time - DeletedAt field.Field - Name field.String // 用户名 - Nickname field.String // 昵称 - Password field.String // 密码 - Role field.Uint8 // 用户在平台的角色 (guest, user, admin) - Status field.Uint8 // 用户状态 (pending, active, inactive) - Space field.String // 用户空间绝对路径 - ImageQuota field.Int64 // 用户在镜像仓库的配额 - ExtraBalance field.Int64 // 用户额外点数余额(内部微点, 充值/奖励) - LastEmailVerifiedAt field.Time // 最后一次邮箱验证时间 - BannedTimestamp field.Time // 用户封禁截止时间，晚于当前时间表示封禁中 - BanRestrictions field.Field // 最近一次封禁配置的限制内容，仅在封禁截止时间有效时生效 - Attributes field.Field // 用户的额外属性 (昵称、邮箱、电话、头像等) - UserAccounts userHasManyUserAccounts + ALL field.Asterisk + ID field.Uint + CreatedAt field.Time + UpdatedAt field.Time + DeletedAt field.Field + Name field.String // 用户名 + Nickname field.String // 昵称 + Password field.String // 密码 + Role field.Uint8 // 用户在平台的角色 (guest, user, admin) + Status field.Uint8 // 用户状态 (pending, active, inactive) + Space field.String // 用户空间绝对路径 + ImageQuota field.Int64 // 用户在镜像仓库的配额 + SpaceQuota field.Int64 // 用户存储空间配额 + OriginalSpaceQuota field.Int64 // 临时扩容前的用户存储空间配额 + JobsFrozen field.Bool // 是否因存储配额冻结作业 + ShrinkStage field.String // 存储配额缩容阶段 + ShrinkStageUpdatedAt field.Time // 存储配额缩容阶段更新时间 + ExtraBalance field.Int64 // 用户额外点数余额(内部微点, 充值/奖励) + LastEmailVerifiedAt field.Time // 最后一次邮箱验证时间 + BannedTimestamp field.Time // 用户封禁截止时间，晚于当前时间表示封禁中 + BanRestrictions field.Field // 最近一次封禁配置的限制内容，仅在封禁截止时间有效时生效 + Attributes field.Field // 用户的额外属性 (昵称、邮箱、电话、头像等) + UserAccounts userHasManyUserAccounts UserDatasets userHasManyUserDatasets @@ -111,6 +121,11 @@ func (u *user) updateTableName(table string) *user { u.Status = field.NewUint8(table, "status") u.Space = field.NewString(table, "space") u.ImageQuota = field.NewInt64(table, "image_quota") + u.SpaceQuota = field.NewInt64(table, "space_quota") + u.OriginalSpaceQuota = field.NewInt64(table, "original_space_quota") + u.JobsFrozen = field.NewBool(table, "jobs_frozen") + u.ShrinkStage = field.NewString(table, "shrink_stage") + u.ShrinkStageUpdatedAt = field.NewTime(table, "shrink_stage_updated_at") u.ExtraBalance = field.NewInt64(table, "extra_balance") u.LastEmailVerifiedAt = field.NewTime(table, "last_email_verified_at") u.BannedTimestamp = field.NewTime(table, "banned_timestamp") @@ -140,7 +155,7 @@ func (u *user) GetFieldByName(fieldName string) (field.OrderExpr, bool) { } func (u *user) fillFieldMap() { - u.fieldMap = make(map[string]field.Expr, 18) + u.fieldMap = make(map[string]field.Expr, 23) u.fieldMap["id"] = u.ID u.fieldMap["created_at"] = u.CreatedAt u.fieldMap["updated_at"] = u.UpdatedAt @@ -152,6 +167,11 @@ func (u *user) fillFieldMap() { u.fieldMap["status"] = u.Status u.fieldMap["space"] = u.Space u.fieldMap["image_quota"] = u.ImageQuota + u.fieldMap["space_quota"] = u.SpaceQuota + u.fieldMap["original_space_quota"] = u.OriginalSpaceQuota + u.fieldMap["jobs_frozen"] = u.JobsFrozen + u.fieldMap["shrink_stage"] = u.ShrinkStage + u.fieldMap["shrink_stage_updated_at"] = u.ShrinkStageUpdatedAt u.fieldMap["extra_balance"] = u.ExtraBalance u.fieldMap["last_email_verified_at"] = u.LastEmailVerifiedAt u.fieldMap["banned_timestamp"] = u.BannedTimestamp diff --git a/backend/docs/docs.go b/backend/docs/docs.go index 7c7ca7a29..505cac086 100644 --- a/backend/docs/docs.go +++ b/backend/docs/docs.go @@ -3229,6 +3229,174 @@ const docTemplate = `{ } } }, + "/v1/admin/storage/capabilities": { + "get": { + "security": [ + { + "Bearer": [] + } + ], + "description": "Detect whether the configured storage supports CephFS usage and quota operations", + "produces": [ + "application/json" + ], + "tags": [ + "Storage" + ], + "summary": "Get storage quota capabilities", + "responses": { + "200": { + "description": "Success", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-internal_handler_StorageCapabilities" + } + } + } + } + }, + "/v1/admin/storage/user-spaces": { + "get": { + "security": [ + { + "Bearer": [] + } + ], + "description": "Get the size of all user spaces from database", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Storage" + ], + "summary": "Get all user space sizes", + "parameters": [ + { + "type": "integer", + "description": "Page number", + "name": "page", + "in": "query" + }, + { + "type": "integer", + "description": "Page size", + "name": "pageSize", + "in": "query" + } + ], + "responses": { + "200": { + "description": "Success", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any" + } + }, + "500": { + "description": "Other errors", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any" + } + } + } + } + }, + "/v1/admin/storage/user-spaces/refresh": { + "post": { + "security": [ + { + "Bearer": [] + } + ], + "description": "Read current CephFS usage for every user directory and update the usage cache", + "produces": [ + "application/json" + ], + "tags": [ + "Storage" + ], + "summary": "Refresh all user space usage", + "responses": { + "200": { + "description": "Success", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-github_com_raids-lab_crater_pkg_patrol_StorageUsageRefreshResult" + } + }, + "500": { + "description": "Other errors", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any" + } + } + } + } + }, + "/v1/admin/storage/user-spaces/{user}/quota": { + "put": { + "security": [ + { + "Bearer": [] + } + ], + "description": "Set the space quota for a user", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Storage" + ], + "summary": "Set user space quota", + "parameters": [ + { + "type": "string", + "description": "Username", + "name": "user", + "in": "path", + "required": true + }, + { + "description": "Space quota request", + "name": "quota", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/internal_handler.SetUserSpaceQuotaRequest" + } + } + ], + "responses": { + "200": { + "description": "Success", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any" + } + }, + "400": { + "description": "Request parameter error", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any" + } + }, + "404": { + "description": "User not found", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any" + } + }, + "500": { + "description": "Other errors", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any" + } + } + } + } + }, "/v1/admin/system-config/gpu-analysis": { "get": { "security": [ @@ -3333,7 +3501,7 @@ const docTemplate = `{ "Bearer": [] } ], - "description": "更新 LLM 的连接信息。如果 validate 为 true，会尝试连接 /check 接口，失败则不保存。", + "description": "更新 LLM 的连接信息。如果 validate 为 true，会尝试连接 /models 接口，失败则不保存。", "consumes": [ "application/json" ], @@ -8434,6 +8602,57 @@ const docTemplate = `{ } } }, + "/v1/operations/cronjob/execute": { + "post": { + "security": [ + { + "Bearer": [] + } + ], + "description": "Execute a patrol job immediately", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Operations" + ], + "summary": "Execute patrol job", + "parameters": [ + { + "description": "Job name", + "name": "jobName", + "in": "body", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any" + } + }, + "400": { + "description": "Request parameter error", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any" + } + }, + "500": { + "description": "Other errors", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any" + } + } + } + } + }, "/v1/operations/keep/{name}": { "put": { "description": "set KeepWhenLowResourceUsage of the job to the opposite value", @@ -8763,6 +8982,111 @@ const docTemplate = `{ } } }, + "/v1/storage/capabilities": { + "get": { + "security": [ + { + "Bearer": [] + } + ], + "description": "Detect whether the configured storage supports CephFS usage and quota operations", + "produces": [ + "application/json" + ], + "tags": [ + "Storage" + ], + "summary": "Get storage quota capabilities", + "responses": { + "200": { + "description": "Success", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-internal_handler_StorageCapabilities" + } + } + } + } + }, + "/v1/storage/dirsize/{path}": { + "get": { + "security": [ + { + "Bearer": [] + } + ], + "description": "Get the size of a directory in CephFS using getfattr command", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Storage" + ], + "summary": "Get directory size in CephFS", + "parameters": [ + { + "type": "string", + "description": "Directory path", + "name": "path", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "Success", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any" + } + }, + "400": { + "description": "Request parameter error", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any" + } + }, + "500": { + "description": "Other errors", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any" + } + } + } + } + }, + "/v1/storage/my-quota": { + "get": { + "security": [ + { + "Bearer": [] + } + ], + "description": "Get the storage quota for the currently authenticated user", + "produces": [ + "application/json" + ], + "tags": [ + "Storage" + ], + "summary": "Get current user's storage quota", + "responses": { + "200": { + "description": "Success", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any" + } + }, + "500": { + "description": "Other errors", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any" + } + } + } + } + }, "/v1/system-config/model-download-limit": { "get": { "security": [ @@ -11095,6 +11419,21 @@ const docTemplate = `{ } } }, + "github_com_raids-lab_crater_internal_resputil.Response-github_com_raids-lab_crater_pkg_patrol_StorageUsageRefreshResult": { + "type": "object", + "properties": { + "code": { + "description": "依然保持 int (ErrorCode) 类型", + "type": "integer" + }, + "data": { + "$ref": "#/definitions/github_com_raids-lab_crater_pkg_patrol.StorageUsageRefreshResult" + }, + "msg": { + "type": "string" + } + } + }, "github_com_raids-lab_crater_internal_resputil.Response-internal_handler_AdjustUserExtraBalanceResp": { "type": "object", "properties": { @@ -11440,6 +11779,21 @@ const docTemplate = `{ } } }, + "github_com_raids-lab_crater_internal_resputil.Response-internal_handler_StorageCapabilities": { + "type": "object", + "properties": { + "code": { + "description": "依然保持 int (ErrorCode) 类型", + "type": "integer" + }, + "data": { + "$ref": "#/definitions/internal_handler.StorageCapabilities" + }, + "msg": { + "type": "string" + } + } + }, "github_com_raids-lab_crater_internal_resputil.Response-internal_handler_TokenReq": { "type": "object", "properties": { @@ -11932,6 +12286,20 @@ const docTemplate = `{ } } }, + "github_com_raids-lab_crater_pkg_patrol.StorageUsageRefreshResult": { + "type": "object", + "properties": { + "failed": { + "type": "integer" + }, + "refreshed_at": { + "type": "string" + }, + "updated": { + "type": "integer" + } + } + }, "gorm.DeletedAt": { "type": "object", "properties": { @@ -12165,6 +12533,38 @@ const docTemplate = `{ } } }, + "internal_handler.AutoScaleRequest": { + "type": "object", + "required": [ + "max_quota", + "min_quota", + "scale_down_ratio", + "scale_up_ratio" + ], + "properties": { + "max_quota": { + "description": "最大配额，-1 表示无限制", + "type": "integer", + "minimum": -1 + }, + "min_quota": { + "description": "最小配额，-1 表示无限制", + "type": "integer", + "minimum": -1 + }, + "scale_down_ratio": { + "description": "缩容比例，如 0.8 表示缩容到当前使用的 0.8 倍", + "type": "number", + "maximum": 1, + "minimum": 0.1 + }, + "scale_up_ratio": { + "description": "扩容比例，如 1.5 表示扩容到当前使用的 1.5 倍", + "type": "number", + "minimum": 1 + } + } + }, "internal_handler.CLICompatibilityInfo": { "type": "object", "properties": { @@ -12981,6 +13381,17 @@ const docTemplate = `{ } } }, + "internal_handler.SetUserSpaceQuotaRequest": { + "type": "object", + "required": [ + "quota" + ], + "properties": { + "quota": { + "type": "integer" + } + } + }, "internal_handler.SharedQueueReq": { "type": "object", "required": [ @@ -13017,6 +13428,56 @@ const docTemplate = `{ } } }, + "internal_handler.StorageCapabilities": { + "type": "object", + "properties": { + "backend": { + "type": "string" + }, + "configured": { + "type": "boolean" + }, + "csi_driver": { + "type": "string" + }, + "pv_name": { + "type": "string" + }, + "pvc_name": { + "type": "string" + }, + "pvc_namespace": { + "type": "string" + }, + "quota_enabled": { + "type": "boolean" + }, + "quota_provider": { + "type": "string" + }, + "quota_readable": { + "type": "boolean" + }, + "quota_writable": { + "type": "boolean" + }, + "reasons": { + "type": "array", + "items": { + "type": "string" + } + }, + "storage_server_available": { + "type": "boolean" + }, + "toolbox_available": { + "type": "boolean" + }, + "usage_readable": { + "type": "boolean" + } + } + }, "internal_handler.SwitchQueueReq": { "type": "object", "required": [ @@ -13208,7 +13669,6 @@ const docTemplate = `{ "type": "string" }, "validate": { - "description": "是否立即校验连接", "type": "boolean" } } diff --git a/backend/docs/swagger.json b/backend/docs/swagger.json index 8d13d0509..84e29dcee 100644 --- a/backend/docs/swagger.json +++ b/backend/docs/swagger.json @@ -3221,6 +3221,174 @@ } } }, + "/v1/admin/storage/capabilities": { + "get": { + "security": [ + { + "Bearer": [] + } + ], + "description": "Detect whether the configured storage supports CephFS usage and quota operations", + "produces": [ + "application/json" + ], + "tags": [ + "Storage" + ], + "summary": "Get storage quota capabilities", + "responses": { + "200": { + "description": "Success", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-internal_handler_StorageCapabilities" + } + } + } + } + }, + "/v1/admin/storage/user-spaces": { + "get": { + "security": [ + { + "Bearer": [] + } + ], + "description": "Get the size of all user spaces from database", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Storage" + ], + "summary": "Get all user space sizes", + "parameters": [ + { + "type": "integer", + "description": "Page number", + "name": "page", + "in": "query" + }, + { + "type": "integer", + "description": "Page size", + "name": "pageSize", + "in": "query" + } + ], + "responses": { + "200": { + "description": "Success", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any" + } + }, + "500": { + "description": "Other errors", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any" + } + } + } + } + }, + "/v1/admin/storage/user-spaces/refresh": { + "post": { + "security": [ + { + "Bearer": [] + } + ], + "description": "Read current CephFS usage for every user directory and update the usage cache", + "produces": [ + "application/json" + ], + "tags": [ + "Storage" + ], + "summary": "Refresh all user space usage", + "responses": { + "200": { + "description": "Success", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-github_com_raids-lab_crater_pkg_patrol_StorageUsageRefreshResult" + } + }, + "500": { + "description": "Other errors", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any" + } + } + } + } + }, + "/v1/admin/storage/user-spaces/{user}/quota": { + "put": { + "security": [ + { + "Bearer": [] + } + ], + "description": "Set the space quota for a user", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Storage" + ], + "summary": "Set user space quota", + "parameters": [ + { + "type": "string", + "description": "Username", + "name": "user", + "in": "path", + "required": true + }, + { + "description": "Space quota request", + "name": "quota", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/internal_handler.SetUserSpaceQuotaRequest" + } + } + ], + "responses": { + "200": { + "description": "Success", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any" + } + }, + "400": { + "description": "Request parameter error", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any" + } + }, + "404": { + "description": "User not found", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any" + } + }, + "500": { + "description": "Other errors", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any" + } + } + } + } + }, "/v1/admin/system-config/gpu-analysis": { "get": { "security": [ @@ -3325,7 +3493,7 @@ "Bearer": [] } ], - "description": "更新 LLM 的连接信息。如果 validate 为 true，会尝试连接 /check 接口，失败则不保存。", + "description": "更新 LLM 的连接信息。如果 validate 为 true，会尝试连接 /models 接口，失败则不保存。", "consumes": [ "application/json" ], @@ -8426,6 +8594,57 @@ } } }, + "/v1/operations/cronjob/execute": { + "post": { + "security": [ + { + "Bearer": [] + } + ], + "description": "Execute a patrol job immediately", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Operations" + ], + "summary": "Execute patrol job", + "parameters": [ + { + "description": "Job name", + "name": "jobName", + "in": "body", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any" + } + }, + "400": { + "description": "Request parameter error", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any" + } + }, + "500": { + "description": "Other errors", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any" + } + } + } + } + }, "/v1/operations/keep/{name}": { "put": { "description": "set KeepWhenLowResourceUsage of the job to the opposite value", @@ -8755,6 +8974,111 @@ } } }, + "/v1/storage/capabilities": { + "get": { + "security": [ + { + "Bearer": [] + } + ], + "description": "Detect whether the configured storage supports CephFS usage and quota operations", + "produces": [ + "application/json" + ], + "tags": [ + "Storage" + ], + "summary": "Get storage quota capabilities", + "responses": { + "200": { + "description": "Success", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-internal_handler_StorageCapabilities" + } + } + } + } + }, + "/v1/storage/dirsize/{path}": { + "get": { + "security": [ + { + "Bearer": [] + } + ], + "description": "Get the size of a directory in CephFS using getfattr command", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Storage" + ], + "summary": "Get directory size in CephFS", + "parameters": [ + { + "type": "string", + "description": "Directory path", + "name": "path", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "Success", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any" + } + }, + "400": { + "description": "Request parameter error", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any" + } + }, + "500": { + "description": "Other errors", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any" + } + } + } + } + }, + "/v1/storage/my-quota": { + "get": { + "security": [ + { + "Bearer": [] + } + ], + "description": "Get the storage quota for the currently authenticated user", + "produces": [ + "application/json" + ], + "tags": [ + "Storage" + ], + "summary": "Get current user's storage quota", + "responses": { + "200": { + "description": "Success", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any" + } + }, + "500": { + "description": "Other errors", + "schema": { + "$ref": "#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any" + } + } + } + } + }, "/v1/system-config/model-download-limit": { "get": { "security": [ @@ -11087,6 +11411,21 @@ } } }, + "github_com_raids-lab_crater_internal_resputil.Response-github_com_raids-lab_crater_pkg_patrol_StorageUsageRefreshResult": { + "type": "object", + "properties": { + "code": { + "description": "依然保持 int (ErrorCode) 类型", + "type": "integer" + }, + "data": { + "$ref": "#/definitions/github_com_raids-lab_crater_pkg_patrol.StorageUsageRefreshResult" + }, + "msg": { + "type": "string" + } + } + }, "github_com_raids-lab_crater_internal_resputil.Response-internal_handler_AdjustUserExtraBalanceResp": { "type": "object", "properties": { @@ -11432,6 +11771,21 @@ } } }, + "github_com_raids-lab_crater_internal_resputil.Response-internal_handler_StorageCapabilities": { + "type": "object", + "properties": { + "code": { + "description": "依然保持 int (ErrorCode) 类型", + "type": "integer" + }, + "data": { + "$ref": "#/definitions/internal_handler.StorageCapabilities" + }, + "msg": { + "type": "string" + } + } + }, "github_com_raids-lab_crater_internal_resputil.Response-internal_handler_TokenReq": { "type": "object", "properties": { @@ -11924,6 +12278,20 @@ } } }, + "github_com_raids-lab_crater_pkg_patrol.StorageUsageRefreshResult": { + "type": "object", + "properties": { + "failed": { + "type": "integer" + }, + "refreshed_at": { + "type": "string" + }, + "updated": { + "type": "integer" + } + } + }, "gorm.DeletedAt": { "type": "object", "properties": { @@ -12157,6 +12525,38 @@ } } }, + "internal_handler.AutoScaleRequest": { + "type": "object", + "required": [ + "max_quota", + "min_quota", + "scale_down_ratio", + "scale_up_ratio" + ], + "properties": { + "max_quota": { + "description": "最大配额，-1 表示无限制", + "type": "integer", + "minimum": -1 + }, + "min_quota": { + "description": "最小配额，-1 表示无限制", + "type": "integer", + "minimum": -1 + }, + "scale_down_ratio": { + "description": "缩容比例，如 0.8 表示缩容到当前使用的 0.8 倍", + "type": "number", + "maximum": 1, + "minimum": 0.1 + }, + "scale_up_ratio": { + "description": "扩容比例，如 1.5 表示扩容到当前使用的 1.5 倍", + "type": "number", + "minimum": 1 + } + } + }, "internal_handler.CLICompatibilityInfo": { "type": "object", "properties": { @@ -12973,6 +13373,17 @@ } } }, + "internal_handler.SetUserSpaceQuotaRequest": { + "type": "object", + "required": [ + "quota" + ], + "properties": { + "quota": { + "type": "integer" + } + } + }, "internal_handler.SharedQueueReq": { "type": "object", "required": [ @@ -13009,6 +13420,56 @@ } } }, + "internal_handler.StorageCapabilities": { + "type": "object", + "properties": { + "backend": { + "type": "string" + }, + "configured": { + "type": "boolean" + }, + "csi_driver": { + "type": "string" + }, + "pv_name": { + "type": "string" + }, + "pvc_name": { + "type": "string" + }, + "pvc_namespace": { + "type": "string" + }, + "quota_enabled": { + "type": "boolean" + }, + "quota_provider": { + "type": "string" + }, + "quota_readable": { + "type": "boolean" + }, + "quota_writable": { + "type": "boolean" + }, + "reasons": { + "type": "array", + "items": { + "type": "string" + } + }, + "storage_server_available": { + "type": "boolean" + }, + "toolbox_available": { + "type": "boolean" + }, + "usage_readable": { + "type": "boolean" + } + } + }, "internal_handler.SwitchQueueReq": { "type": "object", "required": [ @@ -13200,7 +13661,6 @@ "type": "string" }, "validate": { - "description": "是否立即校验连接", "type": "boolean" } } diff --git a/backend/docs/swagger.yaml b/backend/docs/swagger.yaml index 84b77b487..0fccd38b6 100644 --- a/backend/docs/swagger.yaml +++ b/backend/docs/swagger.yaml @@ -586,6 +586,16 @@ definitions: msg: type: string type: object + github_com_raids-lab_crater_internal_resputil.Response-github_com_raids-lab_crater_pkg_patrol_StorageUsageRefreshResult: + properties: + code: + description: 依然保持 int (ErrorCode) 类型 + type: integer + data: + $ref: '#/definitions/github_com_raids-lab_crater_pkg_patrol.StorageUsageRefreshResult' + msg: + type: string + type: object github_com_raids-lab_crater_internal_resputil.Response-internal_handler_AdjustUserExtraBalanceResp: properties: code: @@ -816,6 +826,16 @@ definitions: msg: type: string type: object + github_com_raids-lab_crater_internal_resputil.Response-internal_handler_StorageCapabilities: + properties: + code: + description: 依然保持 int (ErrorCode) 类型 + type: integer + data: + $ref: '#/definitions/internal_handler.StorageCapabilities' + msg: + type: string + type: object github_com_raids-lab_crater_internal_resputil.Response-internal_handler_TokenReq: properties: code: @@ -1148,6 +1168,15 @@ definitions: description: 用户真实名称（用于tooltip） type: string type: object + github_com_raids-lab_crater_pkg_patrol.StorageUsageRefreshResult: + properties: + failed: + type: integer + refreshed_at: + type: string + updated: + type: integer + type: object gorm.DeletedAt: properties: time: @@ -1298,6 +1327,31 @@ definitions: ldapHelp: type: string type: object + internal_handler.AutoScaleRequest: + properties: + max_quota: + description: 最大配额，-1 表示无限制 + minimum: -1 + type: integer + min_quota: + description: 最小配额，-1 表示无限制 + minimum: -1 + type: integer + scale_down_ratio: + description: 缩容比例，如 0.8 表示缩容到当前使用的 0.8 倍 + maximum: 1 + minimum: 0.1 + type: number + scale_up_ratio: + description: 扩容比例，如 1.5 表示扩容到当前使用的 1.5 倍 + minimum: 1 + type: number + required: + - max_quota + - min_quota + - scale_down_ratio + - scale_up_ratio + type: object internal_handler.CLICompatibilityInfo: properties: apiVersion: @@ -1840,6 +1894,13 @@ definitions: enable: type: boolean type: object + internal_handler.SetUserSpaceQuotaRequest: + properties: + quota: + type: integer + required: + - quota + type: object internal_handler.SharedQueueReq: properties: datasetID: @@ -1864,6 +1925,39 @@ definitions: - datasetID - userIDs type: object + internal_handler.StorageCapabilities: + properties: + backend: + type: string + configured: + type: boolean + csi_driver: + type: string + pv_name: + type: string + pvc_name: + type: string + pvc_namespace: + type: string + quota_enabled: + type: boolean + quota_provider: + type: string + quota_readable: + type: boolean + quota_writable: + type: boolean + reasons: + items: + type: string + type: array + storage_server_available: + type: boolean + toolbox_available: + type: boolean + usage_readable: + type: boolean + type: object internal_handler.SwitchQueueReq: properties: queue: @@ -1990,7 +2084,6 @@ definitions: modelName: type: string validate: - description: 是否立即校验连接 type: boolean required: - baseUrl @@ -4955,6 +5048,113 @@ paths: database tags: - Resource + /v1/admin/storage/capabilities: + get: + description: Detect whether the configured storage supports CephFS usage and + quota operations + produces: + - application/json + responses: + "200": + description: Success + schema: + $ref: '#/definitions/github_com_raids-lab_crater_internal_resputil.Response-internal_handler_StorageCapabilities' + security: + - Bearer: [] + summary: Get storage quota capabilities + tags: + - Storage + /v1/admin/storage/user-spaces: + get: + consumes: + - application/json + description: Get the size of all user spaces from database + parameters: + - description: Page number + in: query + name: page + type: integer + - description: Page size + in: query + name: pageSize + type: integer + produces: + - application/json + responses: + "200": + description: Success + schema: + $ref: '#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any' + "500": + description: Other errors + schema: + $ref: '#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any' + security: + - Bearer: [] + summary: Get all user space sizes + tags: + - Storage + /v1/admin/storage/user-spaces/{user}/quota: + put: + consumes: + - application/json + description: Set the space quota for a user + parameters: + - description: Username + in: path + name: user + required: true + type: string + - description: Space quota request + in: body + name: quota + required: true + schema: + $ref: '#/definitions/internal_handler.SetUserSpaceQuotaRequest' + produces: + - application/json + responses: + "200": + description: Success + schema: + $ref: '#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any' + "400": + description: Request parameter error + schema: + $ref: '#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any' + "404": + description: User not found + schema: + $ref: '#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any' + "500": + description: Other errors + schema: + $ref: '#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any' + security: + - Bearer: [] + summary: Set user space quota + tags: + - Storage + /v1/admin/storage/user-spaces/refresh: + post: + description: Read current CephFS usage for every user directory and update the + usage cache + produces: + - application/json + responses: + "200": + description: Success + schema: + $ref: '#/definitions/github_com_raids-lab_crater_internal_resputil.Response-github_com_raids-lab_crater_pkg_patrol_StorageUsageRefreshResult' + "500": + description: Other errors + schema: + $ref: '#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any' + security: + - Bearer: [] + summary: Refresh all user space usage + tags: + - Storage /v1/admin/system-config/gpu-analysis: get: description: 查询当前系统是否开启了自动 GPU 资源滥用检测 @@ -5037,7 +5237,7 @@ paths: put: consumes: - application/json - description: 更新 LLM 的连接信息。如果 validate 为 true，会尝试连接 /check 接口，失败则不保存。 + description: 更新 LLM 的连接信息。如果 validate 为 true，会尝试连接 /models 接口，失败则不保存。 parameters: - description: 配置信息 in: body @@ -8246,6 +8446,38 @@ paths: summary: Update cronjob config tags: - Operations + /v1/operations/cronjob/execute: + post: + consumes: + - application/json + description: Execute a patrol job immediately + parameters: + - description: Job name + in: body + name: jobName + required: true + schema: + type: string + produces: + - application/json + responses: + "200": + description: Success + schema: + $ref: '#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any' + "400": + description: Request parameter error + schema: + $ref: '#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any' + "500": + description: Other errors + schema: + $ref: '#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any' + security: + - Bearer: [] + summary: Execute patrol job + tags: + - Operations /v1/operations/keep/{name}: put: consumes: @@ -8459,6 +8691,72 @@ paths: summary: 获取资源统计信息 tags: - statistics + /v1/storage/capabilities: + get: + description: Detect whether the configured storage supports CephFS usage and + quota operations + produces: + - application/json + responses: + "200": + description: Success + schema: + $ref: '#/definitions/github_com_raids-lab_crater_internal_resputil.Response-internal_handler_StorageCapabilities' + security: + - Bearer: [] + summary: Get storage quota capabilities + tags: + - Storage + /v1/storage/dirsize/{path}: + get: + consumes: + - application/json + description: Get the size of a directory in CephFS using getfattr command + parameters: + - description: Directory path + in: path + name: path + required: true + type: string + produces: + - application/json + responses: + "200": + description: Success + schema: + $ref: '#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any' + "400": + description: Request parameter error + schema: + $ref: '#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any' + "500": + description: Other errors + schema: + $ref: '#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any' + security: + - Bearer: [] + summary: Get directory size in CephFS + tags: + - Storage + /v1/storage/my-quota: + get: + description: Get the storage quota for the currently authenticated user + produces: + - application/json + responses: + "200": + description: Success + schema: + $ref: '#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any' + "500": + description: Other errors + schema: + $ref: '#/definitions/github_com_raids-lab_crater_internal_resputil.Response-any' + security: + - Bearer: [] + summary: Get current user's storage quota + tags: + - Storage /v1/system-config/model-download-limit: get: description: 获取当前用户的并发任务上限、滚动窗口成功下载上限和白名单豁免状态 diff --git a/backend/etc/example-config.yaml b/backend/etc/example-config.yaml index f2ce4a9b5..7f7e21050 100644 --- a/backend/etc/example-config.yaml +++ b/backend/etc/example-config.yaml @@ -75,6 +75,21 @@ postgres: # Persistent volume claim and path prefix configurations # Required: All PVC names and prefix paths must be specified storage: + quota: + # CephFS-only feature. Keep false for NFS and other storage backends. + enabled: false + # auto prefers storage-server and falls back to a Rook Ceph toolbox. + provider: auto + # Optional; defaults to http://webdav-service..svc:7320 + storageServerURL: "" + # Rook Ceph namespace. Other fields derive compatible defaults from this value. + rookNamespace: rook-ceph + # Optional override for clusters using a custom CephFS CSI driver name. + cephFSCSIDriver: "" + # Label selector for the optional toolbox fallback Pod. + toolboxLabelSelector: app=rook-ceph-tools + # Fallback filesystem name when the PV does not expose volumeAttributes.fsName. + cephFSName: cephfs # Path prefixes for different types of storage locations # Required: All prefix paths must be specified prefix: diff --git a/backend/go.mod b/backend/go.mod index 09d21c0c8..10b9b5771 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -18,6 +18,7 @@ require ( github.com/prometheus/common v0.65.0 github.com/robfig/cron/v3 v3.0.0 github.com/samber/lo v1.51.0 + github.com/sashabaranov/go-openai v1.41.2 github.com/smartystreets/goconvey v1.8.1 github.com/swaggo/files v1.0.1 github.com/swaggo/gin-swagger v1.6.0 diff --git a/backend/go.sum b/backend/go.sum index 23e0eb304..b906fbec5 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -269,6 +269,8 @@ github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/samber/lo v1.51.0 h1:kysRYLbHy/MB7kQZf5DSN50JHmMsNEdeY24VzJFu7wI= github.com/samber/lo v1.51.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0= +github.com/sashabaranov/go-openai v1.41.2 h1:vfPRBZNMpnqu8ELsclWcAvF19lDNgh1t6TVfFFOPiSM= +github.com/sashabaranov/go-openai v1.41.2/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= diff --git a/backend/hack/bootstrap-cephfs-quota-agent.sh b/backend/hack/bootstrap-cephfs-quota-agent.sh new file mode 100644 index 000000000..4bb1ee222 --- /dev/null +++ b/backend/hack/bootstrap-cephfs-quota-agent.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash + +set -euo pipefail + +APP_NAMESPACE="${APP_NAMESPACE:-crater-workspace}" +ROOK_NAMESPACE="${ROOK_NAMESPACE:-rook-ceph}" +SOURCE_PVC="${SOURCE_PVC:-crater-rw-storage}" +QUOTA_CLIENT="${QUOTA_CLIENT:-crater-quota}" +QUOTA_PV="${QUOTA_PV:-crater-quota-storage-pv}" +QUOTA_PVC="${QUOTA_PVC:-crater-quota-storage}" +CSI_SECRET="${CSI_SECRET:-crater-quota-csi}" +GENERATED_CEPH_CLIENT_SECRET="${GENERATED_CEPH_CLIENT_SECRET:-rook-ceph-client-${QUOTA_CLIENT}}" + +for command in kubectl grep; do + if ! command -v "$command" >/dev/null 2>&1; then + printf 'required command not found: %s\n' "$command" >&2 + exit 1 + fi +done + +if ! kubectl api-resources --api-group=ceph.rook.io -o name | grep -qx 'cephclients.ceph.rook.io'; then + echo "CephClient CRD is unavailable; install or enable the Rook operator first." >&2 + exit 1 +fi + +source_pv=$(kubectl -n "$APP_NAMESPACE" get pvc "$SOURCE_PVC" -o jsonpath='{.spec.volumeName}') +if [[ -z "$source_pv" ]]; then + echo "PVC $APP_NAMESPACE/$SOURCE_PVC is not bound." >&2 + exit 1 +fi + +capacity=$(kubectl get pv "$source_pv" -o jsonpath='{.spec.capacity.storage}') +storage_class=$(kubectl get pv "$source_pv" -o jsonpath='{.spec.storageClassName}') +driver=$(kubectl get pv "$source_pv" -o jsonpath='{.spec.csi.driver}') +volume_handle=$(kubectl get pv "$source_pv" -o jsonpath='{.spec.csi.volumeHandle}') +cluster_id=$(kubectl get pv "$source_pv" -o jsonpath='{.spec.csi.volumeAttributes.clusterID}') +fs_name=$(kubectl get pv "$source_pv" -o jsonpath='{.spec.csi.volumeAttributes.fsName}') +subvolume_name=$(kubectl get pv "$source_pv" -o jsonpath='{.spec.csi.volumeAttributes.subvolumeName}') +subvolume_path=$(kubectl get pv "$source_pv" -o jsonpath='{.spec.csi.volumeAttributes.subvolumePath}') + +if [[ "$driver" != *cephfs.csi.ceph.com || -z "$subvolume_path" || -z "$fs_name" ]]; then + echo "PVC $APP_NAMESPACE/$SOURCE_PVC is not a supported CephFS CSI volume." >&2 + exit 1 +fi + +# Rook recommends retaining the original dynamically provisioned PV while a +# static PV points at the same CephFS subvolume. +kubectl patch pv "$source_pv" --type=merge \ + -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' >/dev/null + +cat </dev/null || true) + if [[ -n "$status_secret" ]]; then + generated_secret="$status_secret" + fi + if kubectl -n "$ROOK_NAMESPACE" get secret "$generated_secret" >/dev/null 2>&1; then + break + fi + sleep 2 +done + +if ! kubectl -n "$ROOK_NAMESPACE" get secret "$generated_secret" >/dev/null 2>&1; then + echo "Rook did not create a Secret for CephClient $QUOTA_CLIENT." >&2 + exit 1 +fi + +client_key=$(kubectl -n "$ROOK_NAMESPACE" get secret "$generated_secret" \ + -o "go-template={{ index .data \"${QUOTA_CLIENT}\" }}") +if [[ -z "$client_key" ]]; then + echo "The generated CephClient Secret does not contain the expected key." >&2 + exit 1 +fi + +cat </dev/null 2>&1; then + printf 'required command not found: %s\n' "$command" >&2 + exit 1 + fi +done + +REQUIRED_GO_VERSION=$(awk '$1 == "go" { print $2; exit }' "$ROOT_DIR/backend/go.mod") +if [[ -z "$REQUIRED_GO_VERSION" ]]; then + echo "Go version was not found in backend/go.mod" >&2 + exit 1 +fi + +if [[ ! -f "$ROOT_DIR/$CONFIG_FILE" ]]; then + echo "config file not found: $ROOT_DIR/$CONFIG_FILE" >&2 + exit 1 +fi + +pvc_status=$(kubectl -n "$APP_NAMESPACE" get pvc "$QUOTA_PVC" \ + -o jsonpath='{.status.phase}' 2>/dev/null || true) +if [[ "$pvc_status" != "Bound" ]]; then + echo "PVC $APP_NAMESPACE/$QUOTA_PVC is not Bound." >&2 + echo "Run backend/hack/bootstrap-cephfs-quota-agent.sh first." >&2 + exit 1 +fi + +access_token_secret=$(awk ' + /^[[:space:]]*accessTokenSecret:[[:space:]]*/ { + value = $0 + sub(/^[[:space:]]*accessTokenSecret:[[:space:]]*/, "", value) + gsub(/^["'\'' ]+|["'\'' ]+$/, "", value) + print value + exit + } +' "$ROOT_DIR/$CONFIG_FILE") +if [[ -z "$access_token_secret" ]]; then + echo "auth.token.accessTokenSecret was not found in $CONFIG_FILE" >&2 + exit 1 +fi + +mkdir -p "$ROOT_DIR/backend/.gocache" +( + cd "$ROOT_DIR/backend" + GOCACHE="$ROOT_DIR/backend/.gocache" \ + GOTOOLCHAIN="${GOTOOLCHAIN:-go${REQUIRED_GO_VERSION}}" \ + CGO_ENABLED=0 GOOS=linux GOARCH="$TARGET_ARCH" \ + go build -o "$BUILD_OUTPUT" ./cmd/storage-server +) + +internal_token=$(printf 'crater-storage-quota:%s' "$access_token_secret" | sha256sum | awk '{print $1}') +unset access_token_secret +secret_base64=$(printf '%s' "$internal_token" | base64 | tr -d '\r\n') +unset internal_token +cat </tmp/quota-agent.log 2>&1 &' +sleep 2 +kubectl -n "$APP_NAMESPACE" exec "$DEV_POD" -c quota-agent -- sh -c \ + 'pid=$(pidof storage-server || true); if [ -z "$pid" ]; then cat /tmp/quota-agent.log; exit 1; fi; kill -0 "$pid"; tail -n 20 /tmp/quota-agent.log' + +echo +echo "Development quota-agent is running from the local binary." +echo "Start the local tunnel in another terminal:" +echo " kubectl -n ${APP_NAMESPACE} port-forward pod/${DEV_POD} 7330:7320" +echo +echo "Cleanup:" +echo " bash backend/hack/run-cephfs-quota-agent-dev.sh --cleanup" diff --git a/backend/internal/handler/aijob/new.go b/backend/internal/handler/aijob/new.go index 9513b12d8..6776d7180 100644 --- a/backend/internal/handler/aijob/new.go +++ b/backend/internal/handler/aijob/new.go @@ -139,6 +139,12 @@ func (mgr *AIJobMgr) CreateJupyterJob(c *gin.Context) { taskModel.PodTemplate = datatypes.NewJSONType(podSpec) taskModel.Owner = token.Username + + if err := interutil.CheckStorageQuota(token.Username); err != nil { + resputil.HandleError(c, err) + return + } + err = mgr.taskService.Create(taskModel) if err != nil { resputil.Error(c, fmt.Sprintf("create task failed, err %v", err), resputil.NotSpecified) @@ -205,6 +211,12 @@ func (mgr *AIJobMgr) CreateCustom(c *gin.Context) { taskModel.PodTemplate = datatypes.NewJSONType(podSpec) taskModel.Owner = token.Username + + if err := interutil.CheckStorageQuota(token.Username); err != nil { + resputil.HandleError(c, err) + return + } + err = mgr.taskService.Create(taskModel) if err != nil { resputil.Error(c, fmt.Sprintf("create task failed, err %v", err), resputil.NotSpecified) diff --git a/backend/internal/handler/operations/cronjob.go b/backend/internal/handler/operations/cronjob.go index 372b456c5..59a292312 100644 --- a/backend/internal/handler/operations/cronjob.go +++ b/backend/internal/handler/operations/cronjob.go @@ -1,18 +1,23 @@ package operations import ( + "context" "encoding/json" "fmt" "time" "github.com/gin-gonic/gin" "github.com/samber/lo" + "gorm.io/datatypes" "k8s.io/klog/v2" "k8s.io/utils/ptr" "github.com/raids-lab/crater/dao/model" + "github.com/raids-lab/crater/dao/query" + "github.com/raids-lab/crater/internal/bizerr" "github.com/raids-lab/crater/internal/resputil" "github.com/raids-lab/crater/pkg/patrol" + "github.com/raids-lab/crater/pkg/util" ) // UpdateCronjobConfig godoc @@ -289,3 +294,86 @@ func (cm *OperationsMgr) GetLastCronjobRecord(c *gin.Context) { resputil.Success(c, records) } + +// ExecutePatrolJob godoc +// +// @Summary Execute patrol job +// @Description Execute a patrol job immediately +// @Tags Operations +// @Accept json +// @Produce json +// @Security Bearer +// @Param jobName body string true "Job name" +// @Success 200 {object} resputil.Response[any] "Success" +// @Failure 400 {object} resputil.Response[any] "Request parameter error" +// @Failure 500 {object} resputil.Response[any] "Other errors" +// @Router /v1/operations/cronjob/execute [post] +func (mgr *OperationsMgr) ExecutePatrolJob(c *gin.Context) { + var req struct { + JobName string `json:"jobName" binding:"required"` + } + if err := c.ShouldBindJSON(&req); err != nil { + resputil.HandleError(c, bizerr.BadRequest.InvalidRequest.Wrap(err, "invalid request body")) + return + } + + // 获取巡检函数 + var f util.AnyFunc + var err error + switch req.JobName { + case patrol.TRIGGER_GPU_ANALYSIS_JOB: + f, err = patrol.GetPatrolFunc(req.JobName, mgr.cronJobManager.GetPatrolClients(), nil) + default: + resputil.HandleError(c, bizerr.BadRequest.ParameterError.New("unsupported patrol job: "+req.JobName)) + return + } + + if err != nil { + resputil.HandleError(c, bizerr.Internal.ServiceError.Wrap(err, "failed to initialize patrol job")) + return + } + + // 异步执行巡检任务 + go func() { + defer func() { + if r := recover(); r != nil { + klog.Errorf("ExecutePatrolJob: panic in patrol job %s: %v", req.JobName, r) + } + }() + + ctx := context.Background() + executeTime := time.Now() + jobResult, err := f(ctx) + + status := model.CronJobRecordStatusSuccess + if err != nil { + status = model.CronJobRecordStatusFailed + klog.Errorf("ExecutePatrolJob: patrol job %s failed: %v", req.JobName, err) + } else { + klog.Infof("ExecutePatrolJob: patrol job %s completed: %v", req.JobName, jobResult) + } + + // 保存执行记录 + rec := &model.CronJobRecord{ + Name: req.JobName, + ExecuteTime: executeTime, + Message: fmt.Sprintf("manual execute patrol job: %s", req.JobName), + Status: status, + } + + if jobResult != nil { + if data, err := json.Marshal(jobResult); err != nil { + klog.Errorf("ExecutePatrolJob: failed to marshal job result: %v", err) + } else { + rec.JobData = datatypes.JSON(data) + } + } + + if err := query.GetDB().WithContext(ctx).Model(rec).Create(rec).Error; err != nil { + klog.Errorf("ExecutePatrolJob: failed to create record: %v", err) + } + }() + + // 立即返回成功响应 + resputil.Success(c, "任务已开始执行，请稍后在执行记录中查看结果") +} diff --git a/backend/internal/handler/operations/operations.go b/backend/internal/handler/operations/operations.go index 4226b608a..2b40b37ae 100644 --- a/backend/internal/handler/operations/operations.go +++ b/backend/internal/handler/operations/operations.go @@ -71,6 +71,7 @@ func (mgr *OperationsMgr) RegisterAdmin(g *gin.RouterGroup) { g.POST("/cronjob/record/time", mgr.GetCronjobRecordTimeRange) g.POST("/cronjob/record/list", mgr.GetCronjobRecords) g.POST("/cronjob/record/delete", mgr.DeleteCronjobRecords) + g.POST("/cronjob/execute", mgr.ExecutePatrolJob) } func (cm *OperationsMgr) StopCron() { diff --git a/backend/internal/handler/spjob/spjob.go b/backend/internal/handler/spjob/spjob.go index 231fb98e3..7370e0c0c 100644 --- a/backend/internal/handler/spjob/spjob.go +++ b/backend/internal/handler/spjob/spjob.go @@ -128,6 +128,11 @@ func (mgr *SparseJobMgr) Create(c *gin.Context) { return } + if err := util.CheckStorageQuota(token.Username); err != nil { + resputil.HandleError(c, err) + return + } + volumes, volumeMounts, err := vcjob.GenerateVolumeMounts(c, req.VolumeMounts, token) if err != nil { resputil.Error(c, err.Error(), resputil.NotSpecified) diff --git a/backend/internal/handler/storage.go b/backend/internal/handler/storage.go new file mode 100644 index 000000000..64793d51a --- /dev/null +++ b/backend/internal/handler/storage.go @@ -0,0 +1,1111 @@ +package handler + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + "github.com/gin-gonic/gin" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + + "github.com/raids-lab/crater/dao/model" + "github.com/raids-lab/crater/dao/query" + "github.com/raids-lab/crater/internal/bizerr" + "github.com/raids-lab/crater/internal/resputil" + "github.com/raids-lab/crater/internal/util" + "github.com/raids-lab/crater/pkg/ceph" + "github.com/raids-lab/crater/pkg/config" + "github.com/raids-lab/crater/pkg/constants" + "github.com/raids-lab/crater/pkg/monitor" + "github.com/raids-lab/crater/pkg/patrol" + "github.com/raids-lab/crater/pkg/storagegovernance" + "github.com/raids-lab/crater/pkg/storagequota" +) + +const toolboxCapabilityTimeout = 20 * time.Second + +// ---- LLM 任务状态存储 ---- + +//nolint:gochecknoinits // This is the standard way to register a gin handler. +func init() { + Registers = append(Registers, NewStorageMgr) +} + +type StorageMgr struct { + name string + kubeClient kubernetes.Interface + kubeConfig *rest.Config + promClient monitor.PrometheusInterface +} + +type StorageCapabilities struct { + Backend string `json:"backend"` + Configured bool `json:"configured"` + QuotaEnabled bool `json:"quota_enabled"` + PVCName string `json:"pvc_name"` + PVCNamespace string `json:"pvc_namespace,omitempty"` + PVName string `json:"pv_name,omitempty"` + CSIDriver string `json:"csi_driver,omitempty"` + QuotaProvider string `json:"quota_provider"` + StorageServerAvailable bool `json:"storage_server_available"` + ToolboxAvailable bool `json:"toolbox_available"` + UsageReadable bool `json:"usage_readable"` + QuotaReadable bool `json:"quota_readable"` + QuotaWritable bool `json:"quota_writable"` + Reasons []string `json:"reasons,omitempty"` +} + +type SetUserSpaceQuotaRequest struct { + Quota int64 `json:"quota" binding:"required"` +} + +const ( + cephStorageBackend = "cephfs" + unknownStorageBackend = "unknown" +) + +// AutoScaleRequest 自动扩缩容请求 +type AutoScaleRequest struct { + MinQuota int64 `json:"min_quota" binding:"required,min=-1"` // 最小配额，-1 表示无限制 + MaxQuota int64 `json:"max_quota" binding:"required,min=-1"` // 最大配额，-1 表示无限制 + ScaleUpRatio float64 `json:"scale_up_ratio" binding:"required,min=1"` // 扩容比例，如 1.5 表示扩容到当前使用的 1.5 倍 + ScaleDownRatio float64 `json:"scale_down_ratio" binding:"required,min=0.1,max=1"` // 缩容比例，如 0.8 表示缩容到当前使用的 0.8 倍 +} + +func NewStorageMgr(conf *RegisterConfig) Manager { + return &StorageMgr{ + name: "storage", + kubeClient: conf.KubeClient, + kubeConfig: conf.KubeConfig, + promClient: conf.PrometheusClient, + } +} + +func (mgr *StorageMgr) GetName() string { return mgr.name } + +func (mgr *StorageMgr) RegisterPublic(_ *gin.RouterGroup) {} + +func (mgr *StorageMgr) RegisterProtected(g *gin.RouterGroup) { + g.GET("/capabilities", mgr.GetCapabilities) + g.GET("/dirsize/*path", mgr.GetDirectorySize) + g.GET("/my-quota", mgr.GetMyQuota) +} + +func (mgr *StorageMgr) RegisterAdmin(g *gin.RouterGroup) { + g.GET("/capabilities", mgr.GetCapabilities) + g.GET("/user-spaces", mgr.GetAllUserSpaceSizes) + g.POST("/user-spaces/refresh", mgr.RefreshUserSpaceSizes) + g.PUT("/user-spaces/:user/quota", mgr.SetUserSpaceQuota) +} + +// GetCapabilities godoc +// +// @Summary Get storage quota capabilities +// @Description Detect whether the configured storage supports CephFS usage and quota operations +// @Tags Storage +// @Produce json +// @Security Bearer +// @Success 200 {object} resputil.Response[StorageCapabilities] "Success" +// @Router /v1/storage/capabilities [get] +// @Router /v1/admin/storage/capabilities [get] +func (mgr *StorageMgr) GetCapabilities(c *gin.Context) { + resputil.Success(c, mgr.detectCapabilities()) +} + +//nolint:gocyclo // Capability probing reports each independent degradation reason. +func (mgr *StorageMgr) detectCapabilities() StorageCapabilities { + cfg := config.GetConfig() + capability := StorageCapabilities{ + Backend: unknownStorageBackend, + Configured: strings.TrimSpace(cfg.Storage.PVC.ReadWriteMany) != "", + QuotaEnabled: ceph.StorageQuotaEnabled(), + PVCName: strings.TrimSpace(cfg.Storage.PVC.ReadWriteMany), + QuotaProvider: ceph.StorageQuotaProvider(), + } + if !capability.QuotaEnabled { + capability.Reasons = append(capability.Reasons, "storage quota management is disabled") + return capability + } + if !capability.Configured { + capability.Reasons = append(capability.Reasons, "storage.pvc.readWriteMany is not configured") + return capability + } + if mgr.kubeClient == nil { + capability.Reasons = append(capability.Reasons, "kubernetes client is not available") + return capability + } + + pvName, pvcNamespace, driver, err := mgr.detectStoragePV(capability.PVCName) + if err != nil { + capability.Reasons = append(capability.Reasons, err.Error()) + return capability + } + capability.PVName = pvName + capability.PVCNamespace = pvcNamespace + capability.CSIDriver = driver + + expectedDriver := ceph.StorageQuotaCephFSCSIDriver() + if driver != expectedDriver { + if driver != "" { + capability.Backend = driver + } + capability.Reasons = append(capability.Reasons, fmt.Sprintf( + "storage PVC CSI driver %q does not match configured CephFS driver %q", + driver, + expectedDriver, + )) + return capability + } + capability.Backend = cephStorageBackend + + if capability.QuotaProvider == storagequota.ProviderDisabled { + capability.Reasons = append(capability.Reasons, "storage quota provider is disabled") + return capability + } + + if capability.QuotaProvider != storagequota.ProviderToolbox { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + storageServerCapabilities, storageServerErr := ceph.GetStorageServerQuotaCapabilities(ctx) + cancel() + capability.StorageServerAvailable = storageServerErr == nil + if storageServerErr != nil { + capability.Reasons = append( + capability.Reasons, + fmt.Sprintf("storage-server is not available: %v", storageServerErr), + ) + } else { + capability.UsageReadable = storageServerCapabilities.UsageReadable + capability.QuotaReadable = storageServerCapabilities.QuotaReadable + capability.QuotaWritable = storageServerCapabilities.QuotaWritable + capability.Reasons = append(capability.Reasons, storageServerCapabilities.Reasons...) + } + } + + needsToolbox := capability.QuotaProvider == storagequota.ProviderToolbox || + (capability.QuotaProvider == storagequota.ProviderAuto && + (!capability.UsageReadable || !capability.QuotaReadable || !capability.QuotaWritable)) + if needsToolbox { + ctx, cancel := context.WithTimeout(context.Background(), toolboxCapabilityTimeout) + toolboxCapabilities, toolboxErr := ceph.GetToolboxQuotaCapabilities( + ctx, + mgr.kubeClient, + mgr.kubeConfig, + ceph.StorageQuotaRookNamespace(), + ) + cancel() + capability.ToolboxAvailable = toolboxErr == nil && toolboxCapabilities.UsageReadable + if toolboxErr != nil { + capability.Reasons = append( + capability.Reasons, + fmt.Sprintf("toolbox is not available: %v", toolboxErr), + ) + } else { + capability.UsageReadable = capability.UsageReadable || toolboxCapabilities.UsageReadable + capability.QuotaReadable = capability.QuotaReadable || toolboxCapabilities.QuotaReadable + capability.QuotaWritable = capability.QuotaWritable || toolboxCapabilities.QuotaWritable + capability.Reasons = append(capability.Reasons, toolboxCapabilities.Reasons...) + } + } + return capability +} + +//nolint:gocritic // The tuple returns PV name, PVC namespace, and CSI driver as separate API fields. +func (mgr *StorageMgr) detectStoragePV(pvcName string) (string, string, string, error) { + cfg := config.GetConfig() + pvcNamespace := strings.TrimSpace(cfg.Namespaces.Job) + if pvcNamespace == "" { + return "", "", "", bizerr.Internal.K8sServiceError.New("job namespace is not configured") + } + pvc, err := mgr.kubeClient.CoreV1().PersistentVolumeClaims(pvcNamespace). + Get(context.TODO(), pvcName, metav1.GetOptions{}) + if err != nil { + return "", pvcNamespace, "", bizerr.Internal.K8sServiceError.Wrap( + err, + fmt.Sprintf("failed to get storage PVC %s/%s", pvcNamespace, pvcName), + ) + } + + if pvc.Spec.VolumeName == "" { + return "", pvc.Namespace, "", bizerr.Internal.K8sServiceError.New( + fmt.Sprintf("storage PVC %s is not bound to a PV", pvcName), + ) + } + + pv, err := mgr.kubeClient.CoreV1().PersistentVolumes().Get(context.TODO(), pvc.Spec.VolumeName, metav1.GetOptions{}) + if err != nil { + return pvc.Spec.VolumeName, pvc.Namespace, "", bizerr.Internal.K8sServiceError.Wrap( + err, + fmt.Sprintf("failed to get storage PV %s", pvc.Spec.VolumeName), + ) + } + if pv.Spec.CSI == nil { + return pv.Name, pvc.Namespace, "", nil + } + return pv.Name, pvc.Namespace, pv.Spec.CSI.Driver, nil +} + +// GetDirectorySize godoc +// +// @Summary Get directory size in CephFS +// @Description Get the size of a directory in CephFS using getfattr command +// @Tags Storage +// @Accept json +// @Produce json +// @Security Bearer +// @Param path path string true "Directory path" +// @Success 200 {object} resputil.Response[any] "Success" +// @Failure 400 {object} resputil.Response[any] "Request parameter error" +// @Failure 500 {object} resputil.Response[any] "Other errors" +// @Router /v1/storage/dirsize/{path} [get] +func (mgr *StorageMgr) GetDirectorySize(c *gin.Context) { + // 1. 获取路径参数 + path := strings.TrimPrefix(c.Request.URL.Path, "/api/v1/storage/dirsize/") + if path == "" { + resputil.HandleError(c, bizerr.BadRequest.MissingParameter.New("path is required")) + return + } + + // 2. 确保路径以 / 开头 + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + + // 3. 执行 Ceph 命令获取目录大小 + cfg := config.GetConfig() + prefixConfig := ceph.StoragePrefixConfig{ + User: cfg.Storage.Prefix.User, + Account: cfg.Storage.Prefix.Account, + Public: cfg.Storage.Prefix.Public, + } + size, err := ceph.GetCephDirectorySize( + mgr.kubeClient, mgr.kubeConfig, ceph.StorageQuotaRookNamespace(), path, prefixConfig, + ) + if err != nil { + klog.Warningf("GetDirectorySize: failed to get size for %q, returning unknown sentinel: %v", path, err) + size = -1 + } + + // 4. 返回结果 + resputil.Success(c, gin.H{ + "path": path, + "size": size, + "unit": "bytes", + "formatted": formatSize(size), + }) +} + +// GetMyQuota godoc +// +// @Summary Get current user's storage quota +// @Description Get the storage quota for the currently authenticated user +// @Tags Storage +// @Produce json +// @Security Bearer +// @Success 200 {object} resputil.Response[any] "Success" +// @Failure 500 {object} resputil.Response[any] "Other errors" +// @Router /v1/storage/my-quota [get] +func (mgr *StorageMgr) GetMyQuota(c *gin.Context) { + token := util.GetToken(c) + + var row struct { + SpaceQuota int64 `gorm:"column:space_quota"` + } + if err := query.GetDB().Raw( + "SELECT space_quota FROM users WHERE id = ? AND deleted_at IS NULL", token.UserID, + ).Scan(&row).Error; err != nil { + resputil.HandleError(c, bizerr.Internal.DatabaseError.Wrap(err, "failed to get storage quota")) + return + } + + resputil.Success(c, gin.H{ + "space_quota": row.SpaceQuota, + "space_quota_formatted": formatSize(row.SpaceQuota), + }) +} + +// GetAllUserSpaceSizes godoc +// +// @Summary Get all user space sizes +// @Description Get the size of all user spaces from database +// @Tags Storage +// @Accept json +// @Produce json +// @Security Bearer +// @Param page query int false "Page number" +// @Param pageSize query int false "Page size" +// @Success 200 {object} resputil.Response[any] "Success" +// @Failure 500 {object} resputil.Response[any] "Other errors" +// @Router /v1/admin/storage/user-spaces [get] +func (mgr *StorageMgr) GetAllUserSpaceSizes(c *gin.Context) { + // 1. 获取分页参数 + page, pageErr := strconv.Atoi(c.DefaultQuery("page", "1")) + pageSize, pageSizeErr := strconv.Atoi(c.DefaultQuery("pageSize", "10")) + if pageErr != nil || page < 1 || pageSizeErr != nil || pageSize < 1 || pageSize > 1000 { + resputil.HandleError(c, bizerr.BadRequest.ParameterError.New( + "page must be positive and pageSize must be between 1 and 1000", + )) + return + } + + // 2. 从数据库中获取用户空间大小和配额 + type UserSpaceInfo struct { + Username string `json:"username"` + Size int64 `json:"size"` + UpdatedAt *time.Time `json:"updated_at"` + SpaceQuota int64 `json:"space_quota"` + OriginalSpaceQuota *int64 `json:"original_space_quota"` + JobsFrozen bool `json:"jobs_frozen"` + ShrinkStage string `json:"shrink_stage"` + } + + var userSpaceInfos []UserSpaceInfo + var total int64 + + db := query.GetDB() + + // 计算总数 + if err := db.Model(&model.User{}).Where("deleted_at IS NULL").Count(&total).Error; err != nil { + klog.Errorf("GetAllUserSpaceSizes: count users: %v", err) + resputil.HandleError(c, bizerr.Internal.DatabaseError.Wrap(err, "failed to count users for storage usage")) + return + } + + // 计算分页偏移量 + offset := (page - 1) * pageSize + + // 获取分页数据，关联 User 表获取 SpaceQuota 和 OriginalSpaceQuota + if err := db.Table("users"). + Select( + "users.name as username, " + + "COALESCE(user_space_sizes.size, -1) as size, " + + "user_space_sizes.updated_at as updated_at, " + + "users.space_quota as space_quota, " + + "users.original_space_quota as original_space_quota, " + + "users.jobs_frozen as jobs_frozen, " + + "users.shrink_stage as shrink_stage", + ). + Joins("LEFT JOIN user_space_sizes ON user_space_sizes.user_id = users.id"). + Where("users.deleted_at IS NULL"). + Order("users.id ASC"). + Offset(offset).Limit(pageSize). + Find(&userSpaceInfos).Error; err != nil { + klog.Errorf("GetAllUserSpaceSizes: query user storage usage: %v", err) + resputil.HandleError(c, bizerr.Internal.DatabaseError.Wrap(err, "failed to query user storage usage")) + return + } + + // 3. 格式化结果 + formattedUserSpaces := make([]map[string]any, 0, len(userSpaceInfos)) + for i := range userSpaceInfos { + info := userSpaceInfos[i] + item := map[string]any{ + "user": info.Username, + "size": info.Size, + "quota": info.SpaceQuota, + "unit": "bytes", + "formatted": formatSize(info.Size), + "updated_at": info.UpdatedAt, + "quota_formatted": formatSize(info.SpaceQuota), + "is_expanded": info.OriginalSpaceQuota != nil, + "jobs_frozen": info.JobsFrozen, + "shrink_stage": info.ShrinkStage, + } + if info.OriginalSpaceQuota != nil { + item["original_quota"] = *info.OriginalSpaceQuota + item["original_quota_formatted"] = formatSize(*info.OriginalSpaceQuota) + } + formattedUserSpaces = append(formattedUserSpaces, item) + } + + // 4. 返回结果（包含分页信息） + resputil.Success(c, gin.H{ + "items": formattedUserSpaces, + "total": total, + "page": page, + "pageSize": pageSize, + "totalPages": (int(total) + pageSize - 1) / pageSize, + }) +} + +// RefreshUserSpaceSizes godoc +// +// @Summary Refresh all user space usage +// @Description Read current CephFS usage for every user directory and update the usage cache +// @Tags Storage +// @Produce json +// @Security Bearer +// @Success 200 {object} resputil.Response[patrol.StorageUsageRefreshResult] "Success" +// @Failure 500 {object} resputil.Response[any] "Other errors" +// @Router /v1/admin/storage/user-spaces/refresh [post] +func (mgr *StorageMgr) RefreshUserSpaceSizes(c *gin.Context) { + result, err := patrol.RefreshUserSpaceSizes(c.Request.Context(), &patrol.Clients{ + KubeClient: mgr.kubeClient, + KubeConfig: mgr.kubeConfig, + }) + if err != nil { + resputil.HandleError(c, bizerr.Internal.FileSystemError.Wrap(err, "failed to refresh storage usage")) + return + } + + resputil.Success(c, result) +} + +// SetUserSpaceQuota godoc +// +// @Summary Set user space quota +// @Description Set the space quota for a user +// @Tags Storage +// @Accept json +// @Produce json +// @Security Bearer +// @Param user path string true "Username" +// @Param quota body SetUserSpaceQuotaRequest true "Space quota request" +// @Success 200 {object} resputil.Response[any] "Success" +// @Failure 400 {object} resputil.Response[any] "Request parameter error" +// @Failure 404 {object} resputil.Response[any] "User not found" +// @Failure 500 {object} resputil.Response[any] "Other errors" +// @Router /v1/admin/storage/user-spaces/{user}/quota [put] +func (mgr *StorageMgr) SetUserSpaceQuota(c *gin.Context) { + // 1. 获取用户名参数 + user := c.Param("user") + if user == "" { + resputil.HandleError(c, bizerr.BadRequest.MissingParameter.New("username is required")) + return + } + + // 2. 解析请求体 + var req SetUserSpaceQuotaRequest + if err := c.ShouldBindJSON(&req); err != nil { + resputil.HandleError(c, bizerr.BadRequest.InvalidRequest.Wrap( + err, + "quota must be provided as an integer number of bytes", + )) + return + } + + // 3. 验证配额值 + if req.Quota < -1 || req.Quota == 0 { + resputil.HandleError(c, bizerr.BadRequest.ParameterError.New( + "quota must be -1 for unlimited or greater than zero", + )) + return + } + + // 4. 获取用户信息（含临时扩容状态） + db := query.GetDB() + var userRow struct { + model.User + SpaceQuota int64 `gorm:"column:space_quota"` + OriginalSpaceQuota *int64 `gorm:"column:original_space_quota"` + } + if err := db.Model(&model.User{}). + Select("users.*, users.space_quota, users.original_space_quota"). + Where("name = ?", user). + First(&userRow).Error; err != nil { + resputil.HandleError(c, bizerr.NotFound.DataBaseNotFound.Wrap(err, "user was not found")) + return + } + userInfo := userRow.User + auditDetails := map[string]any{ + "old_quota": userRow.SpaceQuota, + "new_quota": req.Quota, + "provider": ceph.StorageQuotaProvider(), + } + + // A manual change overrides any legacy temporary expansion. Apply CephFS + // first so the database never reports a quota that was not enforced. + if userRow.OriginalSpaceQuota != nil { + auditDetails["old_original_quota"] = *userRow.OriginalSpaceQuota + auditDetails["cleared_temporary_expansion"] = true + } + + cfg := config.GetConfig() + prefixConfig := ceph.StoragePrefixConfig{ + User: cfg.Storage.Prefix.User, Account: cfg.Storage.Prefix.Account, Public: cfg.Storage.Prefix.Public, + } + userPath := fmt.Sprintf("/user/%s", userInfo.Space) + if err := ceph.SetCephDirectoryQuota( + mgr.kubeClient, mgr.kubeConfig, ceph.StorageQuotaRookNamespace(), userPath, prefixConfig, req.Quota, + ); err != nil { + klog.Errorf("SetUserSpaceQuota: set CephFS quota for user %q: %v", user, err) + auditDetails["ceph_applied"] = false + RecordOperationLog(c, constants.OpTypeSetStorageQuota, user, constants.OpStatusFailed, err.Error(), auditDetails) + resputil.HandleError(c, bizerr.Internal.FileSystemError.Wrap(err, "failed to apply the CephFS storage quota")) + return + } + auditDetails["ceph_applied"] = true + if err := db.Model(&model.User{}).Where("name = ?", user).Updates(map[string]any{ + "space_quota": req.Quota, + "original_space_quota": nil, + "jobs_frozen": false, + "shrink_stage": nil, + "shrink_stage_updated_at": nil, + }).Error; err != nil { + rollbackErr := ceph.SetCephDirectoryQuota( + mgr.kubeClient, mgr.kubeConfig, ceph.StorageQuotaRookNamespace(), userPath, prefixConfig, userRow.SpaceQuota, + ) + klog.Errorf("SetUserSpaceQuota: update database quota for user %q: %v; CephFS rollback: %v", user, err, rollbackErr) + auditDetails["rollback_succeeded"] = rollbackErr == nil + RecordOperationLog(c, constants.OpTypeSetStorageQuota, user, constants.OpStatusFailed, err.Error(), auditDetails) + resputil.HandleError(c, bizerr.Internal.DatabaseError.Wrap(err, "failed to save the storage quota")) + return + } + + RecordOperationLog(c, constants.OpTypeSetStorageQuota, user, constants.OpStatusSuccess, "", auditDetails) + + resputil.Success(c, gin.H{ + "user": user, + "quota": req.Quota, + "unit": "bytes", + "quota_formatted": formatSize(req.Quota), + "ceph_quota_set": true, + }) +} + +// AutoScaleUserSpaceQuota godoc +// +// @Summary Auto scale user space quota +// @Description Auto scale the space quota for a user based on current usage +// @Tags Storage +// @Accept json +// @Produce json +// @Security Bearer +// @Param user path string true "Username" +// @Param body body AutoScaleRequest true "Auto scale configuration" +// @Success 200 {object} resputil.Response[any] "Success" +// @Failure 400 {object} resputil.Response[any] "Request parameter error" +// @Failure 404 {object} resputil.Response[any] "User not found" +// @Failure 500 {object} resputil.Response[any] "Other errors" +func (mgr *StorageMgr) AutoScaleUserSpaceQuota(c *gin.Context) { + // 1. 获取用户名参数 + user := c.Param("user") + if user == "" { + resputil.HandleError(c, bizerr.BadRequest.MissingParameter.New("username is required")) + return + } + + // 2. 解析请求体 + var req AutoScaleRequest + if err := c.ShouldBindJSON(&req); err != nil { + resputil.HandleError(c, bizerr.BadRequest.InvalidRequest.Wrap(err, "invalid request body")) + return + } + + // 3. 获取用户信息和当前使用空间大小 + db := query.GetDB() + var userInfo model.User + if err := db.Where("name = ?", user).First(&userInfo).Error; err != nil { + resputil.HandleError(c, bizerr.NotFound.DataBaseNotFound.Wrap(err, "user was not found")) + return + } + + var userSpaceSize model.UserSpaceSize + if err := db.Where("user_id = ?", userInfo.ID).First(&userSpaceSize).Error; err != nil { + resputil.HandleError(c, bizerr.NotFound.DataBaseNotFound.Wrap(err, "user storage usage was not found")) + return + } + + // 4. 计算新的配额 + currentUsage := userSpaceSize.Size + newQuota := int64(float64(currentUsage) * req.ScaleUpRatio) + + // 应用最小和最大配额限制 + if req.MinQuota != -1 && newQuota < req.MinQuota { + newQuota = req.MinQuota + } + if req.MaxQuota != -1 && newQuota > req.MaxQuota { + newQuota = req.MaxQuota + } + + // 5. 更新用户配额 + if err := db.Model(&model.User{}).Where("name = ?", user).Update("space_quota", newQuota).Error; err != nil { + resputil.HandleError(c, bizerr.Internal.DatabaseError.Wrap(err, "failed to update user storage quota")) + return + } + + // 6. 实际设置 CephFS 目录配额 + cfg := config.GetConfig() + prefixConfig := ceph.StoragePrefixConfig{ + User: cfg.Storage.Prefix.User, + Account: cfg.Storage.Prefix.Account, + Public: cfg.Storage.Prefix.Public, + } + + // 构建用户空间路径 + userPath := fmt.Sprintf("/user/%s", userInfo.Space) + + // 调用 SetCephDirectoryQuota 设置实际配额 + cephErr := ceph.SetCephDirectoryQuota( + mgr.kubeClient, mgr.kubeConfig, ceph.StorageQuotaRookNamespace(), userPath, prefixConfig, newQuota, + ) + if cephErr != nil { + // 记录错误但不影响响应，确保数据库更新成功 + klog.Errorf("AutoScaleUserSpaceQuota: 设置用户 %s Ceph 配额失败: %v", user, cephErr) + } + + // 7. 返回结果 + resputil.Success(c, gin.H{ + "user": user, + "current_usage": currentUsage, + "new_quota": newQuota, + "unit": "bytes", + "current_usage_formatted": formatSize(currentUsage), + "new_quota_formatted": formatSize(newQuota), + "ceph_quota_set": cephErr == nil, + "ceph_quota_error": cephErr, + }) +} + +// RunAutoShrink triggers one manual scan that shrinks users currently in temporary +// expansion state back to their original quota when it is safe to do so. +func (mgr *StorageMgr) RunAutoShrink(c *gin.Context) { + result, err := patrol.RunAutoShrinkStorageExpansions(c.Request.Context(), &patrol.Clients{ + KubeClient: mgr.kubeClient, + KubeConfig: mgr.kubeConfig, + PromClient: mgr.promClient, + }) + if err != nil { + resputil.HandleError(c, bizerr.Internal.ServiceError.Wrap(err, "failed to run automatic quota shrink")) + return + } + + resputil.Success(c, gin.H{ + "message": result, + }) +} + +// ApplyExpansion godoc +// +// @Summary Apply temporary storage expansion for a user +// @Description Save the current quota as original and set an expanded quota +// @Tags Storage +// @Accept json +// @Produce json +// @Security Bearer +// @Param user path string true "Username" +// @Param body body object true "expand_bytes: bytes to add on top of current quota" +// @Success 200 {object} resputil.Response[any] "Success" +// @Failure 400 {object} resputil.Response[any] "Request parameter error" +// @Failure 500 {object} resputil.Response[any] "Other errors" +func (mgr *StorageMgr) ApplyExpansion(c *gin.Context) { + user := c.Param("user") + if user == "" { + resputil.HandleError(c, bizerr.BadRequest.MissingParameter.New("username is required")) + return + } + + var req struct { + ExpandBytes int64 `json:"expand_bytes" binding:"required,min=1"` + FreezeNewJobs bool `json:"freeze_new_jobs"` + DecisionJobID string `json:"decision_job_id"` + } + if err := c.ShouldBindJSON(&req); err != nil { + resputil.HandleError(c, bizerr.BadRequest.InvalidRequest.Wrap(err, "invalid request body")) + return + } + + db := query.GetDB() + + // 查询当前配额和原始配额 + var row struct { + SpaceQuota int64 `gorm:"column:space_quota"` + OriginalSpaceQuota *int64 `gorm:"column:original_space_quota"` + } + if err := db.Raw( + "SELECT space_quota, original_space_quota FROM users WHERE name = ? AND deleted_at IS NULL", + user, + ).Scan(&row).Error; err != nil { + resputil.HandleError(c, bizerr.Internal.DatabaseError.Wrap(err, "failed to query user storage quota")) + return + } + + if row.OriginalSpaceQuota != nil { + resputil.HandleError(c, bizerr.Conflict.ResourceStatusError.New( + "the user already has a temporary quota expansion; revert it before expanding again", + )) + return + } + + newQuota := row.SpaceQuota + req.ExpandBytes + + // 保存原始配额，并更新为新配额，同时设置 jobs_frozen + if err := db.Exec( + "UPDATE users "+ + "SET original_space_quota = space_quota, space_quota = ?, jobs_frozen = ?, "+ + "shrink_stage = ?, shrink_stage_updated_at = NOW() "+ + "WHERE name = ? AND deleted_at IS NULL", + newQuota, req.FreezeNewJobs, "expanded", user, + ).Error; err != nil { + resputil.HandleError(c, bizerr.Internal.DatabaseError.Wrap(err, "failed to apply temporary quota expansion")) + return + } + + // 同步到 CephFS + var userInfo model.User + if err := db.Where("name = ?", user).First(&userInfo).Error; err == nil { + cfg := config.GetConfig() + prefixConfig := ceph.StoragePrefixConfig{ + User: cfg.Storage.Prefix.User, + Account: cfg.Storage.Prefix.Account, + Public: cfg.Storage.Prefix.Public, + } + if cephErr := ceph.SetCephDirectoryQuota( + mgr.kubeClient, + mgr.kubeConfig, + ceph.StorageQuotaRookNamespace(), + fmt.Sprintf("/user/%s", userInfo.Space), + prefixConfig, + newQuota, + ); cephErr != nil { + klog.Errorf("ApplyExpansion: 设置用户 %s Ceph 配额失败: %v", user, cephErr) + } + } + if req.DecisionJobID != "" { + action := "manual_expand" + if req.FreezeNewJobs { + action = "manual_expand_and_freeze" + } + _ = storagegovernance.MarkDecisionExecution(c.Request.Context(), req.DecisionJobID, action, nil) + } + + resputil.Success(c, gin.H{ + "user": user, + "original_quota": row.SpaceQuota, + "new_quota": newQuota, + "original_quota_formatted": formatSize(row.SpaceQuota), + "new_quota_formatted": formatSize(newQuota), + "jobs_frozen": req.FreezeNewJobs, + }) +} + +// RevertExpansion godoc +// +// @Summary Revert temporary storage expansion for a user +// @Description Restore the user's quota to the original value before expansion +// @Tags Storage +// @Accept json +// @Produce json +// @Security Bearer +// @Param user path string true "Username" +// @Success 200 {object} resputil.Response[any] "Success" +// @Failure 400 {object} resputil.Response[any] "Request parameter error" +// @Failure 500 {object} resputil.Response[any] "Other errors" +func (mgr *StorageMgr) RevertExpansion(c *gin.Context) { + user := c.Param("user") + if user == "" { + resputil.HandleError(c, bizerr.BadRequest.MissingParameter.New("username is required")) + return + } + + db := query.GetDB() + + var row struct { + SpaceQuota int64 `gorm:"column:space_quota"` + OriginalSpaceQuota *int64 `gorm:"column:original_space_quota"` + } + if err := db.Raw( + "SELECT space_quota, original_space_quota FROM users WHERE name = ? AND deleted_at IS NULL", + user, + ).Scan(&row).Error; err != nil { + resputil.HandleError(c, bizerr.Internal.DatabaseError.Wrap(err, "failed to query user storage quota")) + return + } + + if row.OriginalSpaceQuota == nil { + resputil.HandleError(c, bizerr.Conflict.ResourceStatusError.New( + "the user does not have a temporary quota expansion to revert", + )) + return + } + + originalQuota := *row.OriginalSpaceQuota + + // 查询用户 ID 和当前实际用量，决定是否同时解冻 + var userIDRow struct { + ID uint `gorm:"column:id"` + } + db.Raw("SELECT id FROM users WHERE name = ? AND deleted_at IS NULL", user).Scan(&userIDRow) + + var currentSize int64 + var spaceSize model.UserSpaceSize + if err := db.Where("user_id = ?", userIDRow.ID).First(&spaceSize).Error; err == nil { + currentSize = spaceSize.Size + } + + // 只有还原后的理论配额大于当前用量时才自动解冻；否则保持冻结状态 + shouldUnfreeze := originalQuota <= 0 || currentSize < originalQuota + if shouldUnfreeze { + if err := db.Exec( + "UPDATE users "+ + "SET space_quota = ?, original_space_quota = NULL, jobs_frozen = false, "+ + "shrink_stage = NULL, shrink_stage_updated_at = NULL "+ + "WHERE name = ? AND deleted_at IS NULL", + originalQuota, + user, + ).Error; err != nil { + resputil.HandleError(c, bizerr.Internal.DatabaseError.Wrap(err, "failed to revert storage quota")) + return + } + } else { + // 仅还原配额，不解冻（用量仍超出理论配额） + if err := db.Exec( + "UPDATE users "+ + "SET space_quota = ?, original_space_quota = NULL, "+ + "shrink_stage = NULL, shrink_stage_updated_at = NULL "+ + "WHERE name = ? AND deleted_at IS NULL", + originalQuota, + user, + ).Error; err != nil { + resputil.HandleError(c, bizerr.Internal.DatabaseError.Wrap(err, "failed to revert storage quota")) + return + } + } + + // 同步到 CephFS + var userInfo model.User + if err := db.Where("name = ?", user).First(&userInfo).Error; err == nil { + cfg := config.GetConfig() + prefixConfig := ceph.StoragePrefixConfig{ + User: cfg.Storage.Prefix.User, + Account: cfg.Storage.Prefix.Account, + Public: cfg.Storage.Prefix.Public, + } + if cephErr := ceph.SetCephDirectoryQuota( + mgr.kubeClient, + mgr.kubeConfig, + ceph.StorageQuotaRookNamespace(), + fmt.Sprintf("/user/%s", userInfo.Space), + prefixConfig, + originalQuota, + ); cephErr != nil { + klog.Errorf("RevertExpansion: 设置用户 %s Ceph 配额失败: %v", user, cephErr) + } + } + + resputil.Success(c, gin.H{ + "user": user, + "reverted_quota": originalQuota, + "reverted_quota_formatted": formatSize(originalQuota), + "jobs_unfrozen": shouldUnfreeze, + }) +} + +// UnfreezeJobs godoc +// +// @Summary Manually unfreeze job creation for a user +// @Description Clear the jobs_frozen flag, allowing the user to create new jobs again +// @Tags Storage +// @Produce json +// @Security Bearer +// @Param user path string true "Username" +// @Success 200 {object} resputil.Response[any] "Success" +// @Failure 400 {object} resputil.Response[any] "Request parameter error" +// @Failure 500 {object} resputil.Response[any] "Other errors" +func (mgr *StorageMgr) UnfreezeJobs(c *gin.Context) { + user := c.Param("user") + if user == "" { + resputil.HandleError(c, bizerr.BadRequest.MissingParameter.New("username is required")) + return + } + + db := query.GetDB() + if err := db.Exec("UPDATE users SET jobs_frozen = false WHERE name = ? AND deleted_at IS NULL", user).Error; err != nil { + resputil.HandleError(c, bizerr.Internal.DatabaseError.Wrap(err, "failed to unfreeze user jobs")) + return + } + + resputil.Success(c, gin.H{"user": user, "jobs_frozen": false}) +} + +// FreezeJobs manually freezes job creation for a user and optionally binds the action to a decision record. +func (mgr *StorageMgr) FreezeJobs(c *gin.Context) { + user := c.Param("user") + if user == "" { + resputil.HandleError(c, bizerr.BadRequest.MissingParameter.New("username is required")) + return + } + + var req struct { + DecisionJobID string `json:"decision_job_id"` + } + if err := c.ShouldBindJSON(&req); err != nil && err.Error() != "EOF" { + resputil.HandleError(c, bizerr.BadRequest.InvalidRequest.Wrap(err, "invalid request body")) + return + } + + db := query.GetDB() + if err := db.Exec("UPDATE users SET jobs_frozen = true WHERE name = ? AND deleted_at IS NULL", user).Error; err != nil { + if req.DecisionJobID != "" { + _ = storagegovernance.MarkDecisionExecution(c.Request.Context(), req.DecisionJobID, "manual_freeze_failed", err) + } + resputil.HandleError(c, bizerr.Internal.DatabaseError.Wrap(err, "failed to freeze user jobs")) + return + } + + if req.DecisionJobID != "" { + _ = storagegovernance.MarkDecisionExecution(c.Request.Context(), req.DecisionJobID, "manual_freeze", nil) + } + + resputil.Success(c, gin.H{"user": user, "jobs_frozen": true}) +} + +// TriggerLLMDecision godoc +// +// @Summary Trigger LLM storage expansion decision for a user +// @Description Calls Claude agent to analyze whether a user needs temporary storage expansion +// @Tags Storage +// @Accept json +// @Produce json +// @Security Bearer +// @Param user path string true "Username" +// @Success 200 {object} resputil.Response[any] "Success" +// @Failure 500 {object} resputil.Response[any] "Other errors" +// TriggerLLMDecision 异步启动 LLM 分析，立即返回 job_id +func (mgr *StorageMgr) TriggerLLMDecision(c *gin.Context) { + user := c.Param("user") + if user == "" { + resputil.HandleError(c, bizerr.BadRequest.MissingParameter.New("username is required")) + return + } + + engine := storagegovernance.NewEngine( + mgr.kubeClient, + mgr.kubeConfig, + mgr.promClient, + storagegovernance.DefaultConstraintConfig(), + ) + jobID, err := engine.StartAsyncDecision(context.Background(), storagegovernance.DecisionRequest{ + Username: user, + Source: model.StorageDecisionSourceManual, + TriggerReason: "manual llm decision request", + }) + if err != nil { + klog.Errorf("TriggerLLMDecision: user=%s err=%v", user, err) + resputil.HandleError(c, bizerr.Internal.ServiceError.Wrap(err, "failed to start storage decision analysis")) + return + } + + resputil.Success(c, gin.H{"job_id": jobID}) +} + +// GetLLMDecisionStatus 查询 LLM 分析任务状态 +func (mgr *StorageMgr) GetLLMDecisionStatus(c *gin.Context) { + jobID := c.Param("job_id") + + job, err := storagegovernance.GetDecisionStatus(c.Request.Context(), jobID) + + if err != nil { + resputil.HandleError(c, bizerr.NotFound.DataBaseNotFound.Wrap(err, "storage decision job was not found")) + return + } + + resputil.Success(c, job) +} + +// formatSize 格式化大小为人类可读格式 +// ListStorageDecisions returns paginated persisted storage decision records. +func (mgr *StorageMgr) ListStorageDecisions(c *gin.Context) { + page, _ := strconv.Atoi(c.DefaultQuery("page", "1")) + pageSize, _ := strconv.Atoi(c.DefaultQuery("pageSize", "20")) + + result, err := storagegovernance.ListDecisionRecords( + c.Request.Context(), + page, + pageSize, + c.Query("user"), + c.Query("status"), + c.Query("source"), + ) + if err != nil { + resputil.HandleError(c, bizerr.Internal.DatabaseError.Wrap(err, "failed to list storage decisions")) + return + } + + resputil.Success(c, result) +} + +// GetStorageDecision returns one persisted storage decision record with full details. +func (mgr *StorageMgr) GetStorageDecision(c *gin.Context) { + jobID := c.Param("job_id") + if jobID == "" { + resputil.HandleError(c, bizerr.BadRequest.MissingParameter.New("job_id is required")) + return + } + + result, err := storagegovernance.GetDecisionRecord(c.Request.Context(), jobID) + if err != nil { + resputil.HandleError(c, bizerr.NotFound.DataBaseNotFound.Wrap(err, "storage decision was not found")) + return + } + + resputil.Success(c, result) +} + +// ReplayStorageDecisions re-evaluates stored decisions under the current or overridden safety policy. +func (mgr *StorageMgr) ReplayStorageDecisions(c *gin.Context) { + var req struct { + Limit int `json:"limit"` + MaxExpandRatio *float64 `json:"max_expand_ratio"` + MaxExpandBytes *int64 `json:"max_expand_bytes"` + MinPlatformReservedRatio *float64 `json:"min_platform_reserved_ratio"` + MinPlatformReservedBytes *int64 `json:"min_platform_reserved_bytes"` + ExpansionCooldownHours *int `json:"expansion_cooldown_hours"` + ForceFreezeWhenOverQuota *bool `json:"force_freeze_when_over_quota"` + } + if err := c.ShouldBindJSON(&req); err != nil && err.Error() != "EOF" { + resputil.HandleError(c, bizerr.BadRequest.InvalidRequest.Wrap(err, "invalid replay request")) + return + } + + cfg := storagegovernance.DefaultConstraintConfig() + if req.MaxExpandRatio != nil { + cfg.MaxExpandRatio = *req.MaxExpandRatio + } + if req.MaxExpandBytes != nil { + cfg.MaxExpandBytes = *req.MaxExpandBytes + } + if req.MinPlatformReservedRatio != nil { + cfg.MinPlatformReservedRatio = *req.MinPlatformReservedRatio + } + if req.MinPlatformReservedBytes != nil { + cfg.MinPlatformReservedBytes = *req.MinPlatformReservedBytes + } + if req.ExpansionCooldownHours != nil { + cfg.ExpansionCooldown = time.Duration(*req.ExpansionCooldownHours) * time.Hour + } + if req.ForceFreezeWhenOverQuota != nil { + cfg.ForceFreezeWhenOverQuota = *req.ForceFreezeWhenOverQuota + } + + summary, err := storagegovernance.ReplayStoredDecisions(c.Request.Context(), cfg, req.Limit) + if err != nil { + resputil.HandleError(c, bizerr.Internal.ServiceError.Wrap(err, "failed to replay storage decisions")) + return + } + + resputil.Success(c, summary) +} + +func formatSize(bytes int64) string { + const unit = 1024 + if bytes < 0 { + return "Unknown" + } + if bytes == 0 { + return "0 B" + } + if bytes < unit { + return fmt.Sprintf("%d B", bytes) + } + div, exp := int64(unit), 0 + for n := bytes / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp]) +} diff --git a/backend/internal/handler/system_config.go b/backend/internal/handler/system_config.go index b0e9173fc..0dddd26e8 100644 --- a/backend/internal/handler/system_config.go +++ b/backend/internal/handler/system_config.go @@ -58,7 +58,7 @@ func (mgr *SystemConfigMgr) RegisterAdmin(g *gin.RouterGroup) { // 路由组: /v1/admin/system-config g.GET("/llm", mgr.GetLLMConfig) g.PUT("/llm", mgr.UpdateLLMConfig) - // 新增：重置 LLM 配置 + // 重置平台通用 LLM 配置 g.DELETE("/llm", mgr.ResetLLMConfig) g.GET("/gpu-analysis", mgr.GetGpuAnalysisStatus) @@ -89,7 +89,7 @@ type UpdateLLMConfigReq struct { BaseURL string `json:"baseUrl" binding:"required"` APIKey string `json:"apiKey"` ModelName string `json:"modelName" binding:"required"` - Validate bool `json:"validate"` // 是否立即校验连接 + Validate bool `json:"validate"` } type GpuAnalysisStatusResp struct { @@ -237,7 +237,7 @@ func (mgr *SystemConfigMgr) GetLLMConfig(c *gin.Context) { // UpdateLLMConfig godoc // @Summary 更新 LLM 配置 -// @Description 更新 LLM 的连接信息。如果 validate 为 true，会尝试连接 /check 接口，失败则不保存。 +// @Description 更新 LLM 的连接信息。如果 validate 为 true，会尝试连接 /models 接口，失败则不保存。 // @Tags SystemConfig // @Accept json // @Produce json diff --git a/backend/internal/handler/vcjob/custom.go b/backend/internal/handler/vcjob/custom.go index 7635bbb1f..f638e546b 100644 --- a/backend/internal/handler/vcjob/custom.go +++ b/backend/internal/handler/vcjob/custom.go @@ -60,6 +60,12 @@ func (mgr *VolcanojobMgr) CreateTrainingJob(c *gin.Context) { resputil.Error(c, err.Error(), resputil.ServiceError) return } + + if err := util.CheckStorageQuota(token.Username); err != nil { + resputil.HandleError(c, err) + return + } + if !mgr.preCheckCreateJob(c, token, scheduleType, false) { return } diff --git a/backend/internal/handler/vcjob/jupyter.go b/backend/internal/handler/vcjob/jupyter.go index 40e59a07c..23374395a 100644 --- a/backend/internal/handler/vcjob/jupyter.go +++ b/backend/internal/handler/vcjob/jupyter.go @@ -70,6 +70,12 @@ func (mgr *VolcanojobMgr) CreateJupyterJob(c *gin.Context) { resputil.Error(c, err.Error(), resputil.ServiceError) return } + + if err := util.CheckStorageQuota(token.Username); err != nil { + resputil.HandleError(c, err) + return + } + if !mgr.preCheckCreateJob(c, token, scheduleType, true) { return } diff --git a/backend/internal/handler/vcjob/pytorch.go b/backend/internal/handler/vcjob/pytorch.go index b5ba5b523..c71779a9b 100644 --- a/backend/internal/handler/vcjob/pytorch.go +++ b/backend/internal/handler/vcjob/pytorch.go @@ -17,6 +17,7 @@ import ( "github.com/raids-lab/crater/pkg/vcqueue" ) +//nolint:gocyclo // Job assembly coordinates validation, queues, and task generation in one request flow. func (mgr *VolcanojobMgr) CreatePytorchJob(c *gin.Context) { token := util.GetToken(c) @@ -30,6 +31,12 @@ func (mgr *VolcanojobMgr) CreatePytorchJob(c *gin.Context) { resputil.BadRequestError(c, err.Error()) return } + + if err := util.CheckStorageQuota(token.Username); err != nil { + resputil.HandleError(c, err) + return + } + if !mgr.preCheckCreateJob(c, token, scheduleType, false) { return } diff --git a/backend/internal/handler/vcjob/tensorflow.go b/backend/internal/handler/vcjob/tensorflow.go index 0e86c96cb..0f95eb3cc 100644 --- a/backend/internal/handler/vcjob/tensorflow.go +++ b/backend/internal/handler/vcjob/tensorflow.go @@ -66,6 +66,12 @@ func (mgr *VolcanojobMgr) CreateTensorflowJob(c *gin.Context) { resputil.BadRequestError(c, err.Error()) return } + + if err := util.CheckStorageQuota(token.Username); err != nil { + resputil.HandleError(c, err) + return + } + if !mgr.preCheckCreateJob(c, token, scheduleType, false) { return } diff --git a/backend/internal/handler/vcjob/webide.go b/backend/internal/handler/vcjob/webide.go index 5aa1014ec..4edf1da93 100644 --- a/backend/internal/handler/vcjob/webide.go +++ b/backend/internal/handler/vcjob/webide.go @@ -55,6 +55,12 @@ func (mgr *VolcanojobMgr) CreateWebIDEJob(c *gin.Context) { resputil.Error(c, err.Error(), resputil.ServiceError) return } + + if err := util.CheckStorageQuota(token.Username); err != nil { + resputil.HandleError(c, err) + return + } + if !mgr.preCheckCreateJob(c, token, scheduleType, true) { return } diff --git a/backend/internal/service/config_service.go b/backend/internal/service/config_service.go index ae73d7d86..c6ef06e66 100644 --- a/backend/internal/service/config_service.go +++ b/backend/internal/service/config_service.go @@ -28,6 +28,12 @@ import ( // 定义掩码常量 const MaskedAPIKeyPlaceholder = "********************************************" +const DefaultStorageDirectModelBaseURL = "" + +const ( + StorageDecisionConfigSourcePlatform = "platform" + StorageDecisionConfigSourceCustom = "custom" +) const ( DefaultModelDownloadMaxConcurrent = 5 @@ -52,6 +58,14 @@ type ModelDownloadLimitConfig struct { WhitelistUserIDs []uint } +type StorageDecisionConfig struct { + DecisionMode string + ConfigSource string + BaseURL string + APIKey string + ModelName string +} + // cleanBaseURL 内部辅助：清理 URL 结尾的斜杠 func (c *LLMConfig) cleanBaseURL() string { return strings.TrimSuffix(strings.TrimSpace(c.BaseURL), "/") @@ -131,6 +145,12 @@ func defaultSystemConfigValue(key string) string { return FormatBillingAmountConfigValue(defaultBillingIssueAmount) case model.ConfigKeyBillingDefaultIssuePeriodMinute: return "43200" + case model.ConfigKeyStorageDecisionMode: + return "agent" + case model.ConfigKeyStorageDecisionConfigSource: + return StorageDecisionConfigSourcePlatform + case model.ConfigKeyStorageDirectModelBaseURL: + return DefaultStorageDirectModelBaseURL default: return "" } @@ -267,7 +287,12 @@ func (s *ConfigService) updateConfigs(ctx context.Context, updates map[string]st // GetLLMConfig 从数据库按需读取最新配置 func (s *ConfigService) GetLLMConfig(ctx context.Context) (*LLMConfig, error) { - configMap, err := s.getConfigs(ctx, model.ConfigKeyLLMBaseURL, model.ConfigKeyLLMAPIKey, model.ConfigKeyLLMModelName) + configMap, err := s.getConfigs( + ctx, + model.ConfigKeyLLMBaseURL, + model.ConfigKeyLLMAPIKey, + model.ConfigKeyLLMModelName, + ) if err != nil { return nil, err } @@ -293,6 +318,43 @@ func (s *ConfigService) GetLLMConfig(ctx context.Context) (*LLMConfig, error) { }, nil } +func (s *ConfigService) GetStorageDecisionConfig(ctx context.Context) (*StorageDecisionConfig, error) { + configMap, err := s.getConfigs( + ctx, + model.ConfigKeyStorageDecisionMode, + model.ConfigKeyStorageDecisionConfigSource, + model.ConfigKeyStorageDirectModelBaseURL, + model.ConfigKeyStorageDirectModelAPIKey, + model.ConfigKeyStorageDirectModelName, + ) + if err != nil { + return nil, err + } + + directEncryptedKey := configMap[model.ConfigKeyStorageDirectModelAPIKey] + directPlainKey := "" + if directEncryptedKey != "" { + decrypted, decryptErr := crypto.Decrypt(directEncryptedKey) + if decryptErr != nil { + klog.Errorf("Failed to decrypt direct API Key: %v, assuming plain text or empty", decryptErr) + directPlainKey = directEncryptedKey + } else { + directPlainKey = decrypted + } + } + + return &StorageDecisionConfig{ + DecisionMode: normalizeDecisionMode(configMap[model.ConfigKeyStorageDecisionMode]), + ConfigSource: normalizeStorageDecisionConfigSource( + configMap[model.ConfigKeyStorageDecisionConfigSource], + configMap[model.ConfigKeyStorageDirectModelName], + ), + BaseURL: configMap[model.ConfigKeyStorageDirectModelBaseURL], + APIKey: directPlainKey, + ModelName: configMap[model.ConfigKeyStorageDirectModelName], + }, nil +} + // CheckLLMConnection 使用 /models 接口进行校验，并验证 ModelName 是否存在 func (s *ConfigService) CheckLLMConnection(ctx context.Context, cfg *LLMConfig) error { checkURL := cfg.GetCheckURL() @@ -358,6 +420,52 @@ func (s *ConfigService) CheckLLMConnection(ctx context.Context, cfg *LLMConfig) return nil } +func normalizeDecisionMode(mode string) string { + switch strings.TrimSpace(strings.ToLower(mode)) { + case "direct": + return "direct" + default: + return "agent" + } +} + +func normalizeStorageDecisionConfigSource(source, modelName string) string { + switch strings.TrimSpace(strings.ToLower(source)) { + case StorageDecisionConfigSourceCustom: + return StorageDecisionConfigSourceCustom + case StorageDecisionConfigSourcePlatform: + return StorageDecisionConfigSourcePlatform + default: + if strings.TrimSpace(modelName) != "" { + return StorageDecisionConfigSourceCustom + } + return StorageDecisionConfigSourcePlatform + } +} + +func (s *ConfigService) CheckStorageDecisionConnection( + ctx context.Context, + llmCfg *LLMConfig, + cfg *StorageDecisionConfig, +) error { + if normalizeStorageDecisionConfigSource(cfg.ConfigSource, cfg.ModelName) == StorageDecisionConfigSourceCustom { + customCfg := &LLMConfig{ + BaseURL: cfg.BaseURL, + APIKey: cfg.APIKey, + ModelName: cfg.ModelName, + } + return s.CheckLLMConnection(ctx, customCfg) + } + if llmCfg == nil { + var err error + llmCfg, err = s.GetLLMConfig(ctx) + if err != nil { + return err + } + } + return s.CheckLLMConnection(ctx, llmCfg) +} + // SetGpuAnalysisEnabled 设置GPU分析功能的开关，并同步创建或更新定时任务的状态 func (s *ConfigService) SetGpuAnalysisEnabled(c *gin.Context, enable bool) error { var ctx = c.Request.Context() @@ -473,28 +581,30 @@ func (s *ConfigService) ResetLLMConfig(ctx context.Context) error { }) } -// UpdateLLMConfig 更新配置 -func (s *ConfigService) UpdateLLMConfig(ctx context.Context, reqCfg *LLMConfig, validate bool) error { - finalKeyToSave := "" - - if reqCfg.APIKey == MaskedAPIKeyPlaceholder { - oldConfigRaw, err := s.getConfigs(ctx, model.ConfigKeyLLMAPIKey) - if err == nil { - finalKeyToSave = oldConfigRaw[model.ConfigKeyLLMAPIKey] +func (s *ConfigService) ResetStorageDecisionConfig(ctx context.Context) error { + updates := map[string]string{ + model.ConfigKeyStorageDecisionMode: "agent", + model.ConfigKeyStorageDecisionConfigSource: StorageDecisionConfigSourcePlatform, + model.ConfigKeyStorageDirectModelBaseURL: DefaultStorageDirectModelBaseURL, + model.ConfigKeyStorageDirectModelAPIKey: "", + model.ConfigKeyStorageDirectModelName: "", + } - if validate { - plainKey, err := crypto.Decrypt(finalKeyToSave) - if err == nil { - reqCfg.APIKey = plainKey - } + return s.q.Transaction(func(tx *query.Query) error { + for k, v := range updates { + if _, err := tx.SystemConfig.WithContext(ctx).Where(tx.SystemConfig.Key.Eq(k)).Update(tx.SystemConfig.Value, v); err != nil { + return err } } - } else { - encrypted, err := crypto.Encrypt(reqCfg.APIKey) - if err != nil { - return fmt.Errorf("failed to encrypt api key: %w", err) - } - finalKeyToSave = encrypted + return nil + }) +} + +// UpdateLLMConfig 更新配置 +func (s *ConfigService) UpdateLLMConfig(ctx context.Context, reqCfg *LLMConfig, validate bool) error { + finalLLMKeyToSave, err := s.prepareSecretForSave(ctx, model.ConfigKeyLLMAPIKey, reqCfg.APIKey) + if err != nil { + return err } if validate { @@ -505,7 +615,7 @@ func (s *ConfigService) UpdateLLMConfig(ctx context.Context, reqCfg *LLMConfig, updates := map[string]any{ model.ConfigKeyLLMBaseURL: reqCfg.BaseURL, - model.ConfigKeyLLMAPIKey: finalKeyToSave, + model.ConfigKeyLLMAPIKey: finalLLMKeyToSave, model.ConfigKeyLLMModelName: reqCfg.ModelName, } @@ -519,6 +629,73 @@ func (s *ConfigService) UpdateLLMConfig(ctx context.Context, reqCfg *LLMConfig, }) } +func (s *ConfigService) UpdateStorageDecisionConfig( + ctx context.Context, + reqCfg *StorageDecisionConfig, + validate bool, +) error { + reqCfg.DecisionMode = normalizeDecisionMode(reqCfg.DecisionMode) + reqCfg.ConfigSource = normalizeStorageDecisionConfigSource(reqCfg.ConfigSource, reqCfg.ModelName) + + finalKeyToSave, err := s.prepareSecretForSave(ctx, model.ConfigKeyStorageDirectModelAPIKey, reqCfg.APIKey) + if err != nil { + return err + } + + if validate { + llmCfg, getErr := s.GetLLMConfig(ctx) + if getErr != nil { + return getErr + } + if err := s.CheckStorageDecisionConnection(ctx, llmCfg, reqCfg); err != nil { + return bizerr.BadRequest.ParameterError.Wrap(err, "storage decision configuration validation failed") + } + } + + updates := map[string]any{ + model.ConfigKeyStorageDecisionMode: reqCfg.DecisionMode, + model.ConfigKeyStorageDecisionConfigSource: reqCfg.ConfigSource, + model.ConfigKeyStorageDirectModelBaseURL: reqCfg.BaseURL, + model.ConfigKeyStorageDirectModelAPIKey: finalKeyToSave, + model.ConfigKeyStorageDirectModelName: reqCfg.ModelName, + } + + return s.q.Transaction(func(tx *query.Query) error { + for k, v := range updates { + if _, err := tx.SystemConfig.WithContext(ctx).Where(tx.SystemConfig.Key.Eq(k)).Update(tx.SystemConfig.Value, v); err != nil { + return err + } + } + return nil + }) +} + +func (s *ConfigService) prepareSecretForSave( + ctx context.Context, + configKey string, + targetValue string, +) (string, error) { + var currentValue string + + if targetValue == MaskedAPIKeyPlaceholder { + oldConfigRaw, err := s.getConfigs(ctx, configKey) + if err == nil { + currentValue = oldConfigRaw[configKey] + } + return currentValue, nil + } + + if targetValue == "" { + return "", nil + } + + encrypted, err := crypto.Encrypt(targetValue) + if err != nil { + return "", bizerr.Internal.ServiceError.Wrap(err, "failed to encrypt API key") + } + return encrypted, nil +} + // getConfigs 辅助方法 func (s *ConfigService) getConfigs(ctx context.Context, keys ...string) (map[string]string, error) { sc := s.q.SystemConfig diff --git a/backend/internal/storage/file.go b/backend/internal/storage/file.go index eed53600c..1e63e1ffa 100644 --- a/backend/internal/storage/file.go +++ b/backend/internal/storage/file.go @@ -572,7 +572,7 @@ func resolveDatasetStoragePath(datasetURL, relativeURL string) (string, error) { if relativePath == "" { return basePath, nil } - if relativePath == ".." || strings.HasPrefix(relativePath, "../") { + if relativePath == parentDirectoryPath || strings.HasPrefix(relativePath, parentDirectoryPath+"/") { return "", errors.New("dataset path must stay within the resource directory") } return urlpath.Join(basePath, relativePath), nil diff --git a/backend/internal/storage/quota.go b/backend/internal/storage/quota.go new file mode 100644 index 000000000..7e628bf7e --- /dev/null +++ b/backend/internal/storage/quota.go @@ -0,0 +1,192 @@ +package storage + +import ( + "errors" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/gin-gonic/gin" + + "github.com/raids-lab/crater/pkg/config" + "github.com/raids-lab/crater/pkg/storagequota" +) + +const ( + cephDirectoryBytesXattr = "ceph.dir.rbytes" + cephQuotaMaxBytesXattr = "ceph.quota.max_bytes" + parentDirectoryPath = ".." +) + +func RegisterQuotaRoutes(r *gin.Engine) { + group := r.Group("/internal/storage", requireInternalStorageToken()) + group.GET("/capabilities", GetQuotaCapabilities) + group.GET("/usage", GetDirectoryUsage) + group.GET("/quota", GetDirectoryQuota) + group.PUT("/quota", SetDirectoryQuota) +} + +func requireInternalStorageToken() gin.HandlerFunc { + return func(c *gin.Context) { + suppliedToken := c.GetHeader(storagequota.InternalTokenHeader) + if token := strings.TrimSpace(os.Getenv(storagequota.InternalTokenEnv)); token != "" { + if !storagequota.AuthenticateToken(token, suppliedToken) { + c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "invalid internal storage token"}) + return + } + c.Next() + return + } + + secret := strings.TrimSpace(os.Getenv(storagequota.InternalSecretEnv)) + if secret == "" { + secret = config.GetConfig().Auth.Token.AccessTokenSecret + } + if secret == "" || !storagequota.Authenticate(secret, suppliedToken) { + c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "invalid internal storage token"}) + return + } + c.Next() + } +} + +func GetQuotaCapabilities(c *gin.Context) { + capabilities := inspectXattrCapabilities(storageRootDir) + c.JSON(http.StatusOK, capabilities) +} + +func GetDirectoryUsage(c *gin.Context) { + targetPath, relativePath, err := resolveStorageUsagePath(c.Query("path")) + if err != nil { + writeStoragePathError(c, err) + return + } + + value, err := readXattrInt64(targetPath, cephDirectoryBytesXattr) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, storagequota.Usage{Path: relativePath, Bytes: value}) +} + +//nolint:gocritic // The tuple distinguishes the resolved filesystem path from the API-relative path. +func resolveStorageUsagePath(rawPath string) (string, string, error) { + if strings.TrimSpace(rawPath) != "." { + return resolveStorageDirectory(rawPath) + } + + root, err := filepath.Abs(storageRootDir) + if err != nil { + return "", "", err + } + root, _, err = secureStoragePaths(root, root) + if err != nil { + return "", "", err + } + return root, ".", nil +} + +func GetDirectoryQuota(c *gin.Context) { + targetPath, relativePath, err := resolveStorageDirectory(c.Query("path")) + if err != nil { + writeStoragePathError(c, err) + return + } + + value, err := readXattrInt64(targetPath, cephQuotaMaxBytesXattr) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + if value == 0 { + value = -1 + } + c.JSON(http.StatusOK, storagequota.Quota{Path: relativePath, MaxBytes: value}) +} + +func SetDirectoryQuota(c *gin.Context) { + var request storagequota.Quota + if err := c.ShouldBindJSON(&request); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid quota request: " + err.Error()}) + return + } + if request.MaxBytes < -1 || request.MaxBytes == 0 { + c.JSON(http.StatusBadRequest, gin.H{"error": "max_bytes must be -1 or greater than zero"}) + return + } + + targetPath, relativePath, err := resolveStorageDirectory(request.Path) + if err != nil { + writeStoragePathError(c, err) + return + } + + maxBytes := request.MaxBytes + if maxBytes == -1 { + maxBytes = 0 + } + if err := writeXattr(targetPath, cephQuotaMaxBytesXattr, []byte(strconv.FormatInt(maxBytes, 10))); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, storagequota.Quota{Path: relativePath, MaxBytes: request.MaxBytes}) +} + +//nolint:gocritic // The tuple distinguishes the resolved filesystem path from the API-relative path. +func resolveStorageDirectory(rawPath string) (string, string, error) { + rawPath = strings.TrimSpace(strings.ReplaceAll(rawPath, "\\", "/")) + if rawPath == "" || strings.ContainsRune(rawPath, '\x00') { + return "", "", errInvalidStoragePath + } + if strings.HasPrefix(rawPath, "/") { + return "", "", errInvalidStoragePath + } + + cleanRelative := filepath.Clean(filepath.FromSlash(rawPath)) + if cleanRelative == "." || cleanRelative == parentDirectoryPath || filepath.IsAbs(cleanRelative) || + strings.HasPrefix(cleanRelative, parentDirectoryPath+string(filepath.Separator)) { + return "", "", errInvalidStoragePath + } + + root, err := filepath.Abs(storageRootDir) + if err != nil { + return "", "", err + } + target := filepath.Join(root, cleanRelative) + root, target, err = secureStoragePaths(root, target) + if err != nil { + return "", "", err + } + relativeToRoot, err := filepath.Rel(root, target) + if err != nil || relativeToRoot == ".." || strings.HasPrefix(relativeToRoot, ".."+string(filepath.Separator)) { + return "", "", errInvalidStoragePath + } + + info, err := os.Stat(target) + if err != nil { + return "", "", err + } + if !info.IsDir() { + return "", "", errStoragePathNotDirectory + } + return target, filepath.ToSlash(relativeToRoot), nil +} + +var ( + errInvalidStoragePath = errors.New("storage path must be a relative path below the storage root") + errStoragePathNotDirectory = errors.New("storage path is not a directory") +) + +func writeStoragePathError(c *gin.Context, err error) { + switch { + case errors.Is(err, errInvalidStoragePath), errors.Is(err, errStoragePathNotDirectory): + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + case os.IsNotExist(err): + c.JSON(http.StatusNotFound, gin.H{"error": err.Error()}) + default: + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + } +} diff --git a/backend/internal/storage/quota_test.go b/backend/internal/storage/quota_test.go new file mode 100644 index 000000000..5d4824ec4 --- /dev/null +++ b/backend/internal/storage/quota_test.go @@ -0,0 +1,72 @@ +package storage + +import ( + "errors" + "os" + "path/filepath" + "testing" +) + +func TestResolveStorageDirectory(t *testing.T) { + root := t.TempDir() + originalRoot := storageRootDir + storageRootDir = root + t.Cleanup(func() { storageRootDir = originalRoot }) + + validDirectory := filepath.Join(root, "users", "alice") + if err := os.MkdirAll(validDirectory, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "users", "file.txt"), []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + path string + wantRel string + wantErr error + }{ + {name: "valid", path: "users/alice", wantRel: "users/alice"}, + {name: "absolute", path: "/users/alice", wantErr: errInvalidStoragePath}, + {name: "traversal", path: "../outside", wantErr: errInvalidStoragePath}, + {name: "file", path: "users/file.txt", wantErr: errStoragePathNotDirectory}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, relative, err := resolveStorageDirectory(tt.path) + if tt.wantErr != nil && !errors.Is(err, tt.wantErr) { + t.Fatalf("error = %v, want %v", err, tt.wantErr) + } + if tt.wantErr == nil && err != nil { + t.Fatalf("unexpected error: %v", err) + } + if relative != tt.wantRel { + t.Fatalf("relative path = %q, want %q", relative, tt.wantRel) + } + }) + } +} + +func TestResolveStorageUsagePathAllowsRoot(t *testing.T) { + root := t.TempDir() + originalRoot := storageRootDir + storageRootDir = root + t.Cleanup(func() { storageRootDir = originalRoot }) + + target, relative, err := resolveStorageUsagePath(".") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + wantTarget, err := filepath.Abs(root) + if err != nil { + t.Fatal(err) + } + if target != wantTarget { + t.Fatalf("target path = %q, want %q", target, wantTarget) + } + if relative != "." { + t.Fatalf("relative path = %q, want %q", relative, ".") + } +} diff --git a/backend/internal/storage/quota_xattr_linux.go b/backend/internal/storage/quota_xattr_linux.go new file mode 100644 index 000000000..f6b30abcc --- /dev/null +++ b/backend/internal/storage/quota_xattr_linux.go @@ -0,0 +1,100 @@ +//go:build linux + +package storage + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + + "golang.org/x/sys/unix" + + "github.com/raids-lab/crater/pkg/storagequota" +) + +func inspectXattrCapabilities(rootPath string) storagequota.Capabilities { + capabilities := storagequota.Capabilities{} + if _, err := readXattrInt64(rootPath, cephDirectoryBytesXattr); err != nil { + capabilities.Reasons = append(capabilities.Reasons, err.Error()) + return capabilities + } + + capabilities.UsageReadable = true + _, err := readXattr(rootPath, cephQuotaMaxBytesXattr) + if err != nil { + if !errors.Is(err, unix.ENODATA) { + capabilities.Reasons = append(capabilities.Reasons, err.Error()) + return capabilities + } + } + + capabilities.QuotaReadable = true + testDir, err := os.MkdirTemp(rootPath, ".cephfs-quota-capability-") + if err != nil { + capabilities.Reasons = append(capabilities.Reasons, fmt.Sprintf("create quota capability test directory: %v", err)) + return capabilities + } + defer func() { + _ = os.Remove(testDir) + }() + + // A zero quota is unlimited, so the probe verifies the CephX MDS "p" + // capability without restricting the temporary directory. + if err := writeXattr(testDir, cephQuotaMaxBytesXattr, []byte("0")); err != nil { + capabilities.Reasons = append(capabilities.Reasons, err.Error()) + return capabilities + } + capabilities.QuotaWritable = true + return capabilities +} + +//nolint:gocritic // The tuple returns the validated root and target paths used by the caller. +func secureStoragePaths(rootPath, targetPath string) (string, string, error) { + root, err := filepath.EvalSymlinks(rootPath) + if err != nil { + return "", "", err + } + target, err := filepath.EvalSymlinks(targetPath) + if err != nil { + return "", "", err + } + return root, target, nil +} + +func readXattrInt64(targetPath, name string) (int64, error) { + value, err := readXattr(targetPath, name) + if err != nil { + if errors.Is(err, unix.ENODATA) && name == cephQuotaMaxBytesXattr { + return 0, nil + } + return 0, err + } + parsed, err := strconv.ParseInt(strings.TrimSpace(strings.Trim(string(value), "\x00\"")), 10, 64) + if err != nil { + return 0, fmt.Errorf("parse xattr %s on %s: %w", name, targetPath, err) + } + return parsed, nil +} + +func readXattr(targetPath, name string) ([]byte, error) { + size, err := unix.Getxattr(targetPath, name, nil) + if err != nil { + return nil, fmt.Errorf("read xattr %s on %s: %w", name, targetPath, err) + } + buffer := make([]byte, size) + read, err := unix.Getxattr(targetPath, name, buffer) + if err != nil { + return nil, fmt.Errorf("read xattr %s on %s: %w", name, targetPath, err) + } + return buffer[:read], nil +} + +func writeXattr(targetPath, name string, value []byte) error { + if err := unix.Setxattr(targetPath, name, value, 0); err != nil { + return fmt.Errorf("write xattr %s on %s: %w", name, targetPath, err) + } + return nil +} diff --git a/backend/internal/storage/quota_xattr_unsupported.go b/backend/internal/storage/quota_xattr_unsupported.go new file mode 100644 index 000000000..466173824 --- /dev/null +++ b/backend/internal/storage/quota_xattr_unsupported.go @@ -0,0 +1,37 @@ +//go:build !linux + +package storage + +import ( + "fmt" + "path/filepath" + + "github.com/raids-lab/crater/pkg/storagequota" +) + +//nolint:gocritic // The tuple mirrors the Linux implementation's validated root and target paths. +func secureStoragePaths(rootPath, targetPath string) (string, string, error) { + root, err := filepath.Abs(rootPath) + if err != nil { + return "", "", err + } + target, err := filepath.Abs(targetPath) + if err != nil { + return "", "", err + } + return root, target, nil +} + +func inspectXattrCapabilities(_ string) storagequota.Capabilities { + return storagequota.Capabilities{ + Reasons: []string{"CephFS quota xattrs are only supported by storage-server on Linux"}, + } +} + +func readXattrInt64(_, name string) (int64, error) { + return 0, fmt.Errorf("reading xattr %s is not supported on this operating system", name) +} + +func writeXattr(_, name string, _ []byte) error { + return fmt.Errorf("writing xattr %s is not supported on this operating system", name) +} diff --git a/backend/internal/storage/router.go b/backend/internal/storage/router.go index 3f2879044..204a59358 100644 --- a/backend/internal/storage/router.go +++ b/backend/internal/storage/router.go @@ -3,6 +3,8 @@ package storage import "github.com/gin-gonic/gin" func RegisterRoutes(r *gin.Engine) { + RegisterQuotaRoutes(r) + methods := []string{ "PUT", "MKCOL", diff --git a/backend/internal/util/quota.go b/backend/internal/util/quota.go new file mode 100644 index 000000000..a262c8a55 --- /dev/null +++ b/backend/internal/util/quota.go @@ -0,0 +1,98 @@ +//nolint:lll,mnd // Quota checks keep SQL and percentage thresholds inline for operational clarity. +package util + +import ( + "fmt" + + "k8s.io/klog/v2" + + "github.com/raids-lab/crater/dao/model" + "github.com/raids-lab/crater/dao/query" + "github.com/raids-lab/crater/internal/bizerr" + "github.com/raids-lab/crater/pkg/ceph" +) + +// CheckStorageQuota 检查用户存储是否超过理论配额，或作业是否被管理员冻结。 +// 任一条件成立时返回非 nil 错误，调用方应拒绝创建新作业。 +func CheckStorageQuota(username string) error { + if !ceph.StorageQuotaEnabled() { + return nil + } + + db := query.GetDB() + + // 步骤 1：查用户 ID 和 space_quota + var baseRow struct { + ID uint `gorm:"column:id"` + SpaceQuota int64 `gorm:"column:space_quota"` + } + if err := db.Raw( + "SELECT id, space_quota FROM users WHERE name = ? AND deleted_at IS NULL", + username, + ).Scan(&baseRow).Error; err != nil || baseRow.ID == 0 { + klog.Warningf("CheckStorageQuota: user %q not found or query error, skip. err=%v id=%d", username, err, baseRow.ID) + return nil + } + + // 步骤 2：尝试获取 jobs_frozen + var frozenRow struct { + JobsFrozen bool `gorm:"column:jobs_frozen"` + } + if err := db.Raw("SELECT jobs_frozen FROM users WHERE id = ?", baseRow.ID).Scan(&frozenRow).Error; err == nil && frozenRow.JobsFrozen { + klog.Infof("CheckStorageQuota: user=%q jobs_frozen=true, blocking job creation", username) + return bizerr.Conflict.ResourceStatusError.New( + "new job creation has been paused by an administrator; contact an administrator", + ) + } + + theoreticalQuota := baseRow.SpaceQuota + + // 步骤 3：尝试获取 original_space_quota（临时扩容时才有值） + var origRow struct { + OriginalSpaceQuota *int64 `gorm:"column:original_space_quota"` + } + if err := db.Raw("SELECT original_space_quota FROM users WHERE id = ?", baseRow.ID).Scan(&origRow).Error; err == nil && origRow.OriginalSpaceQuota != nil { + theoreticalQuota = *origRow.OriginalSpaceQuota + } + + klog.Infof("CheckStorageQuota: user=%q id=%d space_quota=%d original_space_quota=%v theoretical=%d", + username, baseRow.ID, baseRow.SpaceQuota, origRow.OriginalSpaceQuota, theoreticalQuota) + + // -1 = 无限制，0 = 未设置，均跳过 + if theoreticalQuota <= 0 { + klog.Infof("CheckStorageQuota: user=%q quota=%d (unlimited/unset), skip", username, theoreticalQuota) + return nil + } + + // 步骤 4：从 user_space_sizes 取最近一次记录的用量 + var usage model.UserSpaceSize + if err := db.Where("user_id = ?", baseRow.ID).First(&usage).Error; err != nil { + klog.Warningf("CheckStorageQuota: user=%q no user_space_sizes record (err=%v), skip", username, err) + return nil + } + + klog.Infof("CheckStorageQuota: user=%q size=%d theoretical=%d (%.1f%%)", + username, usage.Size, theoreticalQuota, float64(usage.Size)/float64(theoreticalQuota)*100) + + if usage.Size >= theoreticalQuota { + return bizerr.Conflict.ResourceStatusError.New(fmt.Sprintf( + "storage usage has reached the quota (%s used / %s quota); new jobs cannot be created", + FormatStorageSize(usage.Size), FormatStorageSize(theoreticalQuota), + )) + } + return nil +} + +// FormatStorageSize 将字节数格式化为人类可读的字符串。 +func FormatStorageSize(bytes int64) string { + const unit = 1024 + if bytes < unit { + return fmt.Sprintf("%d B", bytes) + } + div, exp := int64(unit), 0 + for n := bytes / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp]) +} diff --git a/backend/internal/util/quota_test.go b/backend/internal/util/quota_test.go new file mode 100644 index 000000000..3d84266a7 --- /dev/null +++ b/backend/internal/util/quota_test.go @@ -0,0 +1,28 @@ +package util + +import ( + "path/filepath" + "testing" + + "github.com/raids-lab/crater/pkg/config" +) + +func TestCheckStorageQuotaDisabledSkipsDatabase(t *testing.T) { + configPath, err := filepath.Abs("../../etc/debug-config.yaml") + if err != nil { + t.Fatalf("resolve debug config path: %v", err) + } + t.Setenv("CRATER_DEBUG_CONFIG_PATH", configPath) + + cfg := config.GetConfig() + originalEnabled := cfg.Storage.Quota.Enabled + disabled := false + cfg.Storage.Quota.Enabled = &disabled + t.Cleanup(func() { + cfg.Storage.Quota.Enabled = originalEnabled + }) + + if err := CheckStorageQuota("database-is-not-initialized"); err != nil { + t.Fatalf("CheckStorageQuota() with quota disabled returned error: %v", err) + } +} diff --git a/backend/pkg/ceph/ceph.go b/backend/pkg/ceph/ceph.go new file mode 100644 index 000000000..5a08eef76 --- /dev/null +++ b/backend/pkg/ceph/ceph.go @@ -0,0 +1,269 @@ +package ceph + +import ( + "context" + "errors" + "fmt" + "path" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + + cfgpkg "github.com/raids-lab/crater/pkg/config" + "github.com/raids-lab/crater/pkg/storagequota" +) + +const UnknownSizeBytes int64 = -1 + +const toolboxOperationTimeout = 20 * time.Second + +type StoragePrefixConfig struct { + User string + Account string + Public string +} + +func AvailableBytes(totalBytes, usedBytes int64) int64 { + if totalBytes < 0 || usedBytes < 0 { + return UnknownSizeBytes + } + return totalBytes - usedBytes +} + +func sharedStoragePVCName() string { + storagePVCName := "crater-storage" + if cfg := cfgpkg.GetConfig(); cfg != nil { + if value := strings.TrimSpace(cfg.Storage.PVC.ReadWriteMany); value != "" { + storagePVCName = value + } + } + return storagePVCName +} + +func StorageQuotaProvider() string { + cfg := cfgpkg.GetConfig() + if !storageQuotaEnabled(cfg.Storage.Quota.Enabled) { + return storagequota.ProviderDisabled + } + return storagequota.NormalizeProvider(cfg.Storage.Quota.Provider) +} + +func StorageQuotaEnabled() bool { + cfg := cfgpkg.GetConfig() + return storageQuotaEnabled(cfg.Storage.Quota.Enabled) +} + +func StorageQuotaRookNamespace() string { + return cfgpkg.GetConfig().StorageQuotaRookNamespace() +} + +func StorageQuotaCephFSCSIDriver() string { + return cfgpkg.GetConfig().StorageQuotaCephFSCSIDriver() +} + +func StorageQuotaToolboxLabelSelector() string { + return cfgpkg.GetConfig().StorageQuotaToolboxLabelSelector() +} + +func StorageQuotaCephFSName() string { + return cfgpkg.GetConfig().StorageQuotaCephFSName() +} + +func storageQuotaEnabled(enabled *bool) bool { + return enabled != nil && *enabled +} + +func GetStorageServerQuotaCapabilities(ctx context.Context) (storagequota.Capabilities, error) { + return storageQuotaClient().GetCapabilities(ctx) +} + +func logicalPathToStorageRelativePath( + logicalPath string, + prefixConfig StoragePrefixConfig, +) (string, error) { + trimmedPath := strings.Trim(strings.ReplaceAll(logicalPath, "\\", "/"), "/") + parts := strings.SplitN(trimmedPath, "/", 2) + if len(parts) == 0 || parts[0] == "" { + return "", fmt.Errorf("invalid path format: %s", logicalPath) + } + + var storagePrefix string + var remainingPath string + if len(parts) == 2 { + remainingPath = parts[1] + } + switch parts[0] { + case "user": + if remainingPath == "" { + return "", fmt.Errorf("user path must include space name: %s", logicalPath) + } + storagePrefix = prefixConfig.User + case "account": + storagePrefix = prefixConfig.Account + case "public": + storagePrefix = prefixConfig.Public + default: + return "", fmt.Errorf("unknown path type: %s", parts[0]) + } + + relativePath := path.Clean(path.Join(storagePrefix, remainingPath)) + cleanPrefix := path.Clean(strings.Trim(storagePrefix, "/")) + if cleanPrefix == "." || cleanPrefix == ".." || strings.HasPrefix(cleanPrefix, "../") { + return "", fmt.Errorf("invalid storage prefix for %s", parts[0]) + } + if relativePath != cleanPrefix && !strings.HasPrefix(relativePath, cleanPrefix+"/") { + return "", fmt.Errorf("path escapes storage prefix: %s", logicalPath) + } + return relativePath, nil +} + +func storageQuotaClient() *storagequota.Client { + cfg := cfgpkg.GetConfig() + return storagequota.NewClient( + storagequota.ResolveServerURL(cfg.Storage.Quota.StorageServerURL, cfg.Namespaces.Job), + cfg.Auth.Token.AccessTokenSecret, + ) +} + +func GetCephDirectorySize( + clientset kubernetes.Interface, + config *rest.Config, + namespace, logicalPath string, + prefixConfig StoragePrefixConfig, +) (int64, error) { + relativePath, err := logicalPathToStorageRelativePath(logicalPath, prefixConfig) + if err != nil { + return 0, err + } + return getDirectoryUsage(clientset, config, namespace, relativePath) +} + +func SetCephDirectoryQuota( + clientset kubernetes.Interface, + config *rest.Config, + namespace, logicalPath string, + prefixConfig StoragePrefixConfig, + maxBytes int64, +) error { + relativePath, err := logicalPathToStorageRelativePath(logicalPath, prefixConfig) + if err != nil { + return err + } + return setDirectoryQuota(clientset, config, namespace, relativePath, maxBytes) +} + +//nolint:gocritic // Capacity and usage are returned as distinct values consumed by existing callers. +func GetCraterStorageCapacity( + clientset kubernetes.Interface, + config *rest.Config, + namespace string, +) (int64, int64, error) { + storagePVCName := sharedStoragePVCName() + pvcs, err := clientset.CoreV1().PersistentVolumeClaims(namespace).List( + context.Background(), + metav1.ListOptions{FieldSelector: "metadata.name=" + storagePVCName}, + ) + if err != nil { + return UnknownSizeBytes, UnknownSizeBytes, nil + } + if len(pvcs.Items) == 0 { + pvcs, err = clientset.CoreV1().PersistentVolumeClaims("").List( + context.Background(), + metav1.ListOptions{FieldSelector: "metadata.name=" + storagePVCName}, + ) + if err != nil || len(pvcs.Items) == 0 { + return UnknownSizeBytes, UnknownSizeBytes, nil + } + } + + pvc := pvcs.Items[0] + totalBytes := UnknownSizeBytes + if capacity, ok := pvc.Status.Capacity[corev1.ResourceStorage]; ok { + totalBytes = capacity.Value() + } else if requested, ok := pvc.Spec.Resources.Requests[corev1.ResourceStorage]; ok { + totalBytes = requested.Value() + } + + usage, err := getDirectoryUsage(clientset, config, StorageQuotaRookNamespace(), ".") + if err != nil { + return totalBytes, UnknownSizeBytes, nil + } + return totalBytes, usage, nil +} + +func getDirectoryUsage( + clientset kubernetes.Interface, + config *rest.Config, + namespace, relativePath string, +) (int64, error) { + provider := StorageQuotaProvider() + if provider == storagequota.ProviderDisabled { + return 0, fmt.Errorf("storage quota provider is disabled") + } + + var storageServerErr error + if provider == storagequota.ProviderAuto || provider == storagequota.ProviderStorageServer { + usage, err := storageQuotaClient().GetUsage(context.Background(), relativePath) + if err == nil { + return usage.Bytes, nil + } + storageServerErr = fmt.Errorf("read directory usage from storage-server: %w", err) + if provider == storagequota.ProviderStorageServer { + return 0, storageServerErr + } + } + + ctx, cancel := context.WithTimeout(context.Background(), toolboxOperationTimeout) + defer cancel() + usage, toolboxErr := getToolboxUsage(ctx, clientset, config, namespace, relativePath) + if toolboxErr == nil { + return usage, nil + } + if storageServerErr != nil { + return 0, fmt.Errorf("storage quota providers failed: %w", errors.Join(storageServerErr, toolboxErr)) + } + return 0, fmt.Errorf("read directory usage through toolbox: %w", toolboxErr) +} + +func setDirectoryQuota( + clientset kubernetes.Interface, + config *rest.Config, + namespace, relativePath string, + maxBytes int64, +) error { + if maxBytes < -1 || maxBytes == 0 { + return fmt.Errorf("quota must be -1 for unlimited or greater than zero") + } + + provider := StorageQuotaProvider() + if provider == storagequota.ProviderDisabled { + return fmt.Errorf("storage quota provider is disabled") + } + + var storageServerErr error + if provider == storagequota.ProviderAuto || provider == storagequota.ProviderStorageServer { + if _, err := storageQuotaClient().SetQuota(context.Background(), relativePath, maxBytes); err == nil { + return nil + } else { + storageServerErr = fmt.Errorf("write directory quota through storage-server: %w", err) + if provider == storagequota.ProviderStorageServer { + return storageServerErr + } + } + } + + ctx, cancel := context.WithTimeout(context.Background(), toolboxOperationTimeout) + defer cancel() + toolboxQuota := cephQuotaXattrValue(maxBytes) + if err := setToolboxQuota(ctx, clientset, config, namespace, relativePath, toolboxQuota); err == nil { + return nil + } else if storageServerErr != nil { + return fmt.Errorf("storage quota providers failed: %w", errors.Join(storageServerErr, err)) + } else { + return err + } +} diff --git a/backend/pkg/ceph/quota_provider_test.go b/backend/pkg/ceph/quota_provider_test.go new file mode 100644 index 000000000..bbe7f95e7 --- /dev/null +++ b/backend/pkg/ceph/quota_provider_test.go @@ -0,0 +1,71 @@ +package ceph + +import "testing" + +func TestStorageQuotaEnabledRequiresExplicitOptIn(t *testing.T) { + t.Parallel() + + enabled := true + disabled := false + tests := []struct { + name string + enabled *bool + want bool + }{ + {name: "omitted", enabled: nil, want: false}, + {name: "disabled", enabled: &disabled, want: false}, + {name: "enabled", enabled: &enabled, want: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + if got := storageQuotaEnabled(tt.enabled); got != tt.want { + t.Fatalf("storageQuotaEnabled() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestSetDirectoryQuotaRejectsAmbiguousValues(t *testing.T) { + t.Parallel() + + for _, quota := range []int64{-2, 0} { + if err := setDirectoryQuota(nil, nil, "", "users/alice", quota); err == nil { + t.Errorf("setDirectoryQuota() accepted quota %d", quota) + } + } +} + +func TestLogicalPathToStorageRelativePath(t *testing.T) { + t.Parallel() + + prefixes := StoragePrefixConfig{User: "users", Account: "accounts", Public: "public"} + tests := []struct { + name string + path string + want string + wantErr bool + }{ + {name: "user", path: "/user/alice", want: "users/alice"}, + {name: "nested user path", path: "/user/alice/checkpoints", want: "users/alice/checkpoints"}, + {name: "public root", path: "/public", want: "public"}, + {name: "account", path: "/account/lab", want: "accounts/lab"}, + {name: "missing user space", path: "/user", wantErr: true}, + {name: "escape prefix", path: "/user/../../public", wantErr: true}, + {name: "unknown type", path: "/other/alice", wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got, err := logicalPathToStorageRelativePath(tt.path, prefixes) + if (err != nil) != tt.wantErr { + t.Fatalf("error = %v, wantErr %v", err, tt.wantErr) + } + if got != tt.want { + t.Fatalf("path = %q, want %q", got, tt.want) + } + }) + } +} diff --git a/backend/pkg/ceph/toolbox.go b/backend/pkg/ceph/toolbox.go new file mode 100644 index 000000000..91b0eaca4 --- /dev/null +++ b/backend/pkg/ceph/toolbox.go @@ -0,0 +1,336 @@ +package ceph + +import ( + "context" + "fmt" + "net/http" + "path" + "regexp" + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" + + cfgpkg "github.com/raids-lab/crater/pkg/config" + "github.com/raids-lab/crater/pkg/storagequota" +) + +const ( + toolboxMountRoot = "/mnt/crater-cephfs" + cephUsageXattr = "ceph.dir.rbytes" + cephQuotaBytesXattr = "ceph.quota.max_bytes" +) + +func cephQuotaXattrValue(maxBytes int64) int64 { + if maxBytes == -1 { + return 0 + } + return maxBytes +} + +var volumeHandleUUIDPattern = regexp.MustCompile( + `[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}`, +) + +func GetToolboxQuotaCapabilities( + ctx context.Context, + clientset kubernetes.Interface, + config *rest.Config, + namespace string, +) (storagequota.Capabilities, error) { + var capabilities storagequota.Capabilities + pod, storageRoot, err := resolveToolboxStorageRoot(ctx, clientset, config, namespace) + if err != nil { + return capabilities, err + } + + if _, err := readToolboxXattr(ctx, clientset, config, pod, storageRoot, cephUsageXattr); err != nil { + capabilities.Reasons = append(capabilities.Reasons, err.Error()) + return capabilities, nil + } + capabilities.UsageReadable = true + + if _, err := readToolboxXattr(ctx, clientset, config, pod, storageRoot, cephQuotaBytesXattr); err == nil { + capabilities.QuotaReadable = true + } + + probeScript := fmt.Sprintf(` +set -eu +test_dir=$(mktemp -d %s/.crater-quota-capability-XXXXXX) +cleanup() { + setfattr -n %s -v 0 "$test_dir" >/dev/null 2>&1 || true + rmdir "$test_dir" >/dev/null 2>&1 || true +} +trap cleanup EXIT +setfattr -n %s -v 0 "$test_dir" +getfattr --only-values -n %s "$test_dir" >/dev/null +`, shellQuote(storageRoot), cephQuotaBytesXattr, cephQuotaBytesXattr, cephQuotaBytesXattr) + if _, err := execToolbox(ctx, clientset, config, pod, []string{"sh", "-c", probeScript}); err != nil { + capabilities.Reasons = append(capabilities.Reasons, fmt.Sprintf("toolbox quota write probe: %v", err)) + return capabilities, nil + } + + capabilities.QuotaReadable = true + capabilities.QuotaWritable = true + return capabilities, nil +} + +func getToolboxUsage( + ctx context.Context, + clientset kubernetes.Interface, + config *rest.Config, + namespace, relativePath string, +) (int64, error) { + pod, storageRoot, err := resolveToolboxStorageRoot(ctx, clientset, config, namespace) + if err != nil { + return 0, err + } + targetPath, err := toolboxStoragePath(storageRoot, relativePath) + if err != nil { + return 0, err + } + value, err := readToolboxXattr(ctx, clientset, config, pod, targetPath, cephUsageXattr) + if err != nil { + return 0, err + } + usage, err := strconv.ParseInt(strings.TrimSpace(value), 10, 64) + if err != nil { + return 0, fmt.Errorf("parse toolbox usage %q: %w", value, err) + } + return usage, nil +} + +func setToolboxQuota( + ctx context.Context, + clientset kubernetes.Interface, + config *rest.Config, + namespace, relativePath string, + maxBytes int64, +) error { + pod, storageRoot, err := resolveToolboxStorageRoot(ctx, clientset, config, namespace) + if err != nil { + return err + } + targetPath, err := toolboxStoragePath(storageRoot, relativePath) + if err != nil { + return err + } + script := fmt.Sprintf( + "setfattr -n %s -v %s -- %s", + cephQuotaBytesXattr, + strconv.FormatInt(maxBytes, 10), + shellQuote(targetPath), + ) + if _, err := execToolbox(ctx, clientset, config, pod, []string{"sh", "-c", script}); err != nil { + return fmt.Errorf("write CephFS quota through toolbox: %w", err) + } + return nil +} + +//nolint:gocyclo // Resolving the toolbox mount validates each PV and CSI fallback explicitly. +func resolveToolboxStorageRoot( + ctx context.Context, + clientset kubernetes.Interface, + config *rest.Config, + namespace string, +) (*corev1.Pod, string, error) { + if clientset == nil || config == nil { + return nil, "", fmt.Errorf("kubernetes client or REST config is unavailable") + } + pod, err := findToolboxPod(ctx, clientset, namespace, StorageQuotaToolboxLabelSelector()) + if err != nil { + return nil, "", err + } + + pvcNamespace := strings.TrimSpace(cfgpkg.GetConfig().Namespaces.Job) + if pvcNamespace == "" { + return nil, "", fmt.Errorf("job namespace is not configured") + } + pvc, err := clientset.CoreV1().PersistentVolumeClaims(pvcNamespace). + Get(ctx, sharedStoragePVCName(), metav1.GetOptions{}) + if err != nil { + return nil, "", fmt.Errorf("get shared storage PVC %s/%s: %w", pvcNamespace, sharedStoragePVCName(), err) + } + pvName := strings.TrimSpace(pvc.Spec.VolumeName) + if pvName == "" { + return nil, "", fmt.Errorf("shared storage PVC is not bound") + } + pv, err := clientset.CoreV1().PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}) + if err != nil { + return nil, "", fmt.Errorf("get shared storage PV %s: %w", pvName, err) + } + expectedDriver := StorageQuotaCephFSCSIDriver() + if pv.Spec.CSI == nil || pv.Spec.CSI.Driver != expectedDriver { + return nil, "", fmt.Errorf( + "shared storage PV %s does not use configured CephFS CSI driver %q", + pvName, + expectedDriver, + ) + } + + fsName := strings.TrimSpace(pv.Spec.CSI.VolumeAttributes["fsName"]) + if fsName == "" { + fsName = StorageQuotaCephFSName() + } + if err := ensureToolboxCephFSMounted(ctx, clientset, config, pod, fsName); err != nil { + return nil, "", err + } + + if subvolumePath := strings.TrimSpace(pv.Spec.CSI.VolumeAttributes["subvolumePath"]); subvolumePath != "" { + return pod, path.Join(toolboxMountRoot, strings.TrimLeft(subvolumePath, "/")), nil + } + + volumeUUID := volumeHandleUUIDPattern.FindString(pv.Spec.CSI.VolumeHandle) + if volumeUUID == "" { + return nil, "", fmt.Errorf("cannot resolve CephFS subvolume path from PV %s", pvName) + } + volumeRoot := path.Join(toolboxMountRoot, "volumes/csi/csi-vol-"+volumeUUID) + script := fmt.Sprintf( + "find %s -mindepth 1 -maxdepth 1 -type d -print -quit", + shellQuote(volumeRoot), + ) + output, err := execToolbox(ctx, clientset, config, pod, []string{"sh", "-c", script}) + if err != nil { + return nil, "", fmt.Errorf("resolve CephFS subvolume directory: %w", err) + } + storageRoot := strings.TrimSpace(output) + if storageRoot == "" { + return nil, "", fmt.Errorf("CephFS subvolume directory is empty under %s", volumeRoot) + } + return pod, storageRoot, nil +} + +func findToolboxPod( + ctx context.Context, + clientset kubernetes.Interface, + namespace, labelSelector string, +) (*corev1.Pod, error) { + pods, err := clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: labelSelector, + }) + if err != nil { + return nil, fmt.Errorf("list toolbox pods in %s with selector %q: %w", namespace, labelSelector, err) + } + for i := range pods.Items { + pod := &pods.Items[i] + if pod.Status.Phase != corev1.PodRunning { + continue + } + return pod, nil + } + return nil, fmt.Errorf( + "running Rook Ceph toolbox pod matching %q was not found in %s", + labelSelector, + namespace, + ) +} + +func ensureToolboxCephFSMounted( + ctx context.Context, + clientset kubernetes.Interface, + config *rest.Config, + pod *corev1.Pod, + fsName string, +) error { + script := fmt.Sprintf(` +set -eu +mount_root=%s +fs_name=%s +if grep -qs " ${mount_root} " /proc/mounts; then + exit 0 +fi +mkdir -p "${mount_root}" +if command -v ceph-fuse >/dev/null 2>&1; then + ceph-fuse --client_fs "${fs_name}" "${mount_root}" || ceph-fuse "${mount_root}" +else + mount -t ceph :/ "${mount_root}" -o name=admin,fs="${fs_name}" || mount -t ceph :/ "${mount_root}" -o name=admin +fi +grep -qs " ${mount_root} " /proc/mounts +`, shellQuote(toolboxMountRoot), shellQuote(fsName)) + if _, err := execToolbox(ctx, clientset, config, pod, []string{"sh", "-c", script}); err != nil { + return fmt.Errorf("mount CephFS in toolbox: %w", err) + } + return nil +} + +func readToolboxXattr( + ctx context.Context, + clientset kubernetes.Interface, + config *rest.Config, + pod *corev1.Pod, + targetPath, attribute string, +) (string, error) { + script := fmt.Sprintf( + "getfattr --only-values -n %s -- %s", + attribute, + shellQuote(targetPath), + ) + output, err := execToolbox(ctx, clientset, config, pod, []string{"sh", "-c", script}) + if err != nil { + return "", fmt.Errorf("read %s through toolbox: %w", attribute, err) + } + return strings.TrimSpace(output), nil +} + +func toolboxStoragePath(storageRoot, relativePath string) (string, error) { + root := path.Clean(storageRoot) + relative := path.Clean(strings.Trim(strings.ReplaceAll(relativePath, "\\", "/"), "/")) + if relative == "." || relative == "" { + return root, nil + } + if relative == ".." || strings.HasPrefix(relative, "../") { + return "", fmt.Errorf("path escapes toolbox storage root: %s", relativePath) + } + target := path.Join(root, relative) + if target != root && !strings.HasPrefix(target, root+"/") { + return "", fmt.Errorf("path escapes toolbox storage root: %s", relativePath) + } + return target, nil +} + +func execToolbox( + ctx context.Context, + clientset kubernetes.Interface, + config *rest.Config, + pod *corev1.Pod, + command []string, +) (string, error) { + var stdout, stderr strings.Builder + request := clientset.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(pod.Name). + Namespace(pod.Namespace). + SubResource("exec") + + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + return "", fmt.Errorf("register Kubernetes core scheme: %w", err) + } + request.VersionedParams(&corev1.PodExecOptions{ + Command: command, + Stdout: true, + Stderr: true, + }, runtime.NewParameterCodec(scheme)) + + executor, err := remotecommand.NewSPDYExecutor(config, http.MethodPost, request.URL()) + if err != nil { + return "", fmt.Errorf("create toolbox executor: %w", err) + } + if err := executor.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdin: nil, + Stdout: &stdout, + Stderr: &stderr, + }); err != nil { + return "", fmt.Errorf("%w: %s", err, strings.TrimSpace(stderr.String())) + } + return stdout.String(), nil +} + +func shellQuote(value string) string { + return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'" +} diff --git a/backend/pkg/ceph/toolbox_test.go b/backend/pkg/ceph/toolbox_test.go new file mode 100644 index 000000000..af002abca --- /dev/null +++ b/backend/pkg/ceph/toolbox_test.go @@ -0,0 +1,87 @@ +package ceph + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +func TestToolboxStoragePath(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + root string + relative string + want string + wantErr bool + }{ + {name: "root", root: "/mnt/ceph/volume", relative: ".", want: "/mnt/ceph/volume"}, + {name: "user", root: "/mnt/ceph/volume", relative: "users/alice", want: "/mnt/ceph/volume/users/alice"}, + {name: "escape", root: "/mnt/ceph/volume", relative: "../other", wantErr: true}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + got, err := toolboxStoragePath(test.root, test.relative) + if (err != nil) != test.wantErr { + t.Fatalf("toolboxStoragePath() error = %v, wantErr %v", err, test.wantErr) + } + if got != test.want { + t.Fatalf("toolboxStoragePath() = %q, want %q", got, test.want) + } + }) + } +} + +func TestCephQuotaXattrValue(t *testing.T) { + t.Parallel() + + if got := cephQuotaXattrValue(-1); got != 0 { + t.Fatalf("unlimited quota xattr = %d, want 0", got) + } + if got := cephQuotaXattrValue(4 * 1024 * 1024); got != 4*1024*1024 { + t.Fatalf("limited quota xattr = %d", got) + } +} + +func TestFindToolboxPod(t *testing.T) { + t.Parallel() + const selector = "component=ceph-tools" + + clientset := fake.NewSimpleClientset( + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "not-running", Namespace: "storage-system", Labels: map[string]string{"component": "ceph-tools"}}, + Status: corev1.PodStatus{Phase: corev1.PodPending}, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "custom-tools-abc", Namespace: "storage-system", Labels: map[string]string{"component": "ceph-tools"}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "toolbox-with-wrong-label", Namespace: "storage-system", Labels: map[string]string{"app": "other"}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + }, + ) + + pod, err := findToolboxPod(context.Background(), clientset, "storage-system", selector) + if err != nil { + t.Fatalf("findToolboxPod() error = %v", err) + } + if pod.Name != "custom-tools-abc" { + t.Fatalf("findToolboxPod() = %q", pod.Name) + } +} + +func TestVolumeHandleUUIDPattern(t *testing.T) { + t.Parallel() + + handle := "0001-0009-rook-ceph-0000000000000001-7ca9d703-bbbc-4ae5-a03d-ae9f9f7be59e" + if got := volumeHandleUUIDPattern.FindString(handle); got != "7ca9d703-bbbc-4ae5-a03d-ae9f9f7be59e" { + t.Fatalf("UUID = %q", got) + } +} diff --git a/backend/pkg/config/config.go b/backend/pkg/config/config.go index b2d1e0b02..bc9798757 100644 --- a/backend/pkg/config/config.go +++ b/backend/pkg/config/config.go @@ -83,6 +83,34 @@ type Config struct { // Storage contains persistent volume claim and path prefix configurations. // Required: All PVC names and prefix paths must be specified. Storage struct { + // Quota configures how CephFS directory quotas are read and written. + Quota struct { + // Enabled explicitly enables CephFS usage and quota management. + // Nil and false both keep the feature disabled. + Enabled *bool `json:"enabled,omitempty"` + + // Provider selects auto, storageServer, toolbox, or disabled. + // Auto prefers storage-server and falls back to a Rook Ceph toolbox. + Provider string `json:"provider,omitempty"` + + // StorageServerURL is the internal storage-server endpoint. + // If empty, the service URL is derived from Namespaces.Job. + StorageServerURL string `json:"storageServerURL,omitempty"` + + // RookNamespace contains the optional Rook Ceph toolbox and CephClient resources. + RookNamespace string `json:"rookNamespace,omitempty"` + + // CephFSCSIDriver identifies the CephFS CSI driver used by the shared storage PV. + // If empty, it is derived from RookNamespace. + CephFSCSIDriver string `json:"cephFSCSIDriver,omitempty"` + + // ToolboxLabelSelector selects the optional Rook toolbox Pod. + ToolboxLabelSelector string `json:"toolboxLabelSelector,omitempty"` + + // CephFSName is used only when the shared PV does not expose fsName. + CephFSName string `json:"cephFSName,omitempty"` + } `json:"quota,omitempty"` + PVC struct { // ReadWriteMany is the name of the ReadWriteMany Persistent Volume Claim for shared storage. // Required: PVC must exist in the cluster with ReadWriteMany access mode. @@ -432,6 +460,9 @@ func (c *Config) ValidateConfig() error { if c.Storage.Prefix.Public == "" { errors = append(errors, "storage.prefix.public is required") } + if !isStorageQuotaProviderValid(c.Storage.Quota.Provider) { + errors = append(errors, fmt.Sprintf("invalid storage.quota.provider: %s", c.Storage.Quota.Provider)) + } // Validate secrets configuration if c.Secrets.TLSSecretName == "" { @@ -552,6 +583,49 @@ func (c *Config) ValidateConfig() error { return nil } +func isStorageQuotaProviderValid(provider string) bool { + switch strings.ToLower(strings.TrimSpace(provider)) { + case "", "auto", "storageserver", "storage-server", "toolbox", "disabled": + return true + default: + return false + } +} + +const ( + defaultStorageQuotaRookNamespace = "rook-ceph" + defaultStorageQuotaToolboxLabelSelector = "app=rook-ceph-tools" + defaultStorageQuotaCephFSName = "cephfs" +) + +func (c *Config) StorageQuotaRookNamespace() string { + if value := strings.TrimSpace(c.Storage.Quota.RookNamespace); value != "" { + return value + } + return defaultStorageQuotaRookNamespace +} + +func (c *Config) StorageQuotaCephFSCSIDriver() string { + if value := strings.TrimSpace(c.Storage.Quota.CephFSCSIDriver); value != "" { + return value + } + return c.StorageQuotaRookNamespace() + ".cephfs.csi.ceph.com" +} + +func (c *Config) StorageQuotaToolboxLabelSelector() string { + if value := strings.TrimSpace(c.Storage.Quota.ToolboxLabelSelector); value != "" { + return value + } + return defaultStorageQuotaToolboxLabelSelector +} + +func (c *Config) StorageQuotaCephFSName() string { + if value := strings.TrimSpace(c.Storage.Quota.CephFSName); value != "" { + return value + } + return defaultStorageQuotaCephFSName +} + const ldapAliasMaxRunes = 6 // logConfigWarnings collects non-fatal configuration warnings and logs them once. @@ -586,6 +660,8 @@ func (c *Config) logConfigWarnings() { } // PrintConfig prints the configuration in a formatted and readable way, masking sensitive information +// +//nolint:gocyclo // The summary intentionally prints optional configuration sections independently. func (c *Config) PrintConfig() { klog.Info("=== Configuration Summary ===") @@ -615,6 +691,19 @@ func (c *Config) PrintConfig() { } klog.Infof("Storage Prefixes: User=%s, Account=%s, Public=%s", c.Storage.Prefix.User, c.Storage.Prefix.Account, c.Storage.Prefix.Public) + quotaEnabled := c.Storage.Quota.Enabled != nil && *c.Storage.Quota.Enabled + quotaProvider := strings.TrimSpace(c.Storage.Quota.Provider) + if quotaProvider == "" { + quotaProvider = "auto" + } + klog.Infof("Storage Quota: Enabled=%t, Provider=%s", quotaEnabled, quotaProvider) + if quotaEnabled { + klog.Infof("Storage Quota Rook: Namespace=%s, CSI Driver=%s, Toolbox Selector=%s", + c.StorageQuotaRookNamespace(), + c.StorageQuotaCephFSCSIDriver(), + c.StorageQuotaToolboxLabelSelector(), + ) + } // Model Download if c.ModelDownload.Image != "" { diff --git a/backend/pkg/config/storage_quota_test.go b/backend/pkg/config/storage_quota_test.go new file mode 100644 index 000000000..69f7c081b --- /dev/null +++ b/backend/pkg/config/storage_quota_test.go @@ -0,0 +1,70 @@ +package config + +import "testing" + +const customRookNamespace = "storage-system" + +func TestStorageQuotaProviderValidation(t *testing.T) { + t.Parallel() + + tests := []struct { + provider string + want bool + }{ + {provider: "", want: true}, + {provider: "auto", want: true}, + {provider: "storageServer", want: true}, + {provider: "storage-server", want: true}, + {provider: "toolbox", want: true}, + {provider: "disabled", want: true}, + {provider: "cephfs", want: false}, + } + + for _, tt := range tests { + if got := isStorageQuotaProviderValid(tt.provider); got != tt.want { + t.Errorf("isStorageQuotaProviderValid(%q) = %v, want %v", tt.provider, got, tt.want) + } + } +} + +func TestStorageQuotaClusterDefaultsAndOverrides(t *testing.T) { + t.Parallel() + + var defaults Config + if got := defaults.StorageQuotaRookNamespace(); got != "rook-ceph" { + t.Fatalf("default Rook namespace = %q", got) + } + if got := defaults.StorageQuotaCephFSCSIDriver(); got != "rook-ceph.cephfs.csi.ceph.com" { + t.Fatalf("default CephFS CSI driver = %q", got) + } + if got := defaults.StorageQuotaToolboxLabelSelector(); got != "app=rook-ceph-tools" { + t.Fatalf("default toolbox selector = %q", got) + } + if got := defaults.StorageQuotaCephFSName(); got != "cephfs" { + t.Fatalf("default CephFS name = %q", got) + } + + var custom Config + custom.Storage.Quota.RookNamespace = customRookNamespace + custom.Storage.Quota.CephFSCSIDriver = "custom.cephfs.csi.example.com" + custom.Storage.Quota.ToolboxLabelSelector = "app=ceph-toolbox" + custom.Storage.Quota.CephFSName = "shared-fs" + if got := custom.StorageQuotaRookNamespace(); got != customRookNamespace { + t.Fatalf("custom Rook namespace = %q", got) + } + if got := custom.StorageQuotaCephFSCSIDriver(); got != "custom.cephfs.csi.example.com" { + t.Fatalf("custom CephFS CSI driver = %q", got) + } + if got := custom.StorageQuotaToolboxLabelSelector(); got != "app=ceph-toolbox" { + t.Fatalf("custom toolbox selector = %q", got) + } + if got := custom.StorageQuotaCephFSName(); got != "shared-fs" { + t.Fatalf("custom CephFS name = %q", got) + } + + var derived Config + derived.Storage.Quota.RookNamespace = customRookNamespace + if got := derived.StorageQuotaCephFSCSIDriver(); got != customRookNamespace+".cephfs.csi.ceph.com" { + t.Fatalf("derived CephFS CSI driver = %q", got) + } +} diff --git a/backend/pkg/constants/const_op.go b/backend/pkg/constants/const_op.go index 932caa620..5d712c314 100644 --- a/backend/pkg/constants/const_op.go +++ b/backend/pkg/constants/const_op.go @@ -9,6 +9,7 @@ const ( OpTypeDrainNode = "DrainNode" OpTypeUpdateVPA = "UpdateVPA" OpTypeDeleteJob = "DeleteJob" + OpTypeSetStorageQuota = "SetStorageQuota" // Execution Status OpStatusSuccess = "Success" diff --git a/backend/pkg/cronjob/manger.go b/backend/pkg/cronjob/manger.go index f4d9444a1..6a1b3658f 100644 --- a/backend/pkg/cronjob/manger.go +++ b/backend/pkg/cronjob/manger.go @@ -1,21 +1,27 @@ package cronjob import ( + "context" + "fmt" "sync" "time" "github.com/robfig/cron/v3" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/raids-lab/crater/dao/model" "github.com/raids-lab/crater/pkg/cleaner" "github.com/raids-lab/crater/pkg/monitor" "github.com/raids-lab/crater/pkg/patrol" + "github.com/raids-lab/crater/pkg/storagegovernance" ) type CronJobManager struct { Client client.Client KubeClient kubernetes.Interface + KubeConfig *rest.Config PromClient monitor.PrometheusInterface cleanerClients *cleaner.Clients patrolClients *patrol.Clients @@ -26,13 +32,21 @@ type CronJobManager struct { func NewCronJobManager( cli client.Client, kubeClient kubernetes.Interface, + kubeConfig *rest.Config, promClient monitor.PrometheusInterface, gpuAnalysisService patrol.GpuAnalysisServiceInterface, billingService patrol.BillingServiceInterface, ) *CronJobManager { + decisionEngine := storagegovernance.NewEngine( + kubeClient, + kubeConfig, + promClient, + storagegovernance.DefaultConstraintConfig(), + ) return &CronJobManager{ Client: cli, KubeClient: kubeClient, + KubeConfig: kubeConfig, PromClient: promClient, cleanerClients: &cleaner.Clients{ Client: cli, @@ -42,10 +56,98 @@ func NewCronJobManager( patrolClients: &patrol.Clients{ Client: cli, KubeClient: kubeClient, + KubeConfig: kubeConfig, PromClient: promClient, GpuAnalysisService: gpuAnalysisService, BillingService: billingService, + RecordDecision: func(ctx context.Context, jobID string, action string, runErr error) { + _ = storagegovernance.MarkDecisionExecution(ctx, jobID, action, runErr) + }, + StorageAgent: func(tenantID string) (*patrol.AgentDecision, error) { + resp, jobID, err := decisionEngine.DecideAndRecord(context.Background(), storagegovernance.DecisionRequest{ + Username: tenantID, + Source: model.StorageDecisionSourcePatrol, + TriggerReason: "cronjob analyze-storage-alerts", + }) + if err != nil { + return nil, err + } + return &patrol.AgentDecision{ + AllowExpand: resp.AllowExpand, + ExpandBytes: resp.ExpandBytes, + FreezeNewJobs: resp.FreezeNewJobs, + Reason: resp.Reason, + DecisionJobID: jobID, + }, nil + }, + StorageAgentStart: func(ctx context.Context, tenantID string) (string, error) { + return decisionEngine.StartAsyncDecision(ctx, storagegovernance.DecisionRequest{ + Username: tenantID, + Source: model.StorageDecisionSourcePatrol, + TriggerReason: "cronjob analyze-storage-alerts", + }) + }, + StorageAgentAwait: awaitStorageDecisionResult, }, cron: cron.New(cron.WithLocation(time.Local)), } } + +func awaitStorageDecisionResult(ctx context.Context, tenantID, jobID string) (*patrol.AgentDecision, error) { + const ( + defaultTimeout = 5 * time.Minute + pollInterval = 2 * time.Second + ) + + if _, hasDeadline := ctx.Deadline(); !hasDeadline { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, defaultTimeout) + defer cancel() + } + + ticker := time.NewTicker(pollInterval) + defer ticker.Stop() + + for { + status, err := storagegovernance.GetDecisionStatus(ctx, jobID) + if err != nil { + return nil, fmt.Errorf("query storage decision status failed: %w", err) + } + + switch status.Status { + case string(model.StorageDecisionStatusDone): + if status.Result == nil { + return nil, fmt.Errorf("storage decision %s finished without result", jobID) + } + return &patrol.AgentDecision{ + AllowExpand: status.Result.AllowExpand, + ExpandBytes: status.Result.ExpandBytes, + FreezeNewJobs: status.Result.FreezeNewJobs, + Reason: status.Result.Reason, + DecisionJobID: jobID, + }, nil + case string(model.StorageDecisionStatusError): + if status.ErrorMsg == "" { + return nil, fmt.Errorf("storage decision %s failed", jobID) + } + return nil, fmt.Errorf("storage decision %s failed: %s", jobID, status.ErrorMsg) + case string(model.StorageDecisionStatusPending), string(model.StorageDecisionStatusRunning): + select { + case <-ctx.Done(): + return nil, fmt.Errorf("wait storage decision timeout for user %s job %s: %w", tenantID, jobID, ctx.Err()) + case <-ticker.C: + } + default: + select { + case <-ctx.Done(): + return nil, fmt.Errorf("wait storage decision timeout for user %s job %s: %w", tenantID, jobID, ctx.Err()) + case <-ticker.C: + } + } + } +} + +// GetPatrolClients returns the patrol clients +func (cm *CronJobManager) GetPatrolClients() *patrol.Clients { + return cm.patrolClients +} diff --git a/backend/pkg/cronjob/manger_test.go b/backend/pkg/cronjob/manger_test.go index 4d1a30dfa..dca910c3d 100644 --- a/backend/pkg/cronjob/manger_test.go +++ b/backend/pkg/cronjob/manger_test.go @@ -14,7 +14,7 @@ import ( func TestCronJob(t *testing.T) { t.Run("newCronJobFunc", func(t *testing.T) { - manager := NewCronJobManager(nil, nil, nil, nil, nil) + manager := NewCronJobManager(nil, nil, nil, nil, nil, nil) PatchConvey("newCronJobFunc", t, func() { jobName := cleaner.CLEAN_LONG_TIME_RUNNING_JOB jobConfig := datatypes.JSON(`{"batchDays": 4, "interactiveDays": 4}`) @@ -50,7 +50,7 @@ func TestCronJob(t *testing.T) { t.Run("prepareUpdateConfig", func(t *testing.T) { PatchConvey("prepareUpdateConfig", t, func() { - manager := NewCronJobManager(nil, nil, nil, nil, nil) + manager := NewCronJobManager(nil, nil, nil, nil, nil, nil) cur := &model.CronJobConfig{ Name: "test", Type: model.CronJobTypeCleanerFunc, diff --git a/backend/pkg/llm/direct_decision.go b/backend/pkg/llm/direct_decision.go new file mode 100644 index 000000000..2b215232d --- /dev/null +++ b/backend/pkg/llm/direct_decision.go @@ -0,0 +1,525 @@ +package llm + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" + + "k8s.io/klog/v2" + + "github.com/raids-lab/crater/dao/model" + "github.com/raids-lab/crater/dao/query" + "github.com/raids-lab/crater/pkg/crypto" +) + +const ( + StorageDecisionModeEnv = "CRATER_STORAGE_DECISION_MODE" + StorageDecisionModeAgent = "agent" + StorageDecisionModeDirect = "direct" + StorageDecisionConfigSourceEnv = "CRATER_STORAGE_DECISION_CONFIG_SOURCE" + StorageDecisionConfigSourcePlatform = "platform" + StorageDecisionConfigSourceCustom = "custom" + + DirectModelBaseURLEnv = "CRATER_STORAGE_DIRECT_MODEL_BASE_URL" + DirectModelAPIKeyEnv = "CRATER_STORAGE_DIRECT_MODEL_API_KEY" + DirectModelNameEnv = "CRATER_STORAGE_DIRECT_MODEL_NAME" + DefaultDirectModelBaseURL = "" + + directDecisionInstruction = `你是面向 AI 集群的存储治理决策模型。请根据输入的结构化存储治理快照，输出一个 JSON 决策对象。 +输出要求： +1. 只输出 JSON，不要附加解释文本。 +2. JSON 字段固定为，且顺序必须如下： + - reason: string + - allow_expand: boolean + - expand_bytes: integer + - freeze_new_jobs: boolean +3. reason 需要简洁说明决策依据，优先引用 usage_ratio、growth_rate_bytes_per_hour、平台容量和 Prometheus 相关字段。 +4. reason 必须与 allow_expand、expand_bytes、freeze_new_jobs 严格一致： + - allow_expand=false 时，reason 不得写成“建议扩容”“应扩容”或任何支持扩容的表述 + - allow_expand=true 时，expand_bytes 必须为正数，reason 需要明确支持扩容 + - freeze_new_jobs=true 时，reason 必须明确说明需要冻结新作业 + - freeze_new_jobs=false 时，reason 不得写成“建议冻结新作业”或任何支持冻结的表述 +5. expand_bytes 必须使用字节数。 +6. 不要重复输入，不要输出 markdown，不要输出额外内容。` + + directDecisionRefinementInstruction = `你是存储治理决策的一致性修正器。你会看到： +1. 输入 snapshot +2. 首轮 JSON 决策 +3. 外部一致性验证器指出的冲突点 + +请你重写一个新的最终 JSON 决策对象，并修复所有冲突。 +要求： +1. 只输出 JSON，不要解释修改过程。 +2. reason 必须与 allow_expand、expand_bytes、freeze_new_jobs 严格一致。 +3. allow_expand=false 时 expand_bytes 必须为 0。 +4. allow_expand=true 时 expand_bytes 必须为正数。 +5. freeze_new_jobs=true 时 reason 必须明确写出冻结新作业的原因；freeze_new_jobs=false 时 reason 不得写成建议冻结。 +6. 直接基于 snapshot 的证据重写，不要保留首轮输出里自相矛盾的描述。` +) + +type storageDecisionRuntimeConfig struct { + Mode string + ConfigSource string + BaseURL string + APIKey string + ModelName string +} + +type directDecisionAttempt struct { + Decision *LLMDecisionResponse + RawJSON string + RawText string +} + +type directDecisionValidationIssue struct { + Code string `json:"code"` + Message string `json:"message"` +} + +// GetStorageDecisionMode returns the active storage decision mode. +// Database config is used by default, while environment variables can still +// temporarily override it for local testing. +func GetStorageDecisionMode(ctx context.Context) string { + cfg, err := loadStorageDecisionRuntimeConfig(ctx) + if err != nil { + klog.Warningf("GetStorageDecisionMode: failed to load runtime config, fallback to agent: %v", err) + return StorageDecisionModeAgent + } + return normalizeStorageDecisionMode(cfg.Mode) +} + +// AskDirectDecision sends a precomputed storage snapshot to a specialized model. +// It performs a first-pass generation, validates reason/decision consistency, and +// when conflicts are found, feeds them back to the model for one rewrite. +func AskDirectDecision(ctx context.Context, snapshotJSON string) (*LLMDecisionResponse, error) { + llmConfig, err := loadDirectDecisionConfig(ctx) + if err != nil { + return nil, fmt.Errorf("加载运行时 LLM 配置失败: %w", err) + } + + firstAttempt, err := askDirectDecisionOnce(ctx, *llmConfig, snapshotJSON) + if err != nil { + return nil, err + } + + firstIssues := validateDirectDecisionConsistency(*firstAttempt.Decision) + if len(firstIssues) == 0 { + return firstAttempt.Decision, nil + } + + klog.Warningf( + "AskDirectDecision: first-pass direct decision has %d consistency issue(s): %s", + len(firstIssues), + joinValidationIssueMessages(firstIssues), + ) + + refinedAttempt, err := refineDirectDecision( + ctx, + *llmConfig, + snapshotJSON, + firstAttempt.RawJSON, + firstIssues, + ) + if err != nil { + klog.Warningf("AskDirectDecision: refinement failed, keeping first-pass decision: %v", err) + return firstAttempt.Decision, nil + } + + refinedIssues := validateDirectDecisionConsistency(*refinedAttempt.Decision) + switch { + case len(refinedIssues) == 0: + return refinedAttempt.Decision, nil + case len(refinedIssues) < len(firstIssues): + klog.Warningf( + "AskDirectDecision: refined decision still has %d issue(s), but improved from %d: %s", + len(refinedIssues), + len(firstIssues), + joinValidationIssueMessages(refinedIssues), + ) + return refinedAttempt.Decision, nil + default: + klog.Warningf( + "AskDirectDecision: refinement did not improve consistency, keeping first-pass decision. first=%d refined=%d refinedIssues=%s", + len(firstIssues), + len(refinedIssues), + joinValidationIssueMessages(refinedIssues), + ) + return firstAttempt.Decision, nil + } +} + +func askDirectDecisionOnce(ctx context.Context, cfg ProviderConfig, snapshotJSON string) (*directDecisionAttempt, error) { + temperature := 0.0 + prompt := directDecisionInstruction + "\n\n输入:\n" + snapshotJSON + "\n\n输出:\n" + resp, err := callConfiguredCompletion(ctx, cfg, completionRequest{ + Prompt: prompt, + MaxTokens: 256, + Temperature: &temperature, + }) + if err != nil { + return nil, fmt.Errorf("调用直连决策模型失败: %w", err) + } + if len(resp.Choices) == 0 { + return nil, fmt.Errorf("直连决策模型未返回任何候选结果") + } + + return decodeDirectDecisionAttempt(resp.Choices[0].Text) +} + +func refineDirectDecision( + ctx context.Context, + cfg ProviderConfig, + snapshotJSON string, + firstDecisionJSON string, + issues []directDecisionValidationIssue, +) (*directDecisionAttempt, error) { + feedbackJSON, err := json.MarshalIndent(issues, "", " ") + if err != nil { + return nil, fmt.Errorf("failed to marshal direct decision validation feedback: %w", err) + } + + temperature := 0.0 + prompt := directDecisionRefinementInstruction + + "\n\nsnapshot:\n" + snapshotJSON + + "\n\n首轮决策:\n" + firstDecisionJSON + + "\n\n一致性验证反馈:\n" + string(feedbackJSON) + + "\n\n请输出修复后的 JSON 决策:\n" + + resp, err := callConfiguredCompletion(ctx, cfg, completionRequest{ + Prompt: prompt, + MaxTokens: 256, + Temperature: &temperature, + }) + if err != nil { + return nil, fmt.Errorf("failed to refine direct decision: %w", err) + } + if len(resp.Choices) == 0 { + return nil, fmt.Errorf("direct decision refinement returned no choices") + } + + return decodeDirectDecisionAttempt(resp.Choices[0].Text) +} + +func decodeDirectDecisionAttempt(rawText string) (*directDecisionAttempt, error) { + rawJSON := extractFirstJSONObject(rawText) + + var decision LLMDecisionResponse + if err := json.Unmarshal([]byte(rawJSON), &decision); err != nil { + return nil, fmt.Errorf("解析直连决策 JSON 失败: %w\n原始响应: %s", err, rawText) + } + + return &directDecisionAttempt{ + Decision: &decision, + RawJSON: rawJSON, + RawText: rawText, + }, nil +} + +func loadDirectDecisionConfig(ctx context.Context) (*ProviderConfig, error) { + return GetStorageDecisionProviderConfig(ctx) +} + +func GetStorageDecisionProviderConfig(ctx context.Context) (*ProviderConfig, error) { + storageCfg, err := loadStorageDecisionRuntimeConfig(ctx) + if err != nil { + return nil, err + } + + if storageCfg.ConfigSource == StorageDecisionConfigSourceCustom { + if strings.TrimSpace(storageCfg.BaseURL) == "" { + return nil, fmt.Errorf("storage decision base url is not configured") + } + if strings.TrimSpace(storageCfg.ModelName) == "" { + return nil, fmt.Errorf("storage decision model name is not configured") + } + + return &ProviderConfig{ + BaseURL: strings.TrimSpace(storageCfg.BaseURL), + APIKey: strings.TrimSpace(storageCfg.APIKey), + ModelName: strings.TrimSpace(storageCfg.ModelName), + }, nil + } + + return loadRuntimeLLMConfig(ctx) +} + +func loadStorageDecisionRuntimeConfig(ctx context.Context) (*storageDecisionRuntimeConfig, error) { + cfg := &storageDecisionRuntimeConfig{} + + var rows []model.SystemConfig + err := query.GetDB().WithContext(ctx). + Where("key IN ?", []string{ + model.ConfigKeyStorageDecisionMode, + model.ConfigKeyStorageDecisionConfigSource, + model.ConfigKeyStorageDirectModelBaseURL, + model.ConfigKeyStorageDirectModelAPIKey, + model.ConfigKeyStorageDirectModelName, + }). + Find(&rows).Error + if err != nil { + return nil, fmt.Errorf("failed to load storage decision config from database: %w", err) + } + + configMap := make(map[string]string, len(rows)) + for _, row := range rows { + configMap[row.Key] = strings.TrimSpace(row.Value) + } + + cfg.Mode = normalizeStorageDecisionMode(configMap[model.ConfigKeyStorageDecisionMode]) + cfg.ConfigSource = normalizeStorageDecisionConfigSource( + configMap[model.ConfigKeyStorageDecisionConfigSource], + configMap[model.ConfigKeyStorageDirectModelName], + ) + cfg.BaseURL = configMap[model.ConfigKeyStorageDirectModelBaseURL] + cfg.ModelName = configMap[model.ConfigKeyStorageDirectModelName] + + if encryptedKey := configMap[model.ConfigKeyStorageDirectModelAPIKey]; encryptedKey != "" { + plainKey, decryptErr := crypto.Decrypt(encryptedKey) + if decryptErr != nil { + klog.Warningf("loadStorageDecisionRuntimeConfig: failed to decrypt direct model api key, using raw value: %v", decryptErr) + cfg.APIKey = encryptedKey + } else { + cfg.APIKey = plainKey + } + } + + if envMode := strings.TrimSpace(os.Getenv(StorageDecisionModeEnv)); envMode != "" { + cfg.Mode = normalizeStorageDecisionMode(envMode) + } + if envSource := strings.TrimSpace(os.Getenv(StorageDecisionConfigSourceEnv)); envSource != "" { + cfg.ConfigSource = normalizeStorageDecisionConfigSource(envSource, cfg.ModelName) + } + if envBaseURL := strings.TrimSpace(os.Getenv(DirectModelBaseURLEnv)); envBaseURL != "" { + cfg.BaseURL = envBaseURL + } + if envAPIKey := strings.TrimSpace(os.Getenv(DirectModelAPIKeyEnv)); envAPIKey != "" { + cfg.APIKey = envAPIKey + } + if envModelName := strings.TrimSpace(os.Getenv(DirectModelNameEnv)); envModelName != "" { + cfg.ModelName = envModelName + } + if cfg.ConfigSource == StorageDecisionConfigSourceCustom && strings.TrimSpace(cfg.BaseURL) == "" { + cfg.BaseURL = DefaultDirectModelBaseURL + } + + return cfg, nil +} + +func normalizeStorageDecisionMode(mode string) string { + switch strings.TrimSpace(strings.ToLower(mode)) { + case StorageDecisionModeDirect: + return StorageDecisionModeDirect + default: + return StorageDecisionModeAgent + } +} + +func normalizeStorageDecisionConfigSource(source, modelName string) string { + switch strings.TrimSpace(strings.ToLower(source)) { + case StorageDecisionConfigSourceCustom: + return StorageDecisionConfigSourceCustom + case StorageDecisionConfigSourcePlatform: + return StorageDecisionConfigSourcePlatform + default: + if strings.TrimSpace(modelName) != "" { + return StorageDecisionConfigSourceCustom + } + return StorageDecisionConfigSourcePlatform + } +} + +func extractFirstJSONObject(text string) string { + start := strings.Index(text, "{") + if start == -1 { + return strings.TrimSpace(text) + } + + depth := 0 + inString := false + escaped := false + for i := start; i < len(text); i++ { + ch := text[i] + + if escaped { + escaped = false + continue + } + if ch == '\\' { + escaped = true + continue + } + if ch == '"' { + inString = !inString + continue + } + if inString { + continue + } + + switch ch { + case '{': + depth++ + case '}': + depth-- + if depth == 0 { + return strings.TrimSpace(text[start : i+1]) + } + } + } + + return strings.TrimSpace(text[start:]) +} + +func validateDirectDecisionConsistency(decision LLMDecisionResponse) []directDecisionValidationIssue { + issues := make([]directDecisionValidationIssue, 0) + reason := strings.TrimSpace(strings.ToLower(decision.Reason)) + + if reason == "" { + issues = append(issues, directDecisionValidationIssue{ + Code: "empty_reason", + Message: "reason is empty and does not explain the decision fields", + }) + } + + if decision.AllowExpand && decision.ExpandBytes <= 0 { + issues = append(issues, directDecisionValidationIssue{ + Code: "expand_bytes_missing", + Message: fmt.Sprintf( + "allow_expand=true but expand_bytes=%d; expansion decisions must use a positive expand_bytes", + decision.ExpandBytes, + ), + }) + } + if !decision.AllowExpand && decision.ExpandBytes != 0 { + issues = append(issues, directDecisionValidationIssue{ + Code: "expand_bytes_should_be_zero", + Message: fmt.Sprintf("allow_expand=false but expand_bytes=%d; expand_bytes must be 0 when expansion is disabled", decision.ExpandBytes), + }) + } + + if reason == "" { + return issues + } + + hasExpandNegative := containsReasonPhrase(reason, directExpandNegativePhrases) + hasExpandPositive := containsReasonPhrase( + removeReasonPhrases(reason, directExpandNegativePhrases), + directExpandPositivePhrases, + ) + hasFreezeNegative := containsReasonPhrase(reason, directFreezeNegativePhrases) + hasFreezePositive := containsReasonPhrase( + removeReasonPhrases(reason, directFreezeNegativePhrases), + directFreezePositivePhrases, + ) + + if decision.AllowExpand && hasExpandNegative { + issues = append(issues, directDecisionValidationIssue{ + Code: "reason_blocks_expansion", + Message: "allow_expand=true but reason describes expansion as unnecessary, blocked, or forbidden", + }) + } + if !decision.AllowExpand && hasExpandPositive { + issues = append(issues, directDecisionValidationIssue{ + Code: "reason_supports_expansion", + Message: "allow_expand=false but reason still recommends or supports expansion", + }) + } + if decision.FreezeNewJobs && hasFreezeNegative { + issues = append(issues, directDecisionValidationIssue{ + Code: "reason_blocks_freeze", + Message: "freeze_new_jobs=true but reason says freezing new jobs is unnecessary or should not happen", + }) + } + if !decision.FreezeNewJobs && hasFreezePositive { + issues = append(issues, directDecisionValidationIssue{ + Code: "reason_supports_freeze", + Message: "freeze_new_jobs=false but reason still recommends freezing or pausing new jobs", + }) + } + + return issues +} + +func joinValidationIssueMessages(issues []directDecisionValidationIssue) string { + parts := make([]string, 0, len(issues)) + for _, issue := range issues { + parts = append(parts, issue.Code+": "+issue.Message) + } + return strings.Join(parts, "; ") +} + +func containsReasonPhrase(reason string, phrases []string) bool { + for _, phrase := range phrases { + if strings.Contains(reason, phrase) { + return true + } + } + return false +} + +func removeReasonPhrases(reason string, phrases []string) string { + cleaned := reason + for _, phrase := range phrases { + cleaned = strings.ReplaceAll(cleaned, phrase, " ") + } + return cleaned +} + +var directExpandPositivePhrases = []string{ + "建议扩容", + "需要扩容", + "应当扩容", + "应该扩容", + "可以扩容", + "允许扩容", + "优先扩容", + "保守扩容", + "临时扩容", + "扩容保护", + "expand", + "allow expand", +} + +var directExpandNegativePhrases = []string{ + "无需扩容", + "不需要扩容", + "不应扩容", + "不应该扩容", + "不建议扩容", + "不能扩容", + "不可扩容", + "不允许扩容", + "禁止扩容", + "无需临时扩容", + "no expand", + "do not expand", + "deny expansion", +} + +var directFreezePositivePhrases = []string{ + "冻结新作业", + "冻结新任务", + "暂停新作业", + "暂停新任务", + "禁止新作业", + "禁止新任务", + "停止新作业", + "停止新任务", + "freeze new jobs", + "freeze new job", + "freeze jobs", +} + +var directFreezeNegativePhrases = []string{ + "不冻结新作业", + "不冻结新任务", + "无需冻结", + "不需要冻结", + "不必冻结", + "继续接受新作业", + "继续提交新作业", + "no freeze", + "do not freeze", +} diff --git a/backend/pkg/llm/direct_decision_test.go b/backend/pkg/llm/direct_decision_test.go new file mode 100644 index 000000000..53c9bd41a --- /dev/null +++ b/backend/pkg/llm/direct_decision_test.go @@ -0,0 +1,118 @@ +package llm + +import "testing" + +func TestValidateDirectDecisionConsistencyFieldConflicts(t *testing.T) { + tests := []struct { + name string + decision LLMDecisionResponse + wantCode string + }{ + { + name: "expand enabled but bytes missing", + decision: LLMDecisionResponse{ + Reason: "建议扩容保护作业", + AllowExpand: true, + ExpandBytes: 0, + }, + wantCode: "expand_bytes_missing", + }, + { + name: "expand disabled but bytes kept", + decision: LLMDecisionResponse{ + Reason: "无需扩容，保持观察", + AllowExpand: false, + ExpandBytes: 1024, + }, + wantCode: "expand_bytes_should_be_zero", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + issues := validateDirectDecisionConsistency(tc.decision) + if !hasValidationIssue(issues, tc.wantCode) { + t.Fatalf("expected issue %q, got %+v", tc.wantCode, issues) + } + }) + } +} + +func TestValidateDirectDecisionConsistencyReasonConflicts(t *testing.T) { + tests := []struct { + name string + decision LLMDecisionResponse + wantCode string + }{ + { + name: "allow expand but reason says no expansion", + decision: LLMDecisionResponse{ + Reason: "当前无需扩容，继续观察即可", + AllowExpand: true, + ExpandBytes: 1024, + }, + wantCode: "reason_blocks_expansion", + }, + { + name: "disallow expand but reason still supports expansion", + decision: LLMDecisionResponse{ + Reason: "建议扩容保护落盘阶段", + AllowExpand: false, + ExpandBytes: 0, + }, + wantCode: "reason_supports_expansion", + }, + { + name: "freeze enabled but reason says no freeze", + decision: LLMDecisionResponse{ + Reason: "当前不需要冻结新作业，只需观察", + FreezeNewJobs: true, + AllowExpand: false, + ExpandBytes: 0, + }, + wantCode: "reason_blocks_freeze", + }, + { + name: "freeze disabled but reason still recommends freeze", + decision: LLMDecisionResponse{ + Reason: "建议冻结新作业，避免继续上涨", + FreezeNewJobs: false, + AllowExpand: false, + ExpandBytes: 0, + }, + wantCode: "reason_supports_freeze", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + issues := validateDirectDecisionConsistency(tc.decision) + if !hasValidationIssue(issues, tc.wantCode) { + t.Fatalf("expected issue %q, got %+v", tc.wantCode, issues) + } + }) + } +} + +func TestValidateDirectDecisionConsistencyAlignedDecision(t *testing.T) { + decision := LLMDecisionResponse{ + Reason: "usage_ratio 接近阈值且平台仍有余量，建议扩容 21474836480 字节保护作业，本轮不冻结新作业", + AllowExpand: true, + ExpandBytes: 21474836480, + FreezeNewJobs: false, + } + + issues := validateDirectDecisionConsistency(decision) + if len(issues) != 0 { + t.Fatalf("expected no issues, got %+v", issues) + } +} + +func hasValidationIssue(issues []directDecisionValidationIssue, code string) bool { + for _, issue := range issues { + if issue.Code == code { + return true + } + } + return false +} diff --git a/backend/pkg/llm/llm_tools.go b/backend/pkg/llm/llm_tools.go new file mode 100644 index 000000000..989db3140 --- /dev/null +++ b/backend/pkg/llm/llm_tools.go @@ -0,0 +1,1012 @@ +//nolint:gocritic,gocyclo,mnd,lll,staticcheck,unused // Storage governance tool wiring keeps prompts, handlers, and K8s traversal centralized. +package llm + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + + "github.com/raids-lab/crater/dao/model" + "github.com/raids-lab/crater/dao/query" + "github.com/raids-lab/crater/pkg/ceph" + "github.com/raids-lab/crater/pkg/config" + "github.com/raids-lab/crater/pkg/monitor" +) + +const ( + labelKeyTaskUser = "crater.raids.io/task-user" +) + +// ---- 业务响应类型 ---- + +type LLMDecisionResponse struct { + Reason string `json:"reason"` + AllowExpand bool `json:"allow_expand"` + ExpandBytes int64 `json:"expand_bytes"` + FreezeNewJobs bool `json:"freeze_new_jobs"` +} + +type PlatformCapacityResponse struct { + TotalCapacity int64 `json:"total_capacity"` + UsedCapacity int64 `json:"used_capacity"` + AvailableCapacity int64 `json:"available_capacity"` +} + +type TenantPodResponse struct { + PodName string `json:"pod_name"` + Phase string `json:"phase"` + GPURrequests int `json:"gpu_requests"` +} + +type TenantPodsResponse struct { + TenantID string `json:"tenant_id"` + Pods []TenantPodResponse `json:"pods"` +} + +type PodDetailsResponse struct { + PodName string `json:"pod_name"` + StartTime time.Time `json:"start_time"` + RunningTime int `json:"running_time"` + ContainerImages []string `json:"container_images"` + RestartCount int `json:"restart_count"` + GPUModel string `json:"gpu_model"` + GPUCount int `json:"gpu_count"` + CPUCount int `json:"cpu_count"` + MemorySize string `json:"memory_size"` +} + +type ComputeQuotaResponse struct { + TenantID string `json:"tenant_id"` + GPULimit int `json:"gpu_limit"` + GPURequest int `json:"gpu_request"` + CPULimit int `json:"cpu_limit"` + CPURequest int `json:"cpu_request"` + MemoryLimit string `json:"memory_limit"` + MemoryRequest string `json:"memory_request"` +} + +type UsageTrend struct { + Timestamp time.Time `json:"timestamp"` + UsageBytes int64 `json:"usage_bytes"` +} + +type TenantStorageTrendResponse struct { + TenantID string `json:"tenant_id"` + CurrentUsage int64 `json:"current_usage"` + History []UsageTrend `json:"history"` +} + +// ---- DeepSeek / OpenAI 兼容类型 ---- + +type dsToolFunction struct { + Name string `json:"name"` + Description string `json:"description"` + Parameters any `json:"parameters"` +} + +type dsTool struct { + Type string `json:"type"` // "function" + Function dsToolFunction `json:"function"` +} + +type dsToolCallFunction struct { + Name string `json:"name"` + Arguments string `json:"arguments"` +} + +type dsToolCall struct { + ID string `json:"id"` + Type string `json:"type"` + Function dsToolCallFunction `json:"function"` +} + +type dsMessage struct { + Role string `json:"role"` + Content string `json:"content,omitempty"` + ToolCalls []dsToolCall `json:"tool_calls,omitempty"` + ToolCallID string `json:"tool_call_id,omitempty"` +} + +type dsRequest struct { + Model string `json:"model"` + Messages []dsMessage `json:"messages"` + Tools []dsTool `json:"tools,omitempty"` + MaxTokens int `json:"max_tokens,omitempty"` +} + +type dsResponse struct { + Choices []struct { + Message dsMessage `json:"message"` + FinishReason string `json:"finish_reason"` + } `json:"choices"` + Error *struct { + Message string `json:"message"` + } `json:"error,omitempty"` +} + +// ---- 工具定义 ---- + +func getTools() []dsTool { + return []dsTool{ + { + Type: "function", + Function: dsToolFunction{ + Name: "query_platform_capacity", + Description: "获取整个平台（硬限制 8TB）的总量、已用量、可用量", + Parameters: map[string]any{ + "type": "object", + "properties": map[string]any{}, + "required": []string{}, + }, + }, + }, + { + Type: "function", + Function: dsToolFunction{ + Name: "list_tenant_pods", + Description: "列出租户当前活跃的 Pod 列表", + Parameters: map[string]any{ + "type": "object", + "properties": map[string]any{ + "tenant_id": map[string]any{"type": "string", "description": "租户 ID"}, + }, + "required": []string{"tenant_id"}, + }, + }, + }, + { + Type: "function", + Function: dsToolFunction{ + Name: "inspect_pod_details", + Description: "深入查看某个特定 Pod 的启动时间、镜像和重启次数、GPU型号、个数，CPU核数和内存大小", + Parameters: map[string]any{ + "type": "object", + "properties": map[string]any{ + "tenant_id": map[string]any{"type": "string", "description": "租户 ID"}, + "pod_name": map[string]any{"type": "string", "description": "Pod 名称"}, + }, + "required": []string{"tenant_id", "pod_name"}, + }, + }, + }, + { + Type: "function", + Function: dsToolFunction{ + Name: "get_tenant_compute_quota", + Description: "获取租户当前所有活跃 Pod 的 GPU/CPU/内存请求与限制总量", + Parameters: map[string]any{ + "type": "object", + "properties": map[string]any{ + "tenant_id": map[string]any{"type": "string", "description": "租户 ID"}, + }, + "required": []string{"tenant_id"}, + }, + }, + }, + { + Type: "function", + Function: dsToolFunction{ + Name: "query_tenant_storage_trend", + Description: "获取指定租户当前的真实存储占用以及最近的几次历史记录，用于推导增长斜率", + Parameters: map[string]any{ + "type": "object", + "properties": map[string]any{ + "tenant_id": map[string]any{"type": "string", "description": "租户 ID"}, + }, + "required": []string{"tenant_id"}, + }, + }, + }, + { + Type: "function", + Function: dsToolFunction{ + Name: "query_pod_realtime_metrics", + Description: "通过 Prometheus 查询指定 Pod 过去 5 分钟的真实资源利用率（CPU 核数、内存 MB、GPU 利用率 %、GPU 显存 MB）。用于检测僵尸作业：gpu_data_available=true 时以 gpu_util_percent 判断，gpu_data_available=false 时以 cpu_cores 判断（< 0.05 核视为进程挂死）", + Parameters: map[string]any{ + "type": "object", + "properties": map[string]any{ + "pod_name": map[string]any{"type": "string", "description": "Pod 名称"}, + }, + "required": []string{"pod_name"}, + }, + }, + }, + { + Type: "function", + Function: dsToolFunction{ + Name: "query_pod_gpu_history", + Description: "查询指定 Pod 在整个生命周期（或最近 24 小时）内的 GPU 历史利用率，返回平均值和最大值。用于区分两种低 GPU 利用率场景：(1) 作业曾经高强度使用 GPU（max_util_ever > 50%），当前低利用率说明正处于落盘/IO 阶段，是有价值的作业；(2) 从未有过高 GPU 利用率（max_util_ever ≈ 0%），则可能是僵尸作业或纯 CPU 作业。data_available=false 表示 DCGM 未采集到该 Pod 数据", + Parameters: map[string]any{ + "type": "object", + "properties": map[string]any{ + "pod_name": map[string]any{"type": "string", "description": "Pod 名称"}, + "duration_hours": map[string]any{"type": "number", "description": "查询历史时长（小时），默认 24。建议传入作业已运行时长以覆盖完整生命周期"}, + }, + "required": []string{"pod_name"}, + }, + }, + }, + { + Type: "function", + Function: dsToolFunction{ + Name: "diagnose_prometheus", + Description: "诊断 Prometheus 连通性与 DCGM 监控可用性。返回：Prometheus 是否可达、DCGM 指标是否存在、当前 namespace 下被 DCGM 追踪的 Pod 列表，以及（可选）指定 Pod 的原始指标查询结果。当 query_pod_realtime_metrics 返回 gpu_data_available=false 时必须调用此工具排查原因", + Parameters: map[string]any{ + "type": "object", + "properties": map[string]any{ + "pod_name": map[string]any{"type": "string", "description": "（可选）需要额外查询实时指标的 Pod 名称"}, + }, + "required": []string{}, + }, + }, + }, + } +} + +// ---- Agent 主循环 ---- + +func AskAgentForDecision(clientset kubernetes.Interface, restConfig *rest.Config, tenantID string, promClient monitor.PrometheusInterface) (*LLMDecisionResponse, error) { + ctx := context.Background() + llmConfig, err := GetStorageDecisionProviderConfig(ctx) + if err != nil { + return nil, fmt.Errorf("加载存储决策 LLM 配置失败: %w", err) + } + skillText, skillSource, err := loadStorageAgentSkill() + if err != nil { + return nil, fmt.Errorf("加载 storage agent skill 失败: %w", err) + } + + const ( + systemPrompt = `你是一个平台运维 AI 助手，负责分析租户存储告警并给出临时扩容决策。 + +【背景知识】 +GPU 密集型训练完成后，作业通常进入 IO 密集型落盘阶段（写 checkpoint / 模型参数）。 +此阶段特征：GPU 利用率降至 0、CPU 极低（IO bound）、存储快速增长。 +这是有价值的正常行为，必须优先保护。区分落盘作业与僵尸作业的关键依据是 GPU 历史记录： + · 落盘作业：历史上曾有过高 GPU 利用率（max_util_ever > 50%），说明完成了真实的 GPU 计算 + · 僵尸作业：整个生命周期 GPU 利用率始终接近 0，从未进行有效计算 + +【工具调用策略 — 请严格按序执行，禁止冗余调用】 + +第一步（必做）：调用 query_tenant_storage_trend。 + 从 history 记录计算增长速率（growth_rate）： + · 将记录按时间排序，用最新与最早的 usage_bytes 之差除以时间跨度，得到 growth_rate（字节/小时） + · 不足 2 条记录时视为"增长趋势未知" + +▸ usage_ratio >= 100%（已超出配额）： + → freeze_new_jobs=true，allow_expand=false，立即输出 JSON。 + +▸ 90% <= usage_ratio < 100%（接近配额）： + + 第二步：调用 query_platform_capacity 确认平台剩余空间。 + · 平台空间不足（可用量 <= 当前配额 20%）： + → allow_expand=false，freeze_new_jobs=false + → reason 说明平台空间不足，建议管理员释放集群存储，输出 JSON。 + + 第三步（平台空间充足时）：判断是否需要作业层面分析。 + · 增长平缓（growth_rate < 配额 1%/小时）或趋势未知： + → 属于正常数据积累，allow_expand=true，expand_bytes=配额的 20%，freeze_new_jobs=false，输出 JSON。 + · 增长较快（growth_rate >= 配额 1%/小时）： + → 进入第四步，分析活跃作业以辅助决策。 + + 第四步（仅在增长较快时执行）： + → 调用 list_tenant_pods 获取活跃 Pod 列表 + → 对每个 gpu_requests > 0 的 Pod，调用 query_pod_gpu_history（duration_hours 传入该 Pod 的预估运行时长，默认 24） + → 根据 GPU 历史数据判断作业性质： + + · max_util_ever_percent > 50%（曾进行 GPU 密集计算）： + → 判定为【落盘作业】，当前低利用率是正常落盘行为 + → allow_expand=true，expand_bytes=配额的 50%（为落盘预留充足空间） + → reason 中注明："检测到作业 [pod名] 历史 GPU 峰值为 X%，当前处于落盘阶段，扩容保护训练成果" + + · max_util_ever_percent <= 50% 且 data_available=true（DCGM 有数据但 GPU 从未高负载）： + → 判定为【可疑作业】，在 reason 中说明情况，建议人工排查 + → 但仍 allow_expand=true，expand_bytes=配额的 20%（不确定时优先保护用户） + → reason 中注明："作业 [pod名] GPU 历史峰值仅 X%，未见明显 GPU 计算，存储增长原因待排查" + + · data_available=false（DCGM 无数据，无法获取 GPU 历史）： + → 无法区分，保守但偏向保护：allow_expand=true，expand_bytes=配额的 20% + → reason 中注明 GPU 历史数据不可用 + + → freeze_new_jobs=false，输出 JSON。 + +▸ usage_ratio < 90%： + → 无需冻结，无需扩容，立即输出 JSON。 + +【仅在上述流程中明确要求时才调用对应工具；inspect_pod_details / get_tenant_compute_quota / query_pod_realtime_metrics / diagnose_prometheus 不得主动调用】 + +完成分析后，仅输出如下格式的 JSON，不要附加任何其他文字： +{"reason": "<决策理由，必须包含 usage_ratio、growth_rate 及关键作业诊断证据>", "allow_expand": true/false, "expand_bytes": <字节数>, "freeze_new_jobs": true/false}` + maxLoops = 10 + ) + + messages := []dsMessage{ + {Role: "system", Content: systemPrompt}, + } + if skillText != "" { + klog.Infof("AskAgentForDecision[%s] loaded storage agent skill from %s", tenantID, skillSource) + messages = append(messages, dsMessage{ + Role: "system", + Content: "以下为存储治理领域技能补充，请在工具调用和最终决策时遵循：\n" + skillText, + }) + } + messages = append(messages, dsMessage{ + Role: "user", + Content: fmt.Sprintf("租户 %s 触发存储告警（使用率超过 90%%），请调用工具进行全面排查，然后给出是否需要临时扩容的决策。", tenantID), + }) + + for i := 0; i < maxLoops; i++ { + resp, err := callConfiguredLLM(ctx, *llmConfig, dsRequest{ + MaxTokens: 4096, + Tools: getTools(), + Messages: messages, + }) + if err != nil { + return nil, fmt.Errorf("调用配置化 LLM Provider 失败: %w", err) + } + + choice := resp.Choices[0] + messages = append(messages, choice.Message) + + // 无工具调用 → 模型给出了最终决策 + if choice.FinishReason == "stop" || len(choice.Message.ToolCalls) == 0 { + klog.Infof("AskAgentForDecision[%s] 完整分析结果:\n%s", tenantID, choice.Message.Content) + raw := extractJSON(choice.Message.Content) + var decision LLMDecisionResponse + if err := json.Unmarshal([]byte(raw), &decision); err != nil { + return nil, fmt.Errorf("解析决策 JSON 失败: %w\n原始响应: %s", err, choice.Message.Content) + } + return &decision, nil + } + + // 执行所有工具调用，收集结果 + for _, toolCall := range choice.Message.ToolCalls { + result, toolErr := dispatchTool(clientset, restConfig, tenantID, promClient, toolCall) + if toolErr != nil { + result = fmt.Sprintf(`{"error": %q}`, toolErr.Error()) + } + messages = append(messages, dsMessage{ + Role: "tool", + ToolCallID: toolCall.ID, + Content: result, + }) + } + } + + return nil, fmt.Errorf("超过最大对话轮数 (%d)，未能得出决策", maxLoops) +} + +// dispatchTool 根据工具名分发到对应 handler +func dispatchTool(clientset kubernetes.Interface, restConfig *rest.Config, tenantID string, promClient monitor.PrometheusInterface, toolCall dsToolCall) (string, error) { + switch toolCall.Function.Name { + case "query_platform_capacity": + return HandleQueryPlatformCapacity(clientset, restConfig) + + case "list_tenant_pods": + return HandleListTenantPods(clientset, tenantID) + + case "inspect_pod_details": + var args struct { + TenantID string `json:"tenant_id"` + PodName string `json:"pod_name"` + } + if err := json.Unmarshal([]byte(toolCall.Function.Arguments), &args); err != nil { + return "", fmt.Errorf("解析 inspect_pod_details 参数失败: %w", err) + } + return HandleInspectPodDetails(clientset, args.TenantID, args.PodName) + + case "get_tenant_compute_quota": + return HandleGetComputeQuota(clientset, tenantID) + + case "query_tenant_storage_trend": + return HandleQueryTenantStorageTrend(clientset, restConfig, tenantID) + + case "query_pod_realtime_metrics": + var args struct { + PodName string `json:"pod_name"` + } + if err := json.Unmarshal([]byte(toolCall.Function.Arguments), &args); err != nil { + return "", fmt.Errorf("解析 query_pod_realtime_metrics 参数失败: %w", err) + } + return HandleQueryPodRealtimeMetrics(args.PodName, clientset, promClient) + + case "query_pod_gpu_history": + var args struct { + PodName string `json:"pod_name"` + DurationHours float64 `json:"duration_hours"` + } + if err := json.Unmarshal([]byte(toolCall.Function.Arguments), &args); err != nil { + return "", fmt.Errorf("解析 query_pod_gpu_history 参数失败: %w", err) + } + return HandleQueryPodGPUHistory(args.PodName, args.DurationHours, promClient) + + case "diagnose_prometheus": + var args struct { + PodName string `json:"pod_name"` + } + // pod_name 是可选参数，忽略解析错误 + _ = json.Unmarshal([]byte(toolCall.Function.Arguments), &args) + result := DiagnosePrometheus(args.PodName, clientset, promClient) + data, err := json.Marshal(result) + if err != nil { + return "", fmt.Errorf("序列化诊断结果失败: %w", err) + } + return string(data), nil + + default: + return "", fmt.Errorf("未知工具: %s", toolCall.Function.Name) + } +} + +// ---- Prometheus 诊断 ---- + +// PrometheusDiagnosis 是 DiagnosePrometheus 的返回结构。 +type PrometheusDiagnosis struct { + PrometheusURL string `json:"prometheus_url"` + Reachable bool `json:"reachable"` + ConnectError string `json:"connect_error,omitempty"` + DCGMAvailable bool `json:"dcgm_available"` + DCGMSeriesCount int `json:"dcgm_series_count"` + TrackedPods []string `json:"tracked_pods_in_namespace"` + Namespace string `json:"namespace"` + PodMetrics *struct { + PodName string `json:"pod_name"` + CPUCores float64 `json:"cpu_cores"` + MemoryMB float64 `json:"memory_mb"` + GPUUtilPercent float64 `json:"gpu_util_percent"` + GPUDataFound bool `json:"gpu_data_found"` + } `json:"pod_metrics,omitempty"` +} + +// DiagnosePrometheus 验证 Prometheus 连通性、DCGM 指标可用性， +// 并列出 jobNamespace 下被 DCGM 追踪的 Pod。 +// podName 非空时额外查询该 Pod 的实时指标。 +func DiagnosePrometheus(podName string, _ kubernetes.Interface, promClient monitor.PrometheusInterface) *PrometheusDiagnosis { + jobNamespace := config.GetConfig().Namespaces.Job + + diag := &PrometheusDiagnosis{ + PrometheusURL: config.GetConfig().PrometheusAPI, + Namespace: jobNamespace, + TrackedPods: []string{}, + } + + if promClient == nil { + diag.ConnectError = "Prometheus 客户端未初始化" + return diag + } + + // 1. 基础连通性：vector(1) 强制返回向量，任何 Prometheus 均支持 + v, ok, err := promClient.QueryInstant("vector(1)") + if err != nil { + diag.ConnectError = fmt.Sprintf("Prometheus 查询失败: %v", err) + return diag + } + if !ok || v != 1 { + diag.ConnectError = fmt.Sprintf("Prometheus 返回异常值: ok=%v val=%v", ok, v) + return diag + } + diag.Reachable = true + + // 2. 检查 DCGM 是否存在任意时间序列 + if cnt, ok, err := promClient.QueryInstant("count(DCGM_FI_DEV_GPU_UTIL)"); err == nil && ok { + diag.DCGMAvailable = true + diag.DCGMSeriesCount = int(cnt) + } + + // 3. 列出 jobNamespace 下 DCGM 追踪的所有 Pod 名称 + pods := promClient.QueryInstantLabels( + fmt.Sprintf(`count by (pod) (DCGM_FI_DEV_GPU_UTIL{namespace=%q})`, jobNamespace), + "pod") + if pods != nil { + diag.TrackedPods = pods + } + + // 4. 如果指定了 Pod，额外查询其实时指标 + if podName != "" { + pm := &struct { + PodName string `json:"pod_name"` + CPUCores float64 `json:"cpu_cores"` + MemoryMB float64 `json:"memory_mb"` + GPUUtilPercent float64 `json:"gpu_util_percent"` + GPUDataFound bool `json:"gpu_data_found"` + }{PodName: podName} + + if v, ok, _ := promClient.QueryInstant( + fmt.Sprintf(`sum(rate(container_cpu_usage_seconds_total{pod=%q,container!=""}[5m]))`, podName), + ); ok { + pm.CPUCores = v + } + if v, ok, _ := promClient.QueryInstant( + fmt.Sprintf(`sum(container_memory_usage_bytes{pod=%q,container!=""})`, podName), + ); ok { + pm.MemoryMB = v / 1024 / 1024 + } + // GPU：先带 namespace，再退化 + for _, q := range []string{ + fmt.Sprintf(`avg(DCGM_FI_DEV_GPU_UTIL{namespace=%q,pod=%q})`, jobNamespace, podName), + fmt.Sprintf(`avg(DCGM_FI_DEV_GPU_UTIL{pod=%q})`, podName), + } { + if v, ok, _ := promClient.QueryInstant(q); ok { + pm.GPUUtilPercent = v + pm.GPUDataFound = true + break + } + } + diag.PodMetrics = pm + } + + return diag +} + +// HandleQueryPodRealtimeMetrics 通过 Prometheus 查询 Pod 的真实资源利用率， +// 用于识别高申请低利用的僵尸作业。 +func HandleQueryPodRealtimeMetrics(podName string, _ kubernetes.Interface, promClient monitor.PrometheusInterface) (string, error) { + if promClient == nil { + return "", fmt.Errorf("Prometheus 客户端未初始化") + } + + // Job namespace — DCGM 标签与 Kubernetes namespace 绑定，必须传入才能匹配到指标 + jobNamespace := config.GetConfig().Namespaces.Job + + type metricsResult struct { + PodName string `json:"pod_name"` + Namespace string `json:"namespace"` + CPUCores float64 `json:"cpu_cores"` + MemoryMB float64 `json:"memory_mb"` + GPUUtilPercent float64 `json:"gpu_util_percent"` + GPUMemoryMB float64 `json:"gpu_memory_mb"` + GPUDataAvailable bool `json:"gpu_data_available"` + Note string `json:"note"` + } + + res := metricsResult{PodName: podName, Namespace: jobNamespace} + + // CPU 使用量（核数，5 分钟均值） + if v, ok, err := promClient.QueryInstant( + fmt.Sprintf(`sum(rate(container_cpu_usage_seconds_total{pod=%q,container!=""}[5m]))`, podName), + ); err != nil { + klog.Warningf("query_pod_realtime_metrics: CPU 查询失败 pod=%s: %v", podName, err) + } else if ok { + res.CPUCores = v + } + + // 内存使用量（MB） + if v, ok, err := promClient.QueryInstant( + fmt.Sprintf(`sum(container_memory_usage_bytes{pod=%q,container!=""})`, podName), + ); err != nil { + klog.Warningf("query_pod_realtime_metrics: 内存查询失败 pod=%s: %v", podName, err) + } else if ok { + res.MemoryMB = v / 1024 / 1024 + } + + // GPU 利用率（%，来自 DCGM_FI_DEV_GPU_UTIL） + // 优先用 namespace+pod 双标签（与现有 monitor 包一致），若无结果则退化为仅 pod 标签 + gpuUtilQueries := []string{ + fmt.Sprintf(`avg(DCGM_FI_DEV_GPU_UTIL{namespace=%q,pod=%q})`, jobNamespace, podName), + fmt.Sprintf(`avg(DCGM_FI_DEV_GPU_UTIL{pod=%q})`, podName), + } + for _, q := range gpuUtilQueries { + v, ok, err := promClient.QueryInstant(q) + if err != nil { + klog.Warningf("query_pod_realtime_metrics: GPU 利用率查询失败 pod=%s query=%s: %v", podName, q, err) + continue + } + if ok { + res.GPUUtilPercent = v + res.GPUDataAvailable = true + break + } + } + + // GPU 显存使用量（MB，来自 DCGM_FI_DEV_FB_USED） + gpuMemQueries := []string{ + fmt.Sprintf(`avg(DCGM_FI_DEV_FB_USED{namespace=%q,pod=%q})`, jobNamespace, podName), + fmt.Sprintf(`avg(DCGM_FI_DEV_FB_USED{pod=%q})`, podName), + } + for _, q := range gpuMemQueries { + v, ok, err := promClient.QueryInstant(q) + if err != nil { + klog.Warningf("query_pod_realtime_metrics: GPU 显存查询失败 pod=%s query=%s: %v", podName, q, err) + continue + } + if ok { + res.GPUMemoryMB = v + break + } + } + + res.Note = "cpu_cores 为过去 5 分钟均值；memory_mb 为当前值；gpu_util_percent/gpu_memory_mb 来自 DCGM（gpu_data_available=false 表示该 Pod 无 GPU 或 DCGM 未采集到数据）" + + data, err := json.Marshal(res) + if err != nil { + return "", fmt.Errorf("序列化利用率响应失败: %w", err) + } + return string(data), nil +} + +// HandleQueryPodGPUHistory 查询 Pod 在指定时间窗口内的 GPU 历史利用率， +// 用于区分"正在落盘的有价值作业"（历史上曾高强度用过 GPU）与"僵尸作业"（从未有效使用 GPU）。 +func HandleQueryPodGPUHistory(podName string, durationHours float64, promClient monitor.PrometheusInterface) (string, error) { + if promClient == nil { + return "", fmt.Errorf("Prometheus 客户端未初始化") + } + if durationHours <= 0 { + durationHours = 24 + } + + jobNamespace := config.GetConfig().Namespaces.Job + dur := fmt.Sprintf("%.0fh", durationHours) + // 不足 1 小时时用分钟表示，避免 Prometheus 解析错误 + if durationHours < 1 { + dur = fmt.Sprintf("%.0fm", durationHours*60) + } + + type gpuHistoryResult struct { + PodName string `json:"pod_name"` + DurationHours float64 `json:"duration_hours"` + AvgUtil float64 `json:"avg_util_percent"` + MaxUtil float64 `json:"max_util_ever_percent"` + DataAvailable bool `json:"data_available"` + Note string `json:"note"` + } + + res := gpuHistoryResult{ + PodName: podName, + DurationHours: durationHours, + Note: fmt.Sprintf("查询过去 %s 内的 GPU 利用率历史。max_util_ever_percent > 50 表明作业曾进行 GPU 密集型计算（如模型训练），当前低利用率很可能是落盘/IO 阶段", dur), + } + + // 平均利用率：namespace+pod 优先，退化为仅 pod + avgQueries := []string{ + fmt.Sprintf(`avg_over_time(DCGM_FI_DEV_GPU_UTIL{namespace=%q,pod=%q}[%s])`, jobNamespace, podName, dur), + fmt.Sprintf(`avg_over_time(DCGM_FI_DEV_GPU_UTIL{pod=%q}[%s])`, podName, dur), + } + for _, q := range avgQueries { + if v, ok, err := promClient.QueryInstant(q); err == nil && ok { + res.AvgUtil = v + res.DataAvailable = true + break + } + } + + // 峰值利用率：用 max_over_time 捕获历史最高点 + maxQueries := []string{ + fmt.Sprintf(`max_over_time(DCGM_FI_DEV_GPU_UTIL{namespace=%q,pod=%q}[%s])`, jobNamespace, podName, dur), + fmt.Sprintf(`max_over_time(DCGM_FI_DEV_GPU_UTIL{pod=%q}[%s])`, podName, dur), + } + for _, q := range maxQueries { + if v, ok, err := promClient.QueryInstant(q); err == nil && ok { + res.MaxUtil = v + res.DataAvailable = true + break + } + } + + data, err := json.Marshal(res) + if err != nil { + return "", fmt.Errorf("序列化 GPU 历史响应失败: %w", err) + } + return string(data), nil +} + +// ---- Handler 实现 ---- + +func HandleQueryPlatformCapacity(clientset kubernetes.Interface, restConfig *rest.Config) (string, error) { + totalCapacity, usedCapacity, err := ceph.GetCraterStorageCapacity( + clientset, restConfig, config.GetConfig().Namespaces.Job, + ) + if err != nil { + return "", fmt.Errorf("获取平台容量失败: %w", err) + } + availableCapacity := ceph.AvailableBytes(totalCapacity, usedCapacity) + + data, err := json.Marshal(map[string]any{ + "total_capacity_bytes": totalCapacity, + "used_capacity_bytes": usedCapacity, + "available_capacity_bytes": availableCapacity, + "total_capacity_formatted": formatBytes(totalCapacity), + "used_capacity_formatted": formatBytes(usedCapacity), + "note": "所有容量单位均为字节(bytes)，formatted 字段为人类可读格式", + }) + if err != nil { + return "", fmt.Errorf("序列化平台容量响应失败: %w", err) + } + return string(data), nil +} + +func HandleListTenantPods(clientset kubernetes.Interface, tenantID string) (string, error) { + jobNamespace := config.GetConfig().Namespaces.Job + pods, err := clientset.CoreV1().Pods(jobNamespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", labelKeyTaskUser, tenantID), + }) + if err != nil { + return "", fmt.Errorf("列出 Pod 失败: %w", err) + } + + podResponses := make([]TenantPodResponse, 0) + for _, pod := range pods.Items { + phase := string(pod.Status.Phase) + if phase != "Running" && phase != "Pending" { + continue + } + gpuRequests := 0 + for _, c := range pod.Spec.Containers { + for k, v := range c.Resources.Requests { + if strings.Contains(string(k), "nvidia.com/") { + gpuRequests += int(v.Value()) + } + } + } + podResponses = append(podResponses, TenantPodResponse{ + PodName: pod.Name, + Phase: phase, + GPURrequests: gpuRequests, + }) + } + + data, err := json.Marshal(TenantPodsResponse{TenantID: tenantID, Pods: podResponses}) + if err != nil { + return "", fmt.Errorf("序列化租户 Pod 列表响应失败: %w", err) + } + return string(data), nil +} + +func HandleInspectPodDetails(clientset kubernetes.Interface, _ string, podName string) (string, error) { + jobNamespace := config.GetConfig().Namespaces.Job + pod, err := clientset.CoreV1().Pods(jobNamespace).Get(context.TODO(), podName, metav1.GetOptions{}) + if err != nil { + return "", fmt.Errorf("获取 Pod 失败: %w", err) + } + + var startTime time.Time + if pod.Status.StartTime != nil { + startTime = pod.Status.StartTime.Time + } + runningMinutes := 0 + if !startTime.IsZero() { + runningMinutes = int(time.Since(startTime).Minutes()) + } + + images := make([]string, 0, len(pod.Spec.Containers)) + for _, c := range pod.Spec.Containers { + images = append(images, c.Image) + } + + restartCount := 0 + for _, cs := range pod.Status.ContainerStatuses { + restartCount += int(cs.RestartCount) + } + + gpuCount := 0 + cpuMillis := int64(0) + memBytes := int64(0) + for _, c := range pod.Spec.Containers { + for k, v := range c.Resources.Requests { + if strings.Contains(string(k), "nvidia.com/") { + gpuCount += int(v.Value()) + } + } + if cpu, ok := c.Resources.Requests[corev1.ResourceCPU]; ok { + cpuMillis += cpu.MilliValue() + } + if mem, ok := c.Resources.Requests[corev1.ResourceMemory]; ok { + memBytes += mem.Value() + } + } + + gpuModel := "unknown" + if pod.Spec.NodeName != "" { + if node, err := clientset.CoreV1().Nodes().Get(context.TODO(), pod.Spec.NodeName, metav1.GetOptions{}); err == nil { + for k, v := range node.Labels { + if strings.Contains(k, "nvidia.com/gpu.product") || k == "gpu-model" { + gpuModel = v + break + } + } + } + } + + data, err := json.Marshal(PodDetailsResponse{ + PodName: podName, + StartTime: startTime, + RunningTime: runningMinutes, + ContainerImages: images, + RestartCount: restartCount, + GPUModel: gpuModel, + GPUCount: gpuCount, + CPUCount: int(cpuMillis / 1000), + MemorySize: formatBytes(memBytes), + }) + if err != nil { + return "", fmt.Errorf("序列化 Pod 详情响应失败: %w", err) + } + return string(data), nil +} + +func HandleGetComputeQuota(clientset kubernetes.Interface, tenantID string) (string, error) { + jobNamespace := config.GetConfig().Namespaces.Job + pods, err := clientset.CoreV1().Pods(jobNamespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", labelKeyTaskUser, tenantID), + }) + if err != nil { + return "", fmt.Errorf("列出 Pod 失败: %w", err) + } + + gpuRequest, gpuLimit := 0, 0 + cpuReqMillis, cpuLimMillis := int64(0), int64(0) + memReqBytes, memLimBytes := int64(0), int64(0) + + for _, pod := range pods.Items { + for _, c := range pod.Spec.Containers { + for k, v := range c.Resources.Requests { + if strings.Contains(string(k), "nvidia.com/") { + gpuRequest += int(v.Value()) + } + } + for k, v := range c.Resources.Limits { + if strings.Contains(string(k), "nvidia.com/") { + gpuLimit += int(v.Value()) + } + } + if v, ok := c.Resources.Requests[corev1.ResourceCPU]; ok { + cpuReqMillis += v.MilliValue() + } + if v, ok := c.Resources.Limits[corev1.ResourceCPU]; ok { + cpuLimMillis += v.MilliValue() + } + if v, ok := c.Resources.Requests[corev1.ResourceMemory]; ok { + memReqBytes += v.Value() + } + if v, ok := c.Resources.Limits[corev1.ResourceMemory]; ok { + memLimBytes += v.Value() + } + } + } + + data, err := json.Marshal(ComputeQuotaResponse{ + TenantID: tenantID, + GPULimit: gpuLimit, + GPURequest: gpuRequest, + CPULimit: int(cpuLimMillis / 1000), + CPURequest: int(cpuReqMillis / 1000), + MemoryLimit: formatBytes(memLimBytes), + MemoryRequest: formatBytes(memReqBytes), + }) + if err != nil { + return "", fmt.Errorf("序列化计算配额响应失败: %w", err) + } + return string(data), nil +} + +func HandleQueryTenantStorageTrend(clientset kubernetes.Interface, restConfig *rest.Config, tenantID string) (string, error) { + db := query.GetDB() + + // 先查基础用户信息和 space_quota + var userRow struct { + model.User + SpaceQuota int64 `gorm:"column:space_quota"` + } + if err := db.Model(&model.User{}). + Select("users.*, users.space_quota"). + Where("name = ?", tenantID). + First(&userRow).Error; err != nil { + return "", fmt.Errorf("用户 %s 不存在: %w", tenantID, err) + } + user := userRow.User + + // 尝试获取 original_space_quota（临时扩容时才有值），用作理论配额 + // 若迁移未执行列不存在，保持使用 space_quota + spaceQuota := userRow.SpaceQuota + var origRow struct { + OriginalSpaceQuota *int64 `gorm:"column:original_space_quota"` + } + if err := db.Raw("SELECT original_space_quota FROM users WHERE id = ?", user.ID).Scan(&origRow).Error; err == nil && origRow.OriginalSpaceQuota != nil { + spaceQuota = *origRow.OriginalSpaceQuota + } + if user.Space == "" { + return "", fmt.Errorf("用户 %s 的空间路径为空", tenantID) + } + + cfg := config.GetConfig() + prefixConfig := ceph.StoragePrefixConfig{ + User: cfg.Storage.Prefix.User, + Account: cfg.Storage.Prefix.Account, + Public: cfg.Storage.Prefix.Public, + } + currentUsage, err := ceph.GetCephDirectorySize( + clientset, restConfig, ceph.StorageQuotaRookNamespace(), "/user/"+user.Space, prefixConfig, + ) + if err != nil { + currentUsage = ceph.UnknownSizeBytes + } + + var historyRecords []model.TenantUsageHistory + db.Where("tenant_id = ?", user.ID).Order("recorded_at DESC").Limit(10).Find(&historyRecords) + + type historyItem struct { + Timestamp time.Time `json:"timestamp"` + UsageBytes int64 `json:"usage_bytes"` + UsageBytesFormatted string `json:"usage_bytes_formatted"` + } + history := make([]historyItem, 0, len(historyRecords)) + for _, h := range historyRecords { + history = append(history, historyItem{ + Timestamp: h.RecordedAt, + UsageBytes: h.UsageBytes, + UsageBytesFormatted: formatBytes(h.UsageBytes), + }) + } + + usageRatio := "" + if spaceQuota > 0 && currentUsage >= 0 { + usageRatio = fmt.Sprintf("%.1f%%", float64(currentUsage)/float64(spaceQuota)*100) + } else if spaceQuota == -1 { + usageRatio = "unlimited" + } + + data, err := json.Marshal(map[string]any{ + "tenant_id": tenantID, + "current_usage_bytes": currentUsage, + "current_usage_formatted": formatBytes(currentUsage), + "quota_bytes": spaceQuota, + "quota_formatted": formatBytes(spaceQuota), + "usage_ratio": usageRatio, + "history": history, + "note": "所有大小单位均为字节(bytes)，formatted 字段为人类可读格式。quota_bytes=-1 表示无限制。usage_ratio 为当前使用量占配额的百分比。", + }) + if err != nil { + return "", fmt.Errorf("序列化租户存储趋势响应失败: %w", err) + } + return string(data), nil +} + +// ---- 工具函数 ---- + +// extractJSON 从可能包含分析文字的响应中提取最后一个 JSON 对象 +func extractJSON(s string) string { + start := strings.LastIndex(s, "{") + end := strings.LastIndex(s, "}") + if start != -1 && end != -1 && end > start { + return strings.TrimSpace(s[start : end+1]) + } + return strings.TrimSpace(s) +} + +func parseGetfattrValue(output, attr string) int64 { + prefix := attr + "=" + for _, line := range strings.Split(output, "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, prefix) { + val := strings.Trim(strings.TrimPrefix(line, prefix), "\"") + if n, err := strconv.ParseInt(val, 10, 64); err == nil { + return n + } + } + } + return 0 +} + +func formatBytes(bytes int64) string { + const unit = 1024 + if bytes <= 0 { + return "0 B" + } + if bytes < unit { + return fmt.Sprintf("%d B", bytes) + } + div, exp := int64(unit), 0 + for n := bytes / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp]) +} diff --git a/backend/pkg/llm/provider.go b/backend/pkg/llm/provider.go new file mode 100644 index 000000000..9d0f0421e --- /dev/null +++ b/backend/pkg/llm/provider.go @@ -0,0 +1,207 @@ +//nolint:dupl // Chat and completion requests intentionally share nearly identical HTTP transport flow. +package llm + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "strings" + "time" + + "k8s.io/klog/v2" + + "github.com/raids-lab/crater/dao/model" + "github.com/raids-lab/crater/dao/query" + "github.com/raids-lab/crater/pkg/crypto" +) + +type ProviderConfig struct { + BaseURL string + APIKey string + ModelName string +} + +func (c ProviderConfig) CompletionURL() string { + baseURL := strings.TrimSuffix(strings.TrimSpace(c.BaseURL), "/") + if baseURL == "" { + return "" + } + if strings.HasSuffix(baseURL, "/completions") { + return baseURL + } + return baseURL + "/completions" +} + +func (c ProviderConfig) ChatCompletionURL() string { + baseURL := strings.TrimSuffix(strings.TrimSpace(c.BaseURL), "/") + if baseURL == "" { + return "" + } + if strings.HasSuffix(baseURL, "/chat/completions") { + return baseURL + } + return baseURL + "/chat/completions" +} + +func loadRuntimeLLMConfig(ctx context.Context) (*ProviderConfig, error) { + cfg := &ProviderConfig{ + BaseURL: strings.TrimSpace(os.Getenv("LLM_API_BASE_URL")), + APIKey: strings.TrimSpace(os.Getenv("LLM_API_KEY")), + ModelName: strings.TrimSpace(os.Getenv("LLM_MODEL_NAME")), + } + + var rows []model.SystemConfig + err := query.GetDB().WithContext(ctx). + Where("key IN ?", []string{ + model.ConfigKeyLLMBaseURL, + model.ConfigKeyLLMAPIKey, + model.ConfigKeyLLMModelName, + }). + Find(&rows).Error + if err != nil { + if cfg.BaseURL == "" || cfg.ModelName == "" { + return nil, fmt.Errorf("failed to load llm config from database: %w", err) + } + return cfg, nil + } + + configMap := make(map[string]string, len(rows)) + for _, row := range rows { + configMap[row.Key] = strings.TrimSpace(row.Value) + } + + if baseURL := configMap[model.ConfigKeyLLMBaseURL]; baseURL != "" { + cfg.BaseURL = baseURL + } + if modelName := configMap[model.ConfigKeyLLMModelName]; modelName != "" { + cfg.ModelName = modelName + } + if encryptedKey := configMap[model.ConfigKeyLLMAPIKey]; encryptedKey != "" { + plainKey, decryptErr := crypto.Decrypt(encryptedKey) + if decryptErr != nil { + klog.Warningf("loadRuntimeLLMConfig: failed to decrypt llm api key, using raw value: %v", decryptErr) + cfg.APIKey = encryptedKey + } else { + cfg.APIKey = plainKey + } + } + + if cfg.BaseURL == "" { + return nil, fmt.Errorf("llm base url is not configured") + } + if cfg.ModelName == "" { + return nil, fmt.Errorf("llm model name is not configured") + } + + return cfg, nil +} + +type completionRequest struct { + Model string `json:"model"` + Prompt string `json:"prompt"` + MaxTokens int `json:"max_tokens,omitempty"` + Temperature *float64 `json:"temperature,omitempty"` +} + +type completionResponse struct { + Choices []struct { + Text string `json:"text"` + FinishReason string `json:"finish_reason"` + } `json:"choices"` + Error *struct { + Message string `json:"message"` + } `json:"error,omitempty"` +} + +func callConfiguredLLM(ctx context.Context, cfg ProviderConfig, req dsRequest) (*dsResponse, error) { + req.Model = cfg.ModelName + + body, err := json.Marshal(req) + if err != nil { + return nil, fmt.Errorf("序列化请求失败: %w", err) + } + + chatCompletionURL := cfg.ChatCompletionURL() + if chatCompletionURL == "" { + return nil, fmt.Errorf("llm chat completion url is empty") + } + + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, chatCompletionURL, bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("创建 HTTP 请求失败: %w", err) + } + if cfg.APIKey != "" { + httpReq.Header.Set("Authorization", "Bearer "+cfg.APIKey) + } + httpReq.Header.Set("Content-Type", "application/json") + + httpClient := &http.Client{Timeout: 120 * time.Second} + httpResp, err := httpClient.Do(httpReq) + if err != nil { + return nil, fmt.Errorf("HTTP 请求失败: %w", err) + } + defer httpResp.Body.Close() + + var apiResp dsResponse + if err := json.NewDecoder(httpResp.Body).Decode(&apiResp); err != nil { + return nil, fmt.Errorf("解析 API 响应失败: %w", err) + } + + if httpResp.StatusCode != http.StatusOK { + errMsg := fmt.Sprintf("HTTP %d", httpResp.StatusCode) + if apiResp.Error != nil { + errMsg = apiResp.Error.Message + } + return nil, fmt.Errorf("API 错误: %s", errMsg) + } + + return &apiResp, nil +} + +func callConfiguredCompletion(ctx context.Context, cfg ProviderConfig, req completionRequest) (*completionResponse, error) { + req.Model = cfg.ModelName + + body, err := json.Marshal(req) + if err != nil { + return nil, fmt.Errorf("failed to marshal completion request: %w", err) + } + + completionURL := cfg.CompletionURL() + if completionURL == "" { + return nil, fmt.Errorf("llm completion url is empty") + } + + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, completionURL, bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("failed to create completion request: %w", err) + } + if cfg.APIKey != "" { + httpReq.Header.Set("Authorization", "Bearer "+cfg.APIKey) + } + httpReq.Header.Set("Content-Type", "application/json") + + httpClient := &http.Client{Timeout: 120 * time.Second} + httpResp, err := httpClient.Do(httpReq) + if err != nil { + return nil, fmt.Errorf("completion request failed: %w", err) + } + defer httpResp.Body.Close() + + var apiResp completionResponse + if err := json.NewDecoder(httpResp.Body).Decode(&apiResp); err != nil { + return nil, fmt.Errorf("failed to decode completion response: %w", err) + } + + if httpResp.StatusCode != http.StatusOK { + errMsg := fmt.Sprintf("HTTP %d", httpResp.StatusCode) + if apiResp.Error != nil { + errMsg = apiResp.Error.Message + } + return nil, fmt.Errorf("completion API error: %s", errMsg) + } + + return &apiResp, nil +} diff --git a/backend/pkg/llm/skill.go b/backend/pkg/llm/skill.go new file mode 100644 index 000000000..e0ffe6665 --- /dev/null +++ b/backend/pkg/llm/skill.go @@ -0,0 +1,46 @@ +package llm + +import ( + "embed" + "fmt" + "os" + "strconv" + "strings" +) + +const ( + StorageAgentSkillEnabledEnv = "CRATER_STORAGE_AGENT_SKILL_ENABLED" + StorageAgentSkillPathEnv = "CRATER_STORAGE_AGENT_SKILL_PATH" + defaultStorageAgentSkill = "skills/storage-governance-agent/SKILL.md" +) + +//go:embed skills/storage-governance-agent/SKILL.md +var embeddedSkills embed.FS + +func loadStorageAgentSkill() (content, source string, err error) { + enabled := true + if raw := strings.TrimSpace(os.Getenv(StorageAgentSkillEnabledEnv)); raw != "" { + parsed, err := strconv.ParseBool(raw) + if err != nil { + return "", "", fmt.Errorf("invalid %s value %q: %w", StorageAgentSkillEnabledEnv, raw, err) + } + enabled = parsed + } + if !enabled { + return "", "", nil + } + + if customPath := strings.TrimSpace(os.Getenv(StorageAgentSkillPathEnv)); customPath != "" { + data, err := os.ReadFile(customPath) + if err != nil { + return "", "", fmt.Errorf("failed to read storage agent skill from %s: %w", customPath, err) + } + return strings.TrimSpace(string(data)), customPath, nil + } + + data, err := embeddedSkills.ReadFile(defaultStorageAgentSkill) + if err != nil { + return "", "", fmt.Errorf("failed to read embedded storage agent skill: %w", err) + } + return strings.TrimSpace(string(data)), defaultStorageAgentSkill, nil +} diff --git a/backend/pkg/llm/skills/README.md b/backend/pkg/llm/skills/README.md new file mode 100644 index 000000000..b29b381ba --- /dev/null +++ b/backend/pkg/llm/skills/README.md @@ -0,0 +1,19 @@ +# LLM Skills + +This directory stores domain-specific prompt skills for the Crater backend. + +## Storage Governance Agent Skill + +- Default skill file: + [storage-governance-agent/SKILL.md](storage-governance-agent/SKILL.md) +- Loaded only by DeepSeek/OpenAI-compatible agent mode: + [llm_tools.go](../llm_tools.go) + +## Runtime Controls + +- `CRATER_STORAGE_AGENT_SKILL_ENABLED` + - Optional, default `true` + - Set to `false` to disable loading the skill +- `CRATER_STORAGE_AGENT_SKILL_PATH` + - Optional + - When set, the backend loads the skill text from this external file path instead of the embedded default skill diff --git a/backend/pkg/llm/skills/storage-governance-agent/SKILL.md b/backend/pkg/llm/skills/storage-governance-agent/SKILL.md new file mode 100644 index 000000000..8f42ec4c1 --- /dev/null +++ b/backend/pkg/llm/skills/storage-governance-agent/SKILL.md @@ -0,0 +1,22 @@ +# Storage Governance Agent Skill + +## Role +你是存储治理领域技能层，负责补充对存储扩容、冻结、Prometheus 指标和 GPU 历史行为的判断经验。 + +## Core Heuristics +1. 当 `usage_ratio >= 1.0` 时，如果无法安全扩容，应优先考虑 `freeze_new_jobs=true`。 +2. 当 `usage_ratio < 0.9` 时，通常不应扩容，也不应冻结。 +3. 当 `gpu_data_available=true` 且 `max_gpu_history_percent > 50` 时，当前低 GPU 利用率可能属于正常落盘/IO 阶段，不应轻易判定为异常。 +4. 当 `gpu_data_available=false` 时，不要过度自信地下结论，必须在 `reason` 中明确指出监控缺失或证据不足。 +5. 当平台剩余空间充足、且增长速率低于配额阈值时，优先采用保守扩容而不是冻结。 + +## Tool Preference +1. 必须先查看存储趋势与使用率。 +2. 当存在活跃 GPU Pod 时，优先分析 GPU 历史指标来区分“正常落盘”与“可疑作业”。 +3. 只有在 GPU 指标异常缺失或结果矛盾时，才进一步诊断 Prometheus。 +4. 不要做冗余工具调用；每次调用都应服务于最终扩容/冻结判断。 + +## Output Preference +1. 输出必须是纯 JSON。 +2. `reason` 必须引用关键证据字段，例如 `usage_ratio`、`growth_rate`、平台剩余容量、GPU 历史峰值或监控缺失状态。 +3. `reason` 要尽量说明当前属于哪一类场景：正常积累、落盘阶段、可疑增长、超配额冻结、平台容量受限。 diff --git a/backend/pkg/monitor/helper.go b/backend/pkg/monitor/helper.go index 4d517a732..42688719d 100644 --- a/backend/pkg/monitor/helper.go +++ b/backend/pkg/monitor/helper.go @@ -194,6 +194,42 @@ func (p *PrometheusClient) checkGPUUsed(expression string) (int, error) { return 0, fmt.Errorf("expected vector type result but got %s", result.Type()) } +// QueryInstant executes an instant PromQL query and returns the first scalar result. +func (p *PrometheusClient) QueryInstant(query string) (value float64, found bool, err error) { + ctx, cancel := context.WithTimeout(context.Background(), queryTimeout) + defer cancel() + result, _, err := p.v1api.Query(ctx, query, time.Now()) + if err != nil { + return 0, false, err + } + switch result.Type() { + case model.ValVector: + vector := result.(model.Vector) + if len(vector) > 0 { + return float64(vector[0].Value), true, nil + } + case model.ValScalar: + scalar := result.(*model.Scalar) + return float64(scalar.Value), true, nil + } + return 0, false, nil +} + +// QueryInstantLabels executes a PromQL query and returns all values of the given label key. +func (p *PrometheusClient) QueryInstantLabels(query, labelKey string) []string { + vector, err := p.queryVector(query) + if err != nil { + return nil + } + var labels []string + for _, sample := range vector { + if v, ok := sample.Metric[model.LabelName(labelKey)]; ok { + labels = append(labels, string(v)) + } + } + return labels +} + // checkIfGPURequested 检查 Pod 是否申请了 GPU func (p *PrometheusClient) checkIfGPURequested(namespacedName types.NamespacedName) bool { // 这里实现检查逻辑，例如通过查询 Pod 的资源请求或 Prometheus 中的相关指标 diff --git a/backend/pkg/monitor/interface.go b/backend/pkg/monitor/interface.go index f0534dd5f..9929d11ec 100644 --- a/backend/pkg/monitor/interface.go +++ b/backend/pkg/monitor/interface.go @@ -70,4 +70,14 @@ type PrometheusInterface interface { // GetLeastUsedGPUJobList returns the least used GPU job list GetLeastUsedGPUJobList(podName, _time, util string) int + + ///////////// Generic PromQL ////////////// + + // QueryInstant executes an instant PromQL query and returns the first scalar result. + // found=false if the result set is empty (not an error). + QueryInstant(query string) (float64, bool, error) + + // QueryInstantLabels executes a PromQL query and returns all values of the specified + // metric label key across every result sample. + QueryInstantLabels(query, labelKey string) []string } diff --git a/backend/pkg/patrol/patrol.go b/backend/pkg/patrol/patrol.go index c1ffc8092..471580511 100644 --- a/backend/pkg/patrol/patrol.go +++ b/backend/pkg/patrol/patrol.go @@ -1,14 +1,27 @@ +//nolint:gocritic,gocyclo,lll,mnd,revive // Patrol orchestration intentionally centralizes policy execution and thresholds. package patrol import ( "context" "encoding/json" + "errors" "fmt" + "os" + "strconv" + "sync" + "time" "gorm.io/datatypes" + "gorm.io/gorm" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/raids-lab/crater/dao/model" + "github.com/raids-lab/crater/dao/query" + "github.com/raids-lab/crater/pkg/ceph" + "github.com/raids-lab/crater/pkg/config" "github.com/raids-lab/crater/pkg/monitor" "github.com/raids-lab/crater/pkg/util" ) @@ -18,8 +31,19 @@ const ( TRIGGER_GPU_ANALYSIS_JOB = "trigger-gpu-analysis-job" // Billing 基础循环 TRIGGER_BILLING_BASE_LOOP_JOB = "biling-base-loop" - // 未来可以扩展其他巡检任务，例如： - // CHECK_NODE_HEALTH = "check-node-health" + // 存储告警 AI 分析任务 + ANALYZE_STORAGE_ALERTS = "analyze-storage-alerts" + AUTO_SHRINK_STORAGE_EXPANSIONS = "auto-shrink-storage-expansions" + + // AI 分析最大并发数 + defaultMaxConcurrentStorageAnalysis = 3 + autoShrinkToBufferThreshold = 0.90 + autoShrinkRecoverThreshold = 0.80 + autoShrinkObservationWindow = time.Hour + autoShrinkStageExpanded = "expanded" + autoShrinkStageBuffer = "buffer_reduction" + + storageAnalysisConcurrencyEnv = "CRATER_STORAGE_ANALYSIS_MAX_CONCURRENCY" ) type GpuAnalysisServiceInterface interface { @@ -30,18 +54,60 @@ type BillingServiceInterface interface { RunBaseLoopOnce(ctx context.Context) (any, error) } +// AgentDecision 是 LLM 存储扩容决策的结果，定义在 patrol 包以避免循环依赖。 +type AgentDecision struct { + AllowExpand bool + ExpandBytes int64 + FreezeNewJobs bool + Reason string + DecisionJobID string +} + +// StorageAgentFunc 是调用 LLM 进行存储分析的函数签名。 +type StorageAgentFunc func(tenantID string) (*AgentDecision, error) + +type StorageAgentStartFunc func(ctx context.Context, tenantID string) (string, error) + +type StorageAgentAwaitFunc func(ctx context.Context, tenantID string, jobID string) (*AgentDecision, error) + // Clients 包含巡检任务所需的客户端 type Clients struct { Client client.Client KubeClient kubernetes.Interface + KubeConfig *rest.Config PromClient monitor.PrometheusInterface GpuAnalysisService GpuAnalysisServiceInterface BillingService BillingServiceInterface + RecordDecision func(ctx context.Context, jobID string, action string, runErr error) + StorageAgent StorageAgentFunc // 注入的 LLM 分析函数，nil 时跳过 AI 分析 + StorageAgentStart StorageAgentStartFunc + StorageAgentAwait StorageAgentAwaitFunc +} + +func storageAnalysisConcurrency() int { + raw := os.Getenv(storageAnalysisConcurrencyEnv) + if raw == "" { + return defaultMaxConcurrentStorageAnalysis + } + + value, err := strconv.Atoi(raw) + if err != nil || value <= 0 { + klog.Warningf( + "storageAnalysisConcurrency: invalid %s=%q, fallback to %d", + storageAnalysisConcurrencyEnv, + raw, + defaultMaxConcurrentStorageAnalysis, + ) + return defaultMaxConcurrentStorageAnalysis + } + + return value } func NewPatrolClients( cli client.Client, kubeClient kubernetes.Interface, + kubeConfig *rest.Config, promClient monitor.PrometheusInterface, gpuAnalysisService GpuAnalysisServiceInterface, billingService BillingServiceInterface, @@ -49,13 +115,457 @@ func NewPatrolClients( return &Clients{ Client: cli, KubeClient: kubeClient, + KubeConfig: kubeConfig, PromClient: promClient, GpuAnalysisService: gpuAnalysisService, BillingService: billingService, } } +type StorageUsageRefreshResult struct { + Updated int `json:"updated"` + Failed int `json:"failed"` + RefreshedAt time.Time `json:"refreshed_at"` +} + +// RefreshUserSpaceSizes refreshes the cached CephFS usage after an explicit admin request. +func RefreshUserSpaceSizes(ctx context.Context, clients *Clients) (StorageUsageRefreshResult, error) { + refreshResult := StorageUsageRefreshResult{} + if !ceph.StorageQuotaEnabled() { + return refreshResult, fmt.Errorf("storage quota usage refresh is disabled") + } + + var users []model.User + db := query.GetDB().WithContext(ctx) + if err := db.Find(&users).Error; err != nil { + return refreshResult, fmt.Errorf("list users for storage usage refresh: %w", err) + } + + cfg := config.GetConfig() + prefixConfig := ceph.StoragePrefixConfig{ + User: cfg.Storage.Prefix.User, + Account: cfg.Storage.Prefix.Account, + Public: cfg.Storage.Prefix.Public, + } + + for _, user := range users { + if err := ctx.Err(); err != nil { + return refreshResult, fmt.Errorf("storage usage refresh canceled: %w", err) + } + if user.Space == "" { + klog.Warningf("RefreshUserSpaceSizes: user %q has no storage space path", user.Name) + refreshResult.Failed++ + continue + } + + size, err := ceph.GetCephDirectorySize( + clients.KubeClient, clients.KubeConfig, ceph.StorageQuotaRookNamespace(), "/user/"+user.Space, prefixConfig, + ) + if err != nil { + klog.Errorf("RefreshUserSpaceSizes: read usage for user %q: %v", user.Name, err) + refreshResult.Failed++ + continue + } + + var userSpaceSize model.UserSpaceSize + result := db.Where("user_id = ?", user.ID).First(&userSpaceSize) + switch { + case errors.Is(result.Error, gorm.ErrRecordNotFound): + userSpaceSize = model.UserSpaceSize{ + UserID: user.ID, + Username: user.Name, + Size: size, + } + if err := db.Create(&userSpaceSize).Error; err != nil { + klog.Errorf("RefreshUserSpaceSizes: create usage cache for user %q: %v", user.Name, err) + refreshResult.Failed++ + continue + } + case result.Error != nil: + klog.Errorf("RefreshUserSpaceSizes: query usage cache for user %q: %v", user.Name, result.Error) + refreshResult.Failed++ + continue + default: + userSpaceSize.Username = user.Name + userSpaceSize.Size = size + if err := db.Save(&userSpaceSize).Error; err != nil { + klog.Errorf("RefreshUserSpaceSizes: update usage cache for user %q: %v", user.Name, err) + refreshResult.Failed++ + continue + } + } + + refreshResult.Updated++ + } + + refreshResult.RefreshedAt = time.Now() + return refreshResult, nil +} + +// RunAnalyzeStorageAlerts 对超过90%理论配额且未临时扩容的用户并发执行 AI 分析， +// 并自动应用决策（冻结作业 / 临时扩容）。 +// +//nolint:funlen // Storage alert analysis keeps candidate selection, LLM decision, and enforcement in one cron action. +func RunAnalyzeStorageAlerts(ctx context.Context, clients *Clients) (any, error) { + db := query.GetDB() + maxConcurrentStorageAnalysis := storageAnalysisConcurrency() + + // 查询有空间大小记录的用户，附带配额信息 + type userWithUsage struct { + ID uint `gorm:"column:id"` + Name string `gorm:"column:name"` + Space string `gorm:"column:space"` + SpaceQuota int64 `gorm:"column:space_quota"` + OriginalSpaceQuota *int64 `gorm:"column:original_space_quota"` + CurrentSize int64 `gorm:"column:current_size"` + } + + var candidates []userWithUsage + if err := db.Raw(` + SELECT u.id, u.name, u.space, u.space_quota, u.original_space_quota, uss.size AS current_size + FROM users u + JOIN user_space_sizes uss ON uss.user_id = u.id + WHERE u.deleted_at IS NULL + AND u.original_space_quota IS NULL + AND u.space_quota > 0 + `).Scan(&candidates).Error; err != nil { + return nil, fmt.Errorf("查询用户列表失败: %w", err) + } + + // 过滤出超过 90% 的用户 + var alertUsers []userWithUsage + for _, u := range candidates { + if float64(u.CurrentSize)/float64(u.SpaceQuota) >= 0.9 { + alertUsers = append(alertUsers, u) + } + } + + klog.Infof("RunAnalyzeStorageAlerts: %d 个用户超过90%%配额，启动并发 AI 分析（最大并发 %d）", + len(alertUsers), maxConcurrentStorageAnalysis) + + if len(alertUsers) == 0 { + return "无超额用户，无需分析", nil + } + + if clients.StorageAgent == nil { + if clients.StorageAgentStart == nil || clients.StorageAgentAwait == nil { + klog.Warningf("RunAnalyzeStorageAlerts: StorageAgent 未注入，跳过 AI 分析") + return "StorageAgent 未配置", nil + } + } + + cfg := config.GetConfig() + prefixConfig := ceph.StoragePrefixConfig{ + User: cfg.Storage.Prefix.User, + Account: cfg.Storage.Prefix.Account, + Public: cfg.Storage.Prefix.Public, + } + + applyDecision := func(u userWithUsage, decision *AgentDecision) { + klog.Infof("RunAnalyzeStorageAlerts: 用户 %s 决策: allow_expand=%v expand_bytes=%d freeze=%v reason=%s", + u.Name, decision.AllowExpand, decision.ExpandBytes, decision.FreezeNewJobs, decision.Reason) + + recordDecision := func(action string, runErr error) { + if clients.RecordDecision != nil && decision.DecisionJobID != "" { + clients.RecordDecision(ctx, decision.DecisionJobID, action, runErr) + } + } + if decision.AllowExpand && decision.ExpandBytes > 0 { + newQuota := u.SpaceQuota + decision.ExpandBytes + if err := db.Exec( + "UPDATE users SET original_space_quota = space_quota, space_quota = ?, jobs_frozen = ? WHERE id = ? AND deleted_at IS NULL", + newQuota, decision.FreezeNewJobs, u.ID, + ).Error; err != nil { + klog.Errorf("RunAnalyzeStorageAlerts: 用户 %s 写入扩容失败: %v", u.Name, err) + recordDecision("expand_failed", err) + return + } + if u.Space != "" { + if cephErr := ceph.SetCephDirectoryQuota( + clients.KubeClient, clients.KubeConfig, ceph.StorageQuotaRookNamespace(), + "/user/"+u.Space, prefixConfig, newQuota, + ); cephErr != nil { + klog.Errorf("RunAnalyzeStorageAlerts: 用户 %s Ceph 配额同步失败: %v", u.Name, cephErr) + } + } + klog.Infof("RunAnalyzeStorageAlerts: 用户 %s 已临时扩容至 %d bytes", u.Name, newQuota) + recordDecision("expand", nil) + return + } + + if decision.FreezeNewJobs { + if err := db.Exec( + "UPDATE users SET jobs_frozen = true WHERE id = ? AND deleted_at IS NULL", u.ID, + ).Error; err != nil { + klog.Errorf("RunAnalyzeStorageAlerts: 用户 %s 设置 jobs_frozen 失败: %v", u.Name, err) + recordDecision("freeze_failed", err) + return + } + + recordDecision("freeze", nil) + klog.Infof("RunAnalyzeStorageAlerts: 用户 %s 已冻结新作业创建", u.Name) + return + } + + recordDecision("observe", nil) + } + + if clients.StorageAgentStart != nil && clients.StorageAgentAwait != nil { + type pendingDecision struct { + user userWithUsage + jobID string + } + + sem := make(chan struct{}, maxConcurrentStorageAnalysis) + var wg sync.WaitGroup + var mu sync.Mutex + pending := make([]pendingDecision, 0, len(alertUsers)) + + for _, candidate := range alertUsers { + u := candidate + wg.Add(1) + sem <- struct{}{} + go func() { + defer wg.Done() + defer func() { <-sem }() + + klog.Infof("RunAnalyzeStorageAlerts: 开始派发用户 %s 的异步 AI 分析任务", u.Name) + jobID, err := clients.StorageAgentStart(ctx, u.Name) + if err != nil { + klog.Errorf("RunAnalyzeStorageAlerts: 用户 %s AI 分析任务派发失败: %v", u.Name, err) + return + } + + klog.Infof("RunAnalyzeStorageAlerts: 用户 %s AI 分析任务已派发 job_id=%s", u.Name, jobID) + mu.Lock() + pending = append(pending, pendingDecision{user: u, jobID: jobID}) + mu.Unlock() + }() + } + wg.Wait() + + if len(pending) == 0 { + return "没有成功派发任何 AI 分析任务", nil + } + + klog.Infof( + "RunAnalyzeStorageAlerts: %d 个用户的 AI 分析任务已派发完成，开始并发等待结果（最大并发 %d）", + len(pending), + maxConcurrentStorageAnalysis, + ) + + wg = sync.WaitGroup{} + for _, item := range pending { + pendingItem := item + wg.Add(1) + sem <- struct{}{} + go func() { + defer wg.Done() + defer func() { <-sem }() + + decision, err := clients.StorageAgentAwait(ctx, pendingItem.user.Name, pendingItem.jobID) + if err != nil { + klog.Errorf("RunAnalyzeStorageAlerts: 用户 %s 等待 AI 分析结果失败: %v", pendingItem.user.Name, err) + return + } + + applyDecision(pendingItem.user, decision) + }() + } + wg.Wait() + return fmt.Sprintf("分析完成，共处理 %d 个超额用户", len(alertUsers)), nil + } + + // 使用 channel 实现有界并发 + sem := make(chan struct{}, maxConcurrentStorageAnalysis) + var wg sync.WaitGroup + for _, candidate := range alertUsers { + u := candidate + wg.Add(1) + sem <- struct{}{} // 占槽（满时阻塞） + go func() { + defer wg.Done() + defer func() { <-sem }() // 释放槽 + + klog.Infof("RunAnalyzeStorageAlerts: 开始分析用户 %s (size=%d quota=%d %.1f%%)", + u.Name, u.CurrentSize, u.SpaceQuota, float64(u.CurrentSize)/float64(u.SpaceQuota)*100) + + decision, err := clients.StorageAgent(u.Name) + if err != nil { + klog.Errorf("RunAnalyzeStorageAlerts: 用户 %s AI 分析失败: %v", u.Name, err) + return + } + + applyDecision(u, decision) + }() + } + + wg.Wait() + return fmt.Sprintf("分析完成，共处理 %d 个超额用户", len(alertUsers)), nil +} + // GetPatrolFunc 根据作业名称返回对应的巡检函数 +// RunAutoShrinkStorageExpansions automatically recovers temporary storage expansions +// once a user's current usage has fallen below a conservative percentage of the +// original theoretical quota. +func RunAutoShrinkStorageExpansions(ctx context.Context, clients *Clients) (any, error) { + db := query.GetDB() + + type expandedUser struct { + ID uint `gorm:"column:id"` + Name string `gorm:"column:name"` + Space string `gorm:"column:space"` + SpaceQuota int64 `gorm:"column:space_quota"` + OriginalSpaceQuota int64 `gorm:"column:original_space_quota"` + CurrentSize int64 `gorm:"column:current_size"` + ShrinkStage string `gorm:"column:shrink_stage"` + ShrinkStageUpdatedAt *time.Time `gorm:"column:shrink_stage_updated_at"` + } + + var users []expandedUser + if err := db.Raw(` + SELECT u.id, u.name, u.space, u.space_quota, u.original_space_quota, uss.size AS current_size, + u.shrink_stage, u.shrink_stage_updated_at + FROM users u + JOIN user_space_sizes uss ON uss.user_id = u.id + WHERE u.deleted_at IS NULL + AND u.original_space_quota IS NOT NULL + `).Scan(&users).Error; err != nil { + return nil, fmt.Errorf("query expanded users failed: %w", err) + } + + if len(users) == 0 { + return "当前没有处于临时扩容状态的用户，无需执行自动缩容。", nil + } + + cfg := config.GetConfig() + prefixConfig := ceph.StoragePrefixConfig{ + User: cfg.Storage.Prefix.User, + Account: cfg.Storage.Prefix.Account, + Public: cfg.Storage.Prefix.Public, + } + + shrunk := 0 + skipped := 0 + for _, user := range users { + if user.OriginalSpaceQuota <= 0 { + skipped++ + continue + } + + usageRatio := float64(user.CurrentSize) / float64(user.OriginalSpaceQuota) + stage := user.ShrinkStage + if stage == "" { + stage = autoShrinkStageExpanded + } + + switch stage { + case autoShrinkStageExpanded: + if usageRatio >= autoShrinkToBufferThreshold { + skipped++ + continue + } + + bufferQuota := calculateShrinkBufferQuota(user.OriginalSpaceQuota, user.SpaceQuota) + if bufferQuota <= user.OriginalSpaceQuota { + bufferQuota = user.OriginalSpaceQuota + } + + if err := db.Exec( + "UPDATE users SET space_quota = ?, shrink_stage = ?, shrink_stage_updated_at = NOW() WHERE id = ? AND deleted_at IS NULL", + bufferQuota, autoShrinkStageBuffer, user.ID, + ).Error; err != nil { + klog.Errorf("RunAutoShrinkStorageExpansions: user=%s buffer shrink failed: %v", user.Name, err) + skipped++ + continue + } + + if user.Space != "" { + if cephErr := ceph.SetCephDirectoryQuota( + clients.KubeClient, + clients.KubeConfig, + ceph.StorageQuotaRookNamespace(), + "/user/"+user.Space, + prefixConfig, + bufferQuota, + ); cephErr != nil { + klog.Errorf("RunAutoShrinkStorageExpansions: user=%s ceph buffer shrink failed: %v", user.Name, cephErr) + skipped++ + continue + } + } + + shrunk++ + klog.Infof( + "RunAutoShrinkStorageExpansions: user=%s moved to buffer stage quota=%d current_size=%d ratio=%.2f", + user.Name, + bufferQuota, + user.CurrentSize, + usageRatio, + ) + case autoShrinkStageBuffer: + if user.ShrinkStageUpdatedAt == nil || time.Since(*user.ShrinkStageUpdatedAt) < autoShrinkObservationWindow { + skipped++ + continue + } + if usageRatio >= autoShrinkRecoverThreshold { + skipped++ + continue + } + + if err := db.Exec( + "UPDATE users SET space_quota = ?, original_space_quota = NULL, jobs_frozen = false, shrink_stage = NULL, shrink_stage_updated_at = NULL WHERE id = ? AND deleted_at IS NULL", + user.OriginalSpaceQuota, user.ID, + ).Error; err != nil { + klog.Errorf("RunAutoShrinkStorageExpansions: user=%s final shrink failed: %v", user.Name, err) + skipped++ + continue + } + + if user.Space != "" { + if cephErr := ceph.SetCephDirectoryQuota( + clients.KubeClient, + clients.KubeConfig, + ceph.StorageQuotaRookNamespace(), + "/user/"+user.Space, + prefixConfig, + user.OriginalSpaceQuota, + ); cephErr != nil { + klog.Errorf("RunAutoShrinkStorageExpansions: user=%s ceph final shrink failed: %v", user.Name, cephErr) + skipped++ + continue + } + } + + shrunk++ + klog.Infof( + "RunAutoShrinkStorageExpansions: user=%s fully restored quota=%d current_size=%d ratio=%.2f", + user.Name, + user.OriginalSpaceQuota, + user.CurrentSize, + usageRatio, + ) + default: + skipped++ + } + } + + return fmt.Sprintf("自动缩容扫描完成：已处理 %d 个用户，跳过 %d 个用户。", shrunk, skipped), nil +} + +func calculateShrinkBufferQuota(originalQuota, currentQuota int64) int64 { + if currentQuota <= originalQuota { + return originalQuota + } + + delta := currentQuota - originalQuota + bufferQuota := originalQuota + delta/2 + if bufferQuota <= originalQuota { + return originalQuota + } + return bufferQuota +} + func GetPatrolFunc(jobName string, clients *Clients, jobConfig datatypes.JSON) (util.AnyFunc, error) { var f util.AnyFunc switch jobName { @@ -74,7 +584,6 @@ func GetPatrolFunc(jobName string, clients *Clients, jobConfig datatypes.JSON) ( f = func(ctx context.Context) (any, error) { return RunTriggerBillingBaseLoop(ctx, clients) } - default: return nil, fmt.Errorf("unsupported patrol job name: %s", jobName) } diff --git a/backend/pkg/storagegovernance/policy.go b/backend/pkg/storagegovernance/policy.go new file mode 100644 index 000000000..38c146ecf --- /dev/null +++ b/backend/pkg/storagegovernance/policy.go @@ -0,0 +1,282 @@ +//nolint:gocritic,gocyclo // Policy evaluation intentionally operates on full snapshot values in a single rules engine. +package storagegovernance + +import ( + "fmt" + "math" + "strings" + "time" + + "github.com/raids-lab/crater/pkg/llm" +) + +const ( + decisionEvidenceCapacity = 5 + usageRatioNearLimit = 0.95 + usageRatioAlertLimit = 0.90 + highGPUHistoryPercent float64 = 80 + lowGPUHistoryPercent float64 = 10 +) + +func ApplySafetyConstraints( + snapshot DecisionSnapshot, + decision llm.LLMDecisionResponse, + cfg ConstraintConfig, + now time.Time, +) (llm.LLMDecisionResponse, ConstraintEvaluation) { + originalDecision := decision + finalDecision := decision + evaluation := ConstraintEvaluation{ + PolicyVersion: cfg.PolicyVersion, + Violations: []string{}, + Adjustments: []string{}, + } + + if snapshot.UsageRatio < cfg.AlertThreshold { + if finalDecision.AllowExpand || finalDecision.FreezeNewJobs { + evaluation.Violations = append(evaluation.Violations, "存储使用率低于告警阈值") + finalDecision.AllowExpand = false + finalDecision.ExpandBytes = 0 + finalDecision.FreezeNewJobs = false + evaluation.Adjustments = append(evaluation.Adjustments, "已禁用扩容，因为当前存储使用率低于告警阈值") + } + } + + if snapshot.TheoreticalQuotaBytes <= 0 && finalDecision.AllowExpand { + evaluation.Violations = append(evaluation.Violations, "理论配额为无限制或未设置") + finalDecision.AllowExpand = false + finalDecision.ExpandBytes = 0 + evaluation.Adjustments = append(evaluation.Adjustments, "已禁用扩容，因为用户不存在有限的理论配额") + } + + if snapshot.IsCurrentlyExpanded && finalDecision.AllowExpand { + evaluation.Violations = append(evaluation.Violations, "用户当前已处于临时扩容状态") + finalDecision.AllowExpand = false + finalDecision.ExpandBytes = 0 + evaluation.Adjustments = append(evaluation.Adjustments, "已禁用扩容，因为上一轮临时扩容仍在生效") + } + + if snapshot.LastExpandAt != nil && now.Sub(*snapshot.LastExpandAt) < cfg.ExpansionCooldown && finalDecision.AllowExpand { + evaluation.Violations = append(evaluation.Violations, "最近一次扩容仍处于冷却时间窗口内") + finalDecision.AllowExpand = false + finalDecision.ExpandBytes = 0 + evaluation.Adjustments = append(evaluation.Adjustments, "已禁用扩容，因为距上次扩容执行尚未超过冷却时间") + } + + if snapshot.UsageRatio >= 1.0 && finalDecision.AllowExpand { + evaluation.Violations = append(evaluation.Violations, "用户当前已超过理论配额") + finalDecision.AllowExpand = false + finalDecision.ExpandBytes = 0 + evaluation.Adjustments = append(evaluation.Adjustments, "已禁用扩容，因为当前使用量已经超过理论配额") + } + + if finalDecision.AllowExpand && finalDecision.ExpandBytes <= 0 { + evaluation.Violations = append(evaluation.Violations, "启用扩容时，扩容量必须为正数") + finalDecision.AllowExpand = false + finalDecision.ExpandBytes = 0 + evaluation.Adjustments = append(evaluation.Adjustments, "已禁用扩容，因为建议扩容量不是正数") + } + + if finalDecision.AllowExpand { + maxAllowed := int64(math.MaxInt64) + + if snapshot.TheoreticalQuotaBytes > 0 && cfg.MaxExpandRatio > 0 { + maxByRatio := int64(float64(snapshot.TheoreticalQuotaBytes) * cfg.MaxExpandRatio) + if maxByRatio > 0 && maxByRatio < maxAllowed { + maxAllowed = maxByRatio + } + } + + if cfg.MaxExpandBytes > 0 && cfg.MaxExpandBytes < maxAllowed { + maxAllowed = cfg.MaxExpandBytes + } + + if snapshot.PlatformTotalBytes > 0 { + reservedByRatio := int64(float64(snapshot.PlatformTotalBytes) * cfg.MinPlatformReservedRatio) + reservedBytes := maxInt64(cfg.MinPlatformReservedBytes, reservedByRatio) + maxByPlatform := snapshot.PlatformAvailableBytes - reservedBytes + if maxByPlatform < maxAllowed { + maxAllowed = maxByPlatform + } + } + + if maxAllowed <= 0 { + evaluation.Violations = append(evaluation.Violations, "执行扩容将突破平台预留容量下限") + finalDecision.AllowExpand = false + finalDecision.ExpandBytes = 0 + evaluation.Adjustments = append(evaluation.Adjustments, "已禁用扩容，因为平台预留容量会低于安全阈值") + } else if finalDecision.ExpandBytes > maxAllowed { + evaluation.Violations = append(evaluation.Violations, "建议扩容量超过安全上限") + evaluation.Adjustments = append( + evaluation.Adjustments, + fmt.Sprintf("已将扩容量从 %d 字节收敛到 %d 字节", finalDecision.ExpandBytes, maxAllowed), + ) + finalDecision.ExpandBytes = maxAllowed + } + } + + if cfg.ForceFreezeWhenOverQuota && snapshot.UsageRatio >= 1.0 && !finalDecision.AllowExpand && !finalDecision.FreezeNewJobs { + evaluation.Violations = append(evaluation.Violations, "用户当前已超过理论配额") + finalDecision.FreezeNewJobs = true + evaluation.Adjustments = append(evaluation.Adjustments, "已强制冻结新作业，因为当前使用量已经超过理论配额") + } + + if !finalDecision.AllowExpand { + finalDecision.ExpandBytes = 0 + } + + evaluation.Violations = uniqueStrings(evaluation.Violations) + evaluation.Adjustments = uniqueStrings(evaluation.Adjustments) + evaluation.Adjusted = len(evaluation.Adjustments) > 0 + evaluation.Blocked = originalDecision.AllowExpand && !finalDecision.AllowExpand + finalDecision.Reason = rewriteDecisionReason(snapshot, originalDecision, finalDecision, evaluation) + + return finalDecision, evaluation +} + +func rewriteDecisionReason( + snapshot DecisionSnapshot, + originalDecision llm.LLMDecisionResponse, + finalDecision llm.LLMDecisionResponse, + evaluation ConstraintEvaluation, +) string { + evidenceParts := make([]string, 0, decisionEvidenceCapacity) + + switch { + case snapshot.UsageRatio >= 1.0: + evidenceParts = append(evidenceParts, "当前使用量已经超过理论配额") + case snapshot.UsageRatio >= usageRatioNearLimit: + evidenceParts = append(evidenceParts, "存储使用率已逼近阈值") + case snapshot.UsageRatio >= usageRatioAlertLimit: + evidenceParts = append(evidenceParts, "存储使用率接近阈值") + default: + evidenceParts = append(evidenceParts, "当前使用量已回落到安全区间") + } + + if growthPhrase := describeGrowth(snapshot.GrowthRateBytesPerHour); growthPhrase != "" { + evidenceParts = append(evidenceParts, growthPhrase) + } + + if snapshot.GPUDataAvailable { + switch { + case snapshot.MaxGPUHistoryPercent >= highGPUHistoryPercent: + evidenceParts = append(evidenceParts, "GPU 历史峰值较高") + case snapshot.MaxGPUHistoryPercent <= lowGPUHistoryPercent: + evidenceParts = append(evidenceParts, "GPU 历史峰值不足以支撑高价值训练判断") + } + } + + if snapshot.IsCurrentlyExpanded { + evidenceParts = append(evidenceParts, "用户当前已处于临时扩容状态") + } + + switch snapshot.ShrinkStage { + case "expanded": + evidenceParts = append(evidenceParts, "当前处于临时扩容后的观察阶段") + case "buffer_reduction": + evidenceParts = append(evidenceParts, "当前处于缩容缓冲阶段") + } + + baseReason := strings.Join(uniqueStrings(evidenceParts), "，") + actionClause := describeFinalAction(snapshot, originalDecision, finalDecision) + if baseReason == "" { + baseReason = actionClause + } else { + baseReason = baseReason + "，" + actionClause + } + + if evaluation.Adjusted { + return appendConstraintReason(baseReason, evaluation) + } + return baseReason +} + +func describeGrowth(growthRate *float64) string { + if growthRate == nil { + return "" + } + + const gib = 1024 * 1024 * 1024 + switch { + case *growthRate >= 4*gib: + return "增长速度较快" + case *growthRate > 0: + return "增长速率平稳" + case *growthRate <= -0.5*gib: + return "使用量持续下降" + default: + return "使用量整体稳定" + } +} + +func describeFinalAction( + snapshot DecisionSnapshot, + originalDecision llm.LLMDecisionResponse, + finalDecision llm.LLMDecisionResponse, +) string { + if finalDecision.AllowExpand { + if originalDecision.AllowExpand && originalDecision.ExpandBytes != finalDecision.ExpandBytes { + return fmt.Sprintf("最终建议在安全约束下扩容 %d 字节，以保护当前作业写入", finalDecision.ExpandBytes) + } + return fmt.Sprintf("建议扩容 %d 字节，以保护当前作业写入", finalDecision.ExpandBytes) + } + + if finalDecision.FreezeNewJobs { + return "不应继续扩容，并需要冻结新作业" + } + + switch { + case snapshot.ShrinkStage == "buffer_reduction": + return "无需继续扩容，可以进一步恢复到原始配额" + case snapshot.IsCurrentlyExpanded && snapshot.UsageRatio < 0.9: + return "无需继续扩容，更适合进入分阶段缩容观察" + default: + return "不应继续扩容，建议继续观察" + } +} + +func appendConstraintReason(reason string, evaluation ConstraintEvaluation) string { + suffix := "安全约束：" + if len(evaluation.Adjustments) > 0 { + suffix += " " + joinWithSemicolon(evaluation.Adjustments) + } + if len(evaluation.Violations) > 0 { + suffix += " | 违规项：" + joinWithSemicolon(evaluation.Violations) + } + if reason == "" { + return suffix + } + return reason + " | " + suffix +} + +func joinWithSemicolon(items []string) string { + return strings.Join(uniqueStrings(items), "; ") +} + +func uniqueStrings(items []string) []string { + if len(items) == 0 { + return items + } + + seen := make(map[string]struct{}, len(items)) + out := make([]string, 0, len(items)) + for _, item := range items { + trimmed := strings.TrimSpace(item) + if trimmed == "" { + continue + } + if _, ok := seen[trimmed]; ok { + continue + } + seen[trimmed] = struct{}{} + out = append(out, trimmed) + } + return out +} + +func maxInt64(a, b int64) int64 { + if a > b { + return a + } + return b +} diff --git a/backend/pkg/storagegovernance/policy_test.go b/backend/pkg/storagegovernance/policy_test.go new file mode 100644 index 000000000..0a2d6df2e --- /dev/null +++ b/backend/pkg/storagegovernance/policy_test.go @@ -0,0 +1,112 @@ +package storagegovernance + +import ( + "strings" + "testing" + "time" + + "github.com/raids-lab/crater/pkg/llm" +) + +func TestApplySafetyConstraintsClampsExpansion(t *testing.T) { + cfg := DefaultConstraintConfig() + cfg.MinPlatformReservedBytes = 10 + cfg.MinPlatformReservedRatio = 0 + snapshot := DecisionSnapshot{ + Username: "alice", + CurrentUsageBytes: 95, + TheoreticalQuotaBytes: 100, + UsageRatio: 0.95, + PlatformTotalBytes: 1000, + PlatformAvailableBytes: 400, + } + decision := llm.LLMDecisionResponse{ + AllowExpand: true, + ExpandBytes: 80, + Reason: "raw decision", + } + + finalDecision, evaluation := ApplySafetyConstraints(snapshot, decision, cfg, time.Now()) + + if !finalDecision.AllowExpand { + t.Fatalf("expected expansion to remain enabled after clamping") + } + if finalDecision.ExpandBytes != 30 { + t.Fatalf("expected expansion to be clamped to 30, got %d", finalDecision.ExpandBytes) + } + if !evaluation.Adjusted { + t.Fatalf("expected evaluation to mark the decision as adjusted") + } +} + +func TestApplySafetyConstraintsForcesFreezeWhenOverQuota(t *testing.T) { + cfg := DefaultConstraintConfig() + snapshot := DecisionSnapshot{ + Username: "alice", + CurrentUsageBytes: 120, + TheoreticalQuotaBytes: 100, + UsageRatio: 1.20, + PlatformTotalBytes: 1000, + PlatformAvailableBytes: 500, + } + decision := llm.LLMDecisionResponse{ + AllowExpand: false, + ExpandBytes: 0, + FreezeNewJobs: false, + Reason: "raw decision", + } + + finalDecision, evaluation := ApplySafetyConstraints(snapshot, decision, cfg, time.Now()) + + if !finalDecision.FreezeNewJobs { + t.Fatalf("expected freeze_new_jobs to be forced on when user is over quota") + } + if finalDecision.AllowExpand { + t.Fatalf("expected allow_expand to remain disabled when user is over quota") + } + if !evaluation.Adjusted { + t.Fatalf("expected evaluation to mark the decision as adjusted") + } + if strings.Contains(finalDecision.Reason, "建议扩容") || strings.Contains(finalDecision.Reason, "优先扩容") { + t.Fatalf("expected rewritten final reason to stop supporting expansion, got %q", finalDecision.Reason) + } + if !strings.Contains(finalDecision.Reason, "冻结新作业") { + t.Fatalf("expected rewritten final reason to mention freezing new jobs, got %q", finalDecision.Reason) + } +} + +func TestApplySafetyConstraintsBlocksExpansionWhenAlreadyOverQuota(t *testing.T) { + cfg := DefaultConstraintConfig() + snapshot := DecisionSnapshot{ + Username: "alice", + CurrentUsageBytes: 120, + TheoreticalQuotaBytes: 100, + UsageRatio: 1.20, + PlatformTotalBytes: 1000, + PlatformAvailableBytes: 500, + } + decision := llm.LLMDecisionResponse{ + AllowExpand: true, + ExpandBytes: 20, + FreezeNewJobs: false, + Reason: "建议扩容保护作业", + } + + finalDecision, evaluation := ApplySafetyConstraints(snapshot, decision, cfg, time.Now()) + + if finalDecision.AllowExpand { + t.Fatalf("expected expansion to be disabled when user is already over quota") + } + if finalDecision.ExpandBytes != 0 { + t.Fatalf("expected expand_bytes to be reset to 0, got %d", finalDecision.ExpandBytes) + } + if !finalDecision.FreezeNewJobs { + t.Fatalf("expected freeze_new_jobs to be forced on when user is already over quota") + } + if !evaluation.Adjusted { + t.Fatalf("expected evaluation to mark the decision as adjusted") + } + if strings.Contains(finalDecision.Reason, "建议扩容") { + t.Fatalf("expected final reason to be rewritten without expansion wording, got %q", finalDecision.Reason) + } +} diff --git a/backend/pkg/storagegovernance/query.go b/backend/pkg/storagegovernance/query.go new file mode 100644 index 000000000..32af3ebc6 --- /dev/null +++ b/backend/pkg/storagegovernance/query.go @@ -0,0 +1,175 @@ +//nolint:gocritic // Query projection helpers favor value semantics for readability on bounded records. +package storagegovernance + +import ( + "context" + "encoding/json" + + "github.com/raids-lab/crater/dao/model" + "github.com/raids-lab/crater/dao/query" + "github.com/raids-lab/crater/pkg/llm" +) + +type DecisionRecordSummary struct { + JobID string `json:"job_id"` + Username string `json:"username"` + Source model.StorageDecisionSource `json:"source"` + Status model.StorageDecisionStatus `json:"status"` + TriggerReason string `json:"trigger_reason"` + RawAllowExpand bool `json:"raw_allow_expand"` + RawExpandBytes int64 `json:"raw_expand_bytes"` + RawFreezeNewJobs bool `json:"raw_freeze_new_jobs"` + FinalAllowExpand bool `json:"final_allow_expand"` + FinalExpandBytes int64 `json:"final_expand_bytes"` + FinalFreezeNewJobs bool `json:"final_freeze_new_jobs"` + ConstraintAdjusted bool `json:"constraint_adjusted"` + ConstraintBlocked bool `json:"constraint_blocked"` + AppliedAction string `json:"applied_action"` + ErrorMessage string `json:"error_message"` + ConstraintVersion string `json:"constraint_version"` + LatencyMs int64 `json:"latency_ms"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + +type DecisionRecordDetail struct { + DecisionRecordSummary + CurrentShrinkStage string `json:"current_shrink_stage,omitempty"` + Snapshot *DecisionSnapshot `json:"snapshot,omitempty"` + RawDecision *llm.LLMDecisionResponse `json:"raw_decision,omitempty"` + FinalDecision *llm.LLMDecisionResponse `json:"final_decision,omitempty"` + ConstraintResult *ConstraintEvaluation `json:"constraint_result,omitempty"` +} + +type DecisionRecordPage struct { + Items []DecisionRecordSummary `json:"items"` + Total int64 `json:"total"` + Page int `json:"page"` + PageSize int `json:"page_size"` + TotalPages int `json:"total_pages"` +} + +func ListDecisionRecords( + ctx context.Context, + page int, + pageSize int, + username string, + status string, + source string, +) (*DecisionRecordPage, error) { + if page <= 0 { + page = 1 + } + if pageSize <= 0 { + pageSize = 20 + } + + tx := query.GetDB().WithContext(ctx).Model(&model.StorageDecisionRecord{}) + if username != "" { + tx = tx.Where("username = ?", username) + } + if status != "" { + tx = tx.Where("status = ?", status) + } + if source != "" { + tx = tx.Where("source = ?", source) + } + + var total int64 + if err := tx.Count(&total).Error; err != nil { + return nil, err + } + + var records []model.StorageDecisionRecord + if err := tx.Order("created_at desc"). + Offset((page - 1) * pageSize). + Limit(pageSize). + Find(&records).Error; err != nil { + return nil, err + } + + items := make([]DecisionRecordSummary, 0, len(records)) + for _, record := range records { + items = append(items, summarizeRecord(record)) + } + + return &DecisionRecordPage{ + Items: items, + Total: total, + Page: page, + PageSize: pageSize, + TotalPages: int((total + int64(pageSize) - 1) / int64(pageSize)), + }, nil +} + +func GetDecisionRecord(ctx context.Context, jobID string) (*DecisionRecordDetail, error) { + var record model.StorageDecisionRecord + if err := query.GetDB().WithContext(ctx).Where("job_id = ?", jobID).First(&record).Error; err != nil { + return nil, err + } + + detail := &DecisionRecordDetail{ + DecisionRecordSummary: summarizeRecord(record), + } + if len(record.Snapshot) > 0 { + var snapshot DecisionSnapshot + if err := json.Unmarshal(record.Snapshot, &snapshot); err == nil { + detail.Snapshot = &snapshot + } + } + if len(record.RawDecision) > 0 { + var raw llm.LLMDecisionResponse + if err := json.Unmarshal(record.RawDecision, &raw); err == nil { + detail.RawDecision = &raw + } + } + if len(record.FinalDecision) > 0 { + var final llm.LLMDecisionResponse + if err := json.Unmarshal(record.FinalDecision, &final); err == nil { + detail.FinalDecision = &final + } + } + if len(record.ConstraintResult) > 0 { + var evaluation ConstraintEvaluation + if err := json.Unmarshal(record.ConstraintResult, &evaluation); err == nil { + detail.ConstraintResult = &evaluation + } + } + + var userState struct { + ShrinkStage string `gorm:"column:shrink_stage"` + } + if err := query.GetDB().WithContext(ctx). + Raw("SELECT shrink_stage FROM users WHERE name = ? AND deleted_at IS NULL", record.Username). + Scan(&userState).Error; err == nil { + detail.CurrentShrinkStage = userState.ShrinkStage + } + + return detail, nil +} + +func summarizeRecord(record model.StorageDecisionRecord) DecisionRecordSummary { + return DecisionRecordSummary{ + JobID: record.JobID, + Username: record.Username, + Source: record.Source, + Status: record.Status, + TriggerReason: record.TriggerReason, + RawAllowExpand: record.RawAllowExpand, + RawExpandBytes: record.RawExpandBytes, + RawFreezeNewJobs: record.RawFreezeNewJobs, + FinalAllowExpand: record.FinalAllowExpand, + FinalExpandBytes: record.FinalExpandBytes, + FinalFreezeNewJobs: record.FinalFreezeNewJobs, + ConstraintAdjusted: record.ConstraintAdjusted, + ConstraintBlocked: record.ConstraintBlocked, + AppliedAction: record.AppliedAction, + ErrorMessage: record.ErrorMessage, + ConstraintVersion: record.ConstraintVersion, + LatencyMs: record.LatencyMs, + CreatedAt: record.CreatedAt.Format(timeLayout), + UpdatedAt: record.UpdatedAt.Format(timeLayout), + } +} + +const timeLayout = "2006-01-02 15:04:05" diff --git a/backend/pkg/storagegovernance/replay.go b/backend/pkg/storagegovernance/replay.go new file mode 100644 index 000000000..4051e0235 --- /dev/null +++ b/backend/pkg/storagegovernance/replay.go @@ -0,0 +1,88 @@ +package storagegovernance + +import ( + "context" + "encoding/json" + + "github.com/raids-lab/crater/dao/model" + "github.com/raids-lab/crater/dao/query" + "github.com/raids-lab/crater/pkg/llm" +) + +func ReplayStoredDecisions( + ctx context.Context, + cfg ConstraintConfig, + limit int, +) (*ReplaySummary, error) { + if limit <= 0 { + limit = 100 + } + + var records []model.StorageDecisionRecord + if err := query.GetDB().WithContext(ctx). + Where("status = ?", model.StorageDecisionStatusDone). + Order("created_at desc"). + Limit(limit). + Find(&records).Error; err != nil { + return nil, err + } + + summary := &ReplaySummary{ + PolicyVersion: cfg.PolicyVersion, + Records: make([]ReplayRecord, 0, len(records)), + } + + for i := range records { + record := &records[i] + if len(record.Snapshot) == 0 || len(record.RawDecision) == 0 { + continue + } + + var snapshot DecisionSnapshot + if err := json.Unmarshal(record.Snapshot, &snapshot); err != nil { + return nil, err + } + + var rawDecision llm.LLMDecisionResponse + if err := json.Unmarshal(record.RawDecision, &rawDecision); err != nil { + return nil, err + } + + replayedDecision, evaluation := ApplySafetyConstraints(snapshot, rawDecision, cfg, record.CreatedAt) + replayRecord := ReplayRecord{ + JobID: record.JobID, + Username: record.Username, + StoredAdjusted: record.ConstraintAdjusted, + StoredBlocked: record.ConstraintBlocked, + ReplayAdjusted: evaluation.Adjusted, + ReplayBlocked: evaluation.Blocked, + StoredAllowExpand: record.FinalAllowExpand, + ReplayAllowExpand: replayedDecision.AllowExpand, + StoredExpandBytes: record.FinalExpandBytes, + ReplayExpandBytes: replayedDecision.ExpandBytes, + StoredFreeze: record.FinalFreezeNewJobs, + ReplayFreeze: replayedDecision.FreezeNewJobs, + Evaluation: evaluation, + } + + summary.TotalCases++ + if evaluation.Blocked { + summary.BlockedCases++ + } + if evaluation.Adjusted { + summary.ClampedCases++ + } + if !record.FinalFreezeNewJobs && replayedDecision.FreezeNewJobs { + summary.FreezeEscalations++ + } + if record.FinalAllowExpand != replayedDecision.AllowExpand || + record.FinalExpandBytes != replayedDecision.ExpandBytes || + record.FinalFreezeNewJobs != replayedDecision.FreezeNewJobs { + summary.ChangedCases++ + } + + summary.Records = append(summary.Records, replayRecord) + } + + return summary, nil +} diff --git a/backend/pkg/storagegovernance/service.go b/backend/pkg/storagegovernance/service.go new file mode 100644 index 000000000..0205e06b1 --- /dev/null +++ b/backend/pkg/storagegovernance/service.go @@ -0,0 +1,591 @@ +//nolint:gocritic,gocyclo,mnd,unparam // Engine orchestration intentionally centralizes snapshot collection and decision execution. +package storagegovernance + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "strings" + "time" + + "gorm.io/datatypes" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + + "github.com/raids-lab/crater/dao/model" + "github.com/raids-lab/crater/dao/query" + "github.com/raids-lab/crater/pkg/ceph" + "github.com/raids-lab/crater/pkg/config" + "github.com/raids-lab/crater/pkg/llm" + "github.com/raids-lab/crater/pkg/monitor" +) + +type DecisionRequest struct { + Username string + Source model.StorageDecisionSource + TriggerReason string +} + +type StoredDecisionStatus struct { + Status string `json:"status"` + Result *llm.LLMDecisionResponse `json:"result,omitempty"` + ErrorMsg string `json:"error,omitempty"` + ConstraintAdjusted bool `json:"constraint_adjusted,omitempty"` + ConstraintBlocked bool `json:"constraint_blocked,omitempty"` +} + +type Engine struct { + kubeClient kubernetes.Interface + kubeConfig *rest.Config + promClient monitor.PrometheusInterface + config ConstraintConfig +} + +func NewEngine( + kubeClient kubernetes.Interface, + kubeConfig *rest.Config, + promClient monitor.PrometheusInterface, + cfg ConstraintConfig, +) *Engine { + if cfg.PolicyVersion == "" { + cfg = DefaultConstraintConfig() + } + return &Engine{ + kubeClient: kubeClient, + kubeConfig: kubeConfig, + promClient: promClient, + config: cfg, + } +} + +func (e *Engine) StartAsyncDecision(ctx context.Context, req DecisionRequest) (string, error) { + jobID, err := e.createPendingRecord(ctx, req) + if err != nil { + return "", err + } + + go func() { + runCtx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + _, _ = e.RunDecision(runCtx, jobID, req) + }() + + return jobID, nil +} + +func (e *Engine) DecideAndRecord(ctx context.Context, req DecisionRequest) (*llm.LLMDecisionResponse, string, error) { + jobID, err := e.createPendingRecord(ctx, req) + if err != nil { + return nil, "", err + } + + decision, err := e.RunDecision(ctx, jobID, req) + if err != nil { + return nil, jobID, err + } + + return decision, jobID, nil +} + +func (e *Engine) RunDecision(ctx context.Context, jobID string, req DecisionRequest) (*llm.LLMDecisionResponse, error) { + startedAt := time.Now() + _ = query.GetDB().WithContext(ctx). + Model(&model.StorageDecisionRecord{}). + Where("job_id = ?", jobID). + Updates(map[string]any{ + "status": model.StorageDecisionStatusRunning, + "started_at": startedAt, + }).Error + + snapshot, err := e.BuildSnapshot(ctx, req.Username) + if err != nil { + e.markError(ctx, jobID, startedAt, err) + return nil, err + } + + var rawDecision *llm.LLMDecisionResponse + switch llm.GetStorageDecisionMode(ctx) { + case llm.StorageDecisionModeDirect: + snapshotJSON, marshalErr := json.Marshal(snapshot) + if marshalErr != nil { + e.persistFailure(ctx, jobID, snapshot, startedAt, marshalErr) + return nil, marshalErr + } + rawDecision, err = llm.AskDirectDecision(ctx, string(snapshotJSON)) + default: + rawDecision, err = llm.AskAgentForDecision(e.kubeClient, e.kubeConfig, req.Username, e.promClient) + } + if err != nil { + e.persistFailure(ctx, jobID, snapshot, startedAt, err) + return nil, err + } + + finalDecision, evaluation := ApplySafetyConstraints(snapshot, *rawDecision, e.config, time.Now()) + if err := e.persistSuccess(ctx, jobID, req, snapshot, *rawDecision, finalDecision, evaluation, startedAt); err != nil { + return nil, err + } + + return &finalDecision, nil +} + +func (e *Engine) BuildSnapshot(ctx context.Context, username string) (DecisionSnapshot, error) { + var userRow struct { + ID uint `gorm:"column:id"` + Name string `gorm:"column:name"` + Space string `gorm:"column:space"` + SpaceQuota int64 `gorm:"column:space_quota"` + OriginalSpaceQuota *int64 `gorm:"column:original_space_quota"` + JobsFrozen bool `gorm:"column:jobs_frozen"` + ShrinkStage string `gorm:"column:shrink_stage"` + } + + if err := query.GetDB().WithContext(ctx).Raw( + "SELECT id, name, space, space_quota, original_space_quota, jobs_frozen, shrink_stage FROM users WHERE name = ? AND deleted_at IS NULL", + username, + ).Scan(&userRow).Error; err != nil { + return DecisionSnapshot{}, fmt.Errorf("query user snapshot failed: %w", err) + } + if userRow.ID == 0 { + return DecisionSnapshot{}, fmt.Errorf("user %s not found", username) + } + + cfg := config.GetConfig() + prefixConfig := ceph.StoragePrefixConfig{ + User: cfg.Storage.Prefix.User, + Account: cfg.Storage.Prefix.Account, + Public: cfg.Storage.Prefix.Public, + } + + currentUsage, err := ceph.GetCephDirectorySize( + e.kubeClient, + e.kubeConfig, + ceph.StorageQuotaRookNamespace(), + "/user/"+userRow.Space, + prefixConfig, + ) + if err != nil { + currentUsage = ceph.UnknownSizeBytes + } + + theoreticalQuota := userRow.SpaceQuota + if userRow.OriginalSpaceQuota != nil { + theoreticalQuota = *userRow.OriginalSpaceQuota + } + + totalCapacity, usedCapacity, err := ceph.GetCraterStorageCapacity( + e.kubeClient, e.kubeConfig, cfg.Namespaces.Job, + ) + if err != nil { + return DecisionSnapshot{}, fmt.Errorf("get platform capacity failed: %w", err) + } + + runtimeFeatures, err := e.collectTenantRuntimeFeatures(ctx, username) + if err != nil { + return DecisionSnapshot{}, fmt.Errorf("collect tenant runtime features failed: %w", err) + } + + var historyRows []model.TenantUsageHistory + if err := query.GetDB().WithContext(ctx). + Where("tenant_id = ?", userRow.ID). + Order("recorded_at desc"). + Limit(10). + Find(&historyRows).Error; err != nil { + return DecisionSnapshot{}, fmt.Errorf("query usage history failed: %w", err) + } + + recentHistory := make([]UsageHistoryPoint, 0, len(historyRows)) + for _, row := range historyRows { + recentHistory = append(recentHistory, UsageHistoryPoint{ + RecordedAt: row.RecordedAt, + UsageBytes: row.UsageBytes, + }) + } + slices.Reverse(recentHistory) + + var growthRate *float64 + if len(recentHistory) >= 2 { + first := recentHistory[0] + last := recentHistory[len(recentHistory)-1] + hours := last.RecordedAt.Sub(first.RecordedAt).Hours() + if hours > 0 { + value := float64(last.UsageBytes-first.UsageBytes) / hours + growthRate = &value + } + } + + var usageRatio float64 + if theoreticalQuota > 0 && currentUsage >= 0 { + usageRatio = float64(currentUsage) / float64(theoreticalQuota) + } + availableCapacity := ceph.AvailableBytes(totalCapacity, usedCapacity) + + var lastExpandRecord model.StorageDecisionRecord + var lastExpandAt *time.Time + appliedExpandActions := []string{ + "expand", + "manual_expand", + "manual_expand_and_freeze", + } + if err := query.GetDB().WithContext(ctx). + Where( + "username = ? AND status = ? AND applied_action IN ?", + username, + model.StorageDecisionStatusDone, + appliedExpandActions, + ). + Order("updated_at desc"). + First(&lastExpandRecord).Error; err == nil { + lastExpandAt = &lastExpandRecord.UpdatedAt + } + + return DecisionSnapshot{ + Username: username, + UserID: userRow.ID, + CurrentUsageBytes: currentUsage, + CurrentQuotaBytes: userRow.SpaceQuota, + TheoreticalQuotaBytes: theoreticalQuota, + UsageRatio: usageRatio, + GrowthRateBytesPerHour: growthRate, + PlatformTotalBytes: totalCapacity, + PlatformUsedBytes: usedCapacity, + PlatformAvailableBytes: availableCapacity, + IsCurrentlyExpanded: userRow.OriginalSpaceQuota != nil, + JobsFrozen: userRow.JobsFrozen, + ShrinkStage: userRow.ShrinkStage, + ActivePodCount: runtimeFeatures.ActivePodCount, + ActiveGPUPodCount: runtimeFeatures.ActiveGPUPodCount, + ActiveGPURequestTotal: runtimeFeatures.ActiveGPURequestTotal, + ActiveCPURequestCores: runtimeFeatures.ActiveCPURequestCores, + ActiveMemoryRequestMB: runtimeFeatures.ActiveMemoryRequestMB, + RealtimeCPUCores: runtimeFeatures.RealtimeCPUCores, + RealtimeMemoryMB: runtimeFeatures.RealtimeMemoryMB, + RealtimeGPUUtilPercent: runtimeFeatures.RealtimeGPUUtilPercent, + RealtimeGPUMemoryMB: runtimeFeatures.RealtimeGPUMemoryMB, + GPUDataAvailable: runtimeFeatures.GPUDataAvailable, + MaxGPUHistoryPercent: runtimeFeatures.MaxGPUHistoryPercent, + LastExpandAt: lastExpandAt, + RecentHistory: recentHistory, + }, nil +} + +type tenantRuntimeFeatures struct { + ActivePodCount int + ActiveGPUPodCount int + ActiveGPURequestTotal int + ActiveCPURequestCores float64 + ActiveMemoryRequestMB float64 + RealtimeCPUCores float64 + RealtimeMemoryMB float64 + RealtimeGPUUtilPercent float64 + RealtimeGPUMemoryMB float64 + GPUDataAvailable bool + MaxGPUHistoryPercent float64 +} + +func (e *Engine) collectTenantRuntimeFeatures(ctx context.Context, username string) (*tenantRuntimeFeatures, error) { + features := &tenantRuntimeFeatures{} + jobNamespace := config.GetConfig().Namespaces.Job + + pods, err := e.kubeClient.CoreV1().Pods(jobNamespace).List(ctx, metav1.ListOptions{ + LabelSelector: "crater.raids.io/task-user=" + username, + }) + if err != nil { + return nil, err + } + + gpuUtilSum := 0.0 + gpuUtilCount := 0 + gpuMemSum := 0.0 + + for _, pod := range pods.Items { + phase := string(pod.Status.Phase) + if phase != "Running" && phase != "Pending" { + continue + } + + features.ActivePodCount++ + podGPURequests := 0 + for _, container := range pod.Spec.Containers { + for resourceName, quantity := range container.Resources.Requests { + if resourceName == corev1.ResourceCPU { + features.ActiveCPURequestCores += float64(quantity.MilliValue()) / 1000 + } + if resourceName == corev1.ResourceMemory { + features.ActiveMemoryRequestMB += float64(quantity.Value()) / 1024 / 1024 + } + if strings.Contains(string(resourceName), "nvidia.com/") { + podGPURequests += int(quantity.Value()) + features.ActiveGPURequestTotal += int(quantity.Value()) + } + } + } + if podGPURequests > 0 { + features.ActiveGPUPodCount++ + } + + if e.promClient != nil { + runtimeMetrics, err := e.queryPodRealtimeMetrics(pod.Name) + if err == nil { + features.RealtimeCPUCores += runtimeMetrics.CPUCores + features.RealtimeMemoryMB += runtimeMetrics.MemoryMB + if runtimeMetrics.GPUDataAvailable { + features.GPUDataAvailable = true + gpuUtilSum += runtimeMetrics.GPUUtilPercent + gpuUtilCount++ + gpuMemSum += runtimeMetrics.GPUMemoryMB + } + } + + if podGPURequests > 0 { + historyMetrics, err := e.queryPodGPUHistory(pod.Name, 24) + if err == nil && historyMetrics.MaxUtil > features.MaxGPUHistoryPercent { + features.MaxGPUHistoryPercent = historyMetrics.MaxUtil + } + } + } + } + + if gpuUtilCount > 0 { + features.RealtimeGPUUtilPercent = gpuUtilSum / float64(gpuUtilCount) + features.RealtimeGPUMemoryMB = gpuMemSum / float64(gpuUtilCount) + } + + return features, nil +} + +type podRealtimeMetrics struct { + CPUCores float64 + MemoryMB float64 + GPUUtilPercent float64 + GPUMemoryMB float64 + GPUDataAvailable bool +} + +func (e *Engine) queryPodRealtimeMetrics(podName string) (*podRealtimeMetrics, error) { + jobNamespace := config.GetConfig().Namespaces.Job + result := &podRealtimeMetrics{} + + if v, ok, err := e.promClient.QueryInstant( + fmt.Sprintf(`sum(rate(container_cpu_usage_seconds_total{pod=%q,container!=""}[5m]))`, podName), + ); err == nil && ok { + result.CPUCores = v + } + if v, ok, err := e.promClient.QueryInstant( + fmt.Sprintf(`sum(container_memory_usage_bytes{pod=%q,container!=""})`, podName), + ); err == nil && ok { + result.MemoryMB = v / 1024 / 1024 + } + + for _, query := range []string{ + fmt.Sprintf(`avg(DCGM_FI_DEV_GPU_UTIL{namespace=%q,pod=%q})`, jobNamespace, podName), + fmt.Sprintf(`avg(DCGM_FI_DEV_GPU_UTIL{pod=%q})`, podName), + } { + if v, ok, err := e.promClient.QueryInstant(query); err == nil && ok { + result.GPUUtilPercent = v + result.GPUDataAvailable = true + break + } + } + + for _, query := range []string{ + fmt.Sprintf(`avg(DCGM_FI_DEV_FB_USED{namespace=%q,pod=%q})`, jobNamespace, podName), + fmt.Sprintf(`avg(DCGM_FI_DEV_FB_USED{pod=%q})`, podName), + } { + if v, ok, err := e.promClient.QueryInstant(query); err == nil && ok { + result.GPUMemoryMB = v + break + } + } + + return result, nil +} + +type podGPUHistory struct { + MaxUtil float64 +} + +func (e *Engine) queryPodGPUHistory(podName string, durationHours float64) (*podGPUHistory, error) { + jobNamespace := config.GetConfig().Namespaces.Job + duration := fmt.Sprintf("%.0fh", durationHours) + if durationHours < 1 { + duration = fmt.Sprintf("%.0fm", durationHours*60) + } + + result := &podGPUHistory{} + for _, query := range []string{ + fmt.Sprintf(`max_over_time(DCGM_FI_DEV_GPU_UTIL{namespace=%q,pod=%q}[%s])`, jobNamespace, podName, duration), + fmt.Sprintf(`max_over_time(DCGM_FI_DEV_GPU_UTIL{pod=%q}[%s])`, podName, duration), + } { + if v, ok, err := e.promClient.QueryInstant(query); err == nil && ok { + result.MaxUtil = v + return result, nil + } + } + + return result, nil +} + +func GetDecisionStatus(ctx context.Context, jobID string) (*StoredDecisionStatus, error) { + var record model.StorageDecisionRecord + if err := query.GetDB().WithContext(ctx).Where("job_id = ?", jobID).First(&record).Error; err != nil { + return nil, err + } + + status := &StoredDecisionStatus{ + Status: string(record.Status), + ErrorMsg: record.ErrorMessage, + ConstraintAdjusted: record.ConstraintAdjusted, + ConstraintBlocked: record.ConstraintBlocked, + } + + if len(record.FinalDecision) > 0 { + var decision llm.LLMDecisionResponse + if err := json.Unmarshal(record.FinalDecision, &decision); err == nil { + status.Result = &decision + } + } + + return status, nil +} + +func MarkDecisionExecution(ctx context.Context, jobID, action string, runErr error) error { + updates := map[string]any{ + "applied_action": action, + } + if runErr != nil { + updates["error_message"] = runErr.Error() + } else { + updates["error_message"] = "" + } + + return query.GetDB().WithContext(ctx). + Model(&model.StorageDecisionRecord{}). + Where("job_id = ?", jobID). + Updates(updates).Error +} + +func (e *Engine) createPendingRecord(ctx context.Context, req DecisionRequest) (string, error) { + var userRow struct { + ID uint `gorm:"column:id"` + } + if err := query.GetDB().WithContext(ctx).Raw( + "SELECT id FROM users WHERE name = ? AND deleted_at IS NULL", + req.Username, + ).Scan(&userRow).Error; err != nil { + return "", err + } + if userRow.ID == 0 { + return "", fmt.Errorf("user %s not found", req.Username) + } + + jobID := NewJobID() + record := model.StorageDecisionRecord{ + JobID: jobID, + UserID: userRow.ID, + Username: req.Username, + Source: req.Source, + Status: model.StorageDecisionStatusPending, + TriggerReason: req.TriggerReason, + StartedAt: nil, + } + if err := query.GetDB().WithContext(ctx).Create(&record).Error; err != nil { + return "", err + } + return jobID, nil +} + +func (e *Engine) persistFailure( + ctx context.Context, + jobID string, + snapshot DecisionSnapshot, + startedAt time.Time, + runErr error, +) { + updates := map[string]any{ + "status": model.StorageDecisionStatusError, + "error_message": runErr.Error(), + "finished_at": time.Now(), + "latency_ms": time.Since(startedAt).Milliseconds(), + } + if data, err := json.Marshal(snapshot); err == nil { + updates["snapshot"] = datatypes.JSON(data) + } + _ = query.GetDB().WithContext(ctx). + Model(&model.StorageDecisionRecord{}). + Where("job_id = ?", jobID). + Updates(updates).Error +} + +func (e *Engine) markError(ctx context.Context, jobID string, startedAt time.Time, runErr error) { + _ = query.GetDB().WithContext(ctx). + Model(&model.StorageDecisionRecord{}). + Where("job_id = ?", jobID). + Updates(map[string]any{ + "status": model.StorageDecisionStatusError, + "error_message": runErr.Error(), + "finished_at": time.Now(), + "latency_ms": time.Since(startedAt).Milliseconds(), + }).Error +} + +func (e *Engine) persistSuccess( + ctx context.Context, + jobID string, + req DecisionRequest, + snapshot DecisionSnapshot, + rawDecision llm.LLMDecisionResponse, + finalDecision llm.LLMDecisionResponse, + evaluation ConstraintEvaluation, + startedAt time.Time, +) error { + snapshotJSON, err := json.Marshal(snapshot) + if err != nil { + return err + } + rawDecisionJSON, err := json.Marshal(rawDecision) + if err != nil { + return err + } + finalDecisionJSON, err := json.Marshal(finalDecision) + if err != nil { + return err + } + evaluationJSON, err := json.Marshal(evaluation) + if err != nil { + return err + } + + return query.GetDB().WithContext(ctx). + Model(&model.StorageDecisionRecord{}). + Where("job_id = ?", jobID). + Updates(map[string]any{ + "user_id": snapshot.UserID, + "username": req.Username, + "source": req.Source, + "status": model.StorageDecisionStatusDone, + "snapshot": datatypes.JSON(snapshotJSON), + "raw_decision": datatypes.JSON(rawDecisionJSON), + "final_decision": datatypes.JSON(finalDecisionJSON), + "constraint_result": datatypes.JSON(evaluationJSON), + "raw_allow_expand": rawDecision.AllowExpand, + "raw_expand_bytes": rawDecision.ExpandBytes, + "raw_freeze_new_jobs": rawDecision.FreezeNewJobs, + "final_allow_expand": finalDecision.AllowExpand, + "final_expand_bytes": finalDecision.ExpandBytes, + "final_freeze_new_jobs": finalDecision.FreezeNewJobs, + "constraint_adjusted": evaluation.Adjusted, + "constraint_blocked": evaluation.Blocked, + "constraint_version": evaluation.PolicyVersion, + "applied_action": "", + "error_message": "", + "finished_at": time.Now(), + "latency_ms": time.Since(startedAt).Milliseconds(), + }).Error +} + +func NewJobID() string { + return fmt.Sprintf("sd-%d", time.Now().UnixNano()) +} diff --git a/backend/pkg/storagegovernance/types.go b/backend/pkg/storagegovernance/types.go new file mode 100644 index 000000000..bc5f8e184 --- /dev/null +++ b/backend/pkg/storagegovernance/types.go @@ -0,0 +1,95 @@ +package storagegovernance + +import "time" + +type UsageHistoryPoint struct { + RecordedAt time.Time `json:"recorded_at"` + UsageBytes int64 `json:"usage_bytes"` +} + +type DecisionSnapshot struct { + Username string `json:"username"` + UserID uint `json:"user_id"` + CurrentUsageBytes int64 `json:"current_usage_bytes"` + CurrentQuotaBytes int64 `json:"current_quota_bytes"` + TheoreticalQuotaBytes int64 `json:"theoretical_quota_bytes"` + UsageRatio float64 `json:"usage_ratio"` + GrowthRateBytesPerHour *float64 `json:"growth_rate_bytes_per_hour,omitempty"` + PlatformTotalBytes int64 `json:"platform_total_bytes"` + PlatformUsedBytes int64 `json:"platform_used_bytes"` + PlatformAvailableBytes int64 `json:"platform_available_bytes"` + IsCurrentlyExpanded bool `json:"is_currently_expanded"` + JobsFrozen bool `json:"jobs_frozen"` + ShrinkStage string `json:"shrink_stage"` + ActivePodCount int `json:"active_pod_count"` + ActiveGPUPodCount int `json:"active_gpu_pod_count"` + ActiveGPURequestTotal int `json:"active_gpu_request_total"` + ActiveCPURequestCores float64 `json:"active_cpu_request_cores"` + ActiveMemoryRequestMB float64 `json:"active_memory_request_mb"` + RealtimeCPUCores float64 `json:"realtime_cpu_cores"` + RealtimeMemoryMB float64 `json:"realtime_memory_mb"` + RealtimeGPUUtilPercent float64 `json:"realtime_gpu_util_percent"` + RealtimeGPUMemoryMB float64 `json:"realtime_gpu_memory_mb"` + GPUDataAvailable bool `json:"gpu_data_available"` + MaxGPUHistoryPercent float64 `json:"max_gpu_history_percent"` + LastExpandAt *time.Time `json:"last_expand_at,omitempty"` + RecentHistory []UsageHistoryPoint `json:"recent_history,omitempty"` +} + +type ConstraintConfig struct { + PolicyVersion string + AlertThreshold float64 + MaxExpandRatio float64 + MaxExpandBytes int64 + MinPlatformReservedRatio float64 + MinPlatformReservedBytes int64 + ExpansionCooldown time.Duration + ForceFreezeWhenOverQuota bool +} + +func DefaultConstraintConfig() ConstraintConfig { + return ConstraintConfig{ + PolicyVersion: "storage-safety-v1", + AlertThreshold: 0.90, + MaxExpandRatio: 0.30, + MaxExpandBytes: 500 * 1024 * 1024 * 1024, + MinPlatformReservedRatio: 0.10, + MinPlatformReservedBytes: 200 * 1024 * 1024 * 1024, + ExpansionCooldown: 6 * time.Hour, + ForceFreezeWhenOverQuota: true, + } +} + +type ConstraintEvaluation struct { + PolicyVersion string `json:"policy_version"` + Adjusted bool `json:"adjusted"` + Blocked bool `json:"blocked"` + Violations []string `json:"violations"` + Adjustments []string `json:"adjustments"` +} + +type ReplayRecord struct { + JobID string `json:"job_id"` + Username string `json:"username"` + StoredAdjusted bool `json:"stored_adjusted"` + StoredBlocked bool `json:"stored_blocked"` + ReplayAdjusted bool `json:"replay_adjusted"` + ReplayBlocked bool `json:"replay_blocked"` + StoredAllowExpand bool `json:"stored_allow_expand"` + ReplayAllowExpand bool `json:"replay_allow_expand"` + StoredExpandBytes int64 `json:"stored_expand_bytes"` + ReplayExpandBytes int64 `json:"replay_expand_bytes"` + StoredFreeze bool `json:"stored_freeze"` + ReplayFreeze bool `json:"replay_freeze"` + Evaluation ConstraintEvaluation `json:"evaluation"` +} + +type ReplaySummary struct { + TotalCases int `json:"total_cases"` + ChangedCases int `json:"changed_cases"` + BlockedCases int `json:"blocked_cases"` + ClampedCases int `json:"clamped_cases"` + FreezeEscalations int `json:"freeze_escalations"` + PolicyVersion string `json:"policy_version"` + Records []ReplayRecord `json:"records,omitempty"` +} diff --git a/backend/pkg/storagequota/client.go b/backend/pkg/storagequota/client.go new file mode 100644 index 000000000..b645d2210 --- /dev/null +++ b/backend/pkg/storagequota/client.go @@ -0,0 +1,196 @@ +package storagequota + +import ( + "bytes" + "context" + "crypto/sha256" + "crypto/subtle" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "time" +) + +const ( + InternalTokenHeader = "X-Crater-Internal-Token" //nolint:gosec // This is a header name, not a credential. + InternalTokenEnv = "CRATER_STORAGE_INTERNAL_TOKEN" + InternalSecretEnv = "CRATER_STORAGE_INTERNAL_SECRET" + ServerURLEnv = "CRATER_STORAGE_QUOTA_SERVER_URL" + + ProviderAuto = "auto" + ProviderStorageServer = "storageServer" + ProviderToolbox = "toolbox" + ProviderDisabled = "disabled" + maxResponseBodyBytes = 1 << 20 +) + +type Capabilities struct { + UsageReadable bool `json:"usage_readable"` + QuotaReadable bool `json:"quota_readable"` + QuotaWritable bool `json:"quota_writable"` + Reasons []string `json:"reasons,omitempty"` +} + +type Usage struct { + Path string `json:"path"` + Bytes int64 `json:"bytes"` +} + +type Quota struct { + Path string `json:"path"` + MaxBytes int64 `json:"max_bytes"` +} + +type Client struct { + baseURL string + token string + httpClient *http.Client +} + +func NewClient(baseURL, accessTokenSecret string) *Client { + return &Client{ + baseURL: strings.TrimRight(strings.TrimSpace(baseURL), "/"), + token: DeriveInternalToken(accessTokenSecret), + httpClient: &http.Client{ + Timeout: 5 * time.Second, + }, + } +} + +func ResolveServerURL(configuredURL, namespace string) string { + if envURL := strings.TrimSpace(os.Getenv(ServerURLEnv)); envURL != "" { + return strings.TrimRight(envURL, "/") + } + if configuredURL = strings.TrimSpace(configuredURL); configuredURL != "" { + return strings.TrimRight(configuredURL, "/") + } + if namespace = strings.TrimSpace(namespace); namespace != "" { + return fmt.Sprintf("http://webdav-service.%s.svc:7320", namespace) + } + return "http://webdav-service:7320" +} + +func NormalizeProvider(provider string) string { + switch strings.ToLower(strings.TrimSpace(provider)) { + case "", ProviderAuto: + return ProviderAuto + case strings.ToLower(ProviderStorageServer), "storage-server": + return ProviderStorageServer + case ProviderToolbox: + return ProviderToolbox + case ProviderDisabled: + return ProviderDisabled + default: + return ProviderDisabled + } +} + +func DeriveInternalToken(accessTokenSecret string) string { + sum := sha256.Sum256([]byte("crater-storage-quota:" + accessTokenSecret)) + return hex.EncodeToString(sum[:]) +} + +func Authenticate(accessTokenSecret, suppliedToken string) bool { + return AuthenticateToken(DeriveInternalToken(accessTokenSecret), suppliedToken) +} + +func AuthenticateToken(expectedToken, suppliedToken string) bool { + expected := []byte(strings.TrimSpace(expectedToken)) + supplied := []byte(strings.TrimSpace(suppliedToken)) + return len(expected) == len(supplied) && subtle.ConstantTimeCompare(expected, supplied) == 1 +} + +func (c *Client) GetCapabilities(ctx context.Context) (Capabilities, error) { + var result Capabilities + err := c.do(ctx, http.MethodGet, "/internal/storage/capabilities", nil, nil, &result) + return result, err +} + +func (c *Client) GetUsage(ctx context.Context, relativePath string) (Usage, error) { + var result Usage + err := c.do(ctx, http.MethodGet, "/internal/storage/usage", url.Values{"path": {relativePath}}, nil, &result) + return result, err +} + +func (c *Client) GetQuota(ctx context.Context, relativePath string) (Quota, error) { + var result Quota + err := c.do(ctx, http.MethodGet, "/internal/storage/quota", url.Values{"path": {relativePath}}, nil, &result) + return result, err +} + +func (c *Client) SetQuota(ctx context.Context, relativePath string, maxBytes int64) (Quota, error) { + var result Quota + err := c.do(ctx, http.MethodPut, "/internal/storage/quota", nil, Quota{ + Path: relativePath, + MaxBytes: maxBytes, + }, &result) + return result, err +} + +func (c *Client) do( + ctx context.Context, + method, endpoint string, + query url.Values, + body any, + result any, +) error { + if c.baseURL == "" { + return fmt.Errorf("storage quota server URL is empty") + } + + requestURL := c.baseURL + endpoint + if len(query) > 0 { + requestURL += "?" + query.Encode() + } + + var bodyReader io.Reader = http.NoBody + if body != nil { + payload, err := json.Marshal(body) + if err != nil { + return fmt.Errorf("marshal storage quota request: %w", err) + } + bodyReader = bytes.NewReader(payload) + } + + req, err := http.NewRequestWithContext(ctx, method, requestURL, bodyReader) + if err != nil { + return fmt.Errorf("create storage quota request: %w", err) + } + req.Header.Set(InternalTokenHeader, c.token) + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("call storage quota server: %w", err) + } + defer resp.Body.Close() + + payload, err := io.ReadAll(io.LimitReader(resp.Body, maxResponseBodyBytes)) + if err != nil { + return fmt.Errorf("read storage quota response: %w", err) + } + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + var errorBody struct { + Error string `json:"error"` + } + _ = json.Unmarshal(payload, &errorBody) + if errorBody.Error == "" { + errorBody.Error = strings.TrimSpace(string(payload)) + } + return fmt.Errorf("storage quota server returned %s: %s", resp.Status, errorBody.Error) + } + if result == nil { + return nil + } + if err := json.Unmarshal(payload, result); err != nil { + return fmt.Errorf("decode storage quota response: %w", err) + } + return nil +} diff --git a/backend/pkg/storagequota/client_test.go b/backend/pkg/storagequota/client_test.go new file mode 100644 index 000000000..b771872f4 --- /dev/null +++ b/backend/pkg/storagequota/client_test.go @@ -0,0 +1,84 @@ +package storagequota + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +func TestAuthenticate(t *testing.T) { + t.Parallel() + + token := DeriveInternalToken("secret") + if !Authenticate("secret", token) { + t.Fatal("expected derived token to authenticate") + } + if Authenticate("secret", token+"x") { + t.Fatal("expected invalid token to be rejected") + } + if !AuthenticateToken(token, " "+token+" ") { + t.Fatal("expected direct internal token to authenticate") + } + if AuthenticateToken(token, "") { + t.Fatal("expected empty direct token to be rejected") + } +} + +func TestClientGetUsageAndSetQuota(t *testing.T) { + t.Parallel() + + const secret = "test-secret" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !Authenticate(secret, r.Header.Get(InternalTokenHeader)) { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + switch { + case r.Method == http.MethodGet && r.URL.Path == "/internal/storage/usage": + if got := r.URL.Query().Get("path"); got != "users/alice" { + t.Errorf("unexpected usage path %q", got) + } + _ = json.NewEncoder(w).Encode(Usage{Path: "users/alice", Bytes: 42}) + case r.Method == http.MethodPut && r.URL.Path == "/internal/storage/quota": + var request Quota + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Errorf("decode quota request: %v", err) + } + _ = json.NewEncoder(w).Encode(request) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + client := NewClient(server.URL, secret) + usage, err := client.GetUsage(context.Background(), "users/alice") + if err != nil || usage.Bytes != 42 { + t.Fatalf("GetUsage() = %+v, %v", usage, err) + } + quota, err := client.SetQuota(context.Background(), "users/alice", 1024) + if err != nil || quota.MaxBytes != 1024 { + t.Fatalf("SetQuota() = %+v, %v", quota, err) + } +} + +func TestNormalizeProvider(t *testing.T) { + t.Parallel() + + tests := map[string]string{ + "": ProviderAuto, + "AUTO": ProviderAuto, + "storage-server": ProviderStorageServer, + "storageServer": ProviderStorageServer, + "toolbox": ProviderToolbox, + "disabled": ProviderDisabled, + "invalid": ProviderDisabled, + } + for input, want := range tests { + if got := NormalizeProvider(input); got != want { + t.Errorf("NormalizeProvider(%q) = %q, want %q", input, got, want) + } + } +} diff --git a/charts/crater/Chart.yaml b/charts/crater/Chart.yaml index 7d59287f4..8aef579ae 100644 --- a/charts/crater/Chart.yaml +++ b/charts/crater/Chart.yaml @@ -15,13 +15,13 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.1.2 +version: 1.2.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "1.1.2" +appVersion: "1.2.0" # Additional metadata home: https://github.com/raids-lab/crater diff --git a/charts/crater/README.md b/charts/crater/README.md index 28f83782f..dd6d7d4c3 100644 --- a/charts/crater/README.md +++ b/charts/crater/README.md @@ -1,6 +1,6 @@ # crater -![Version: 1.1.2](https://img.shields.io/badge/Version-1.1.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.1.2](https://img.shields.io/badge/AppVersion-1.1.2-informational?style=flat-square) +![Version: 1.2.0](https://img.shields.io/badge/Version-1.2.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.2.0](https://img.shields.io/badge/AppVersion-1.2.0-informational?style=flat-square) A comprehensive AI development platform for Kubernetes that provides GPU resource management, containerized development environments, and workflow orchestration. @@ -94,7 +94,14 @@ A comprehensive AI development platform for Kubernetes that provides GPU resourc | backendConfig.smtp.password | string | `""` | Password for SMTP authentication (Required if Enable is true) Must match the specified user's password | | backendConfig.smtp.port | int | `25` | SMTP server port number (Required if Enable is true) Typically 25, 465, or 587 | | backendConfig.smtp.user | string | `"example"` | Username for SMTP authentication (Required if Enable is true) Must be a valid SMTP user | -| backendConfig.storage | object | `{"prefix":{"account":"accounts","public":"public","user":"users"},"pvc":{"readOnlyMany":null,"readWriteMany":"crater-rw-storage"}}` | Persistent volume claim and path prefix configurations (Required) All PVC names and prefix paths must be specified | +| backendConfig.storage | object | `{"prefix":{"account":"accounts","public":"public","user":"users"},"pvc":{"readOnlyMany":null,"readWriteMany":"crater-rw-storage"},"quota":{"cephFSCSIDriver":"","cephFSName":"cephfs","enabled":false,"provider":"auto","rookNamespace":"rook-ceph","storageServerURL":"","toolboxLabelSelector":"app=rook-ceph-tools"}}` | Persistent volume claim and path prefix configurations (Required) All PVC names and prefix paths must be specified | +| backendConfig.storage.quota.cephFSCSIDriver | string | `""` | Optional CephFS CSI driver override; empty derives `.cephfs.csi.ceph.com`. | +| backendConfig.storage.quota.cephFSName | string | `"cephfs"` | Fallback filesystem name when the PV does not expose `volumeAttributes.fsName`. | +| backendConfig.storage.quota.enabled | bool | `false` | Enable CephFS usage and quota management. Keep disabled for NFS and other storage backends. | +| backendConfig.storage.quota.provider | string | `"auto"` | Select auto, storageServer, toolbox, or disabled. Auto uses a Rook Ceph toolbox only as a fallback. | +| backendConfig.storage.quota.rookNamespace | string | `"rook-ceph"` | Namespace containing Rook Ceph resources and the optional toolbox. | +| backendConfig.storage.quota.storageServerURL | string | `""` | Internal storage-server endpoint; empty derives the service URL from the job namespace. | +| backendConfig.storage.quota.toolboxLabelSelector | string | `"app=rook-ceph-tools"` | Label selector for the optional toolbox fallback Pod. | | backendConfig.storage.prefix | object | `{"account":"accounts","public":"public","user":"users"}` | Path prefixes for different types of storage locations (Required) All prefix paths must be specified | | backendConfig.storage.prefix.account | string | `"accounts"` | Account prefix for account-related storage paths (Required) Must be a valid path within the storage system | | backendConfig.storage.prefix.public | string | `"public"` | Public prefix for publicly accessible storage paths (Required) Must be a valid path within the storage system | @@ -226,6 +233,10 @@ A comprehensive AI development platform for Kubernetes that provides GPU resourc | namespaces.job | string | `"crater-workspace"` | Namespace for running jobs | | nodeSelector | object | `{"node-role.kubernetes.io/control-plane":""}` | Node selector for all Deployments Prevents control components from being scheduled to GPU worker nodes | | protocol | string | `"http"` | Protocol for server communication ("http" or "https") | +| quotaAgent | object | `{"enabled":false,"existingClaim":"","replicas":1,"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"50m","memory":"64Mi"}}}` | Dedicated CephFS quota agent that reuses `images.storage` and a pre-created static PVC with path-scoped CephX `p` capability | +| quotaAgent.enabled | bool | `false` | Create the internal quota-agent Deployment and Service | +| quotaAgent.existingClaim | string | `""` | Pre-created static CephFS PVC using the dedicated quota CephX identity | +| quotaAgent.replicas | int | `1` | Number of quota-agent replicas | | storage | object | `{"create":true,"pvcName":"crater-rw-storage","request":"10Gi","storageClass":"nfs"}` | Persistent Volume Claim configuration | | storage.create | bool | `true` | Whether to create PVC or use existing pvc. | | storage.pvcName | string | `"crater-rw-storage"` | PVC name (existing or created, also used in backendConfig) | diff --git a/charts/crater/templates/_helpers.tpl b/charts/crater/templates/_helpers.tpl index 22e1a9b00..d7068d70a 100644 --- a/charts/crater/templates/_helpers.tpl +++ b/charts/crater/templates/_helpers.tpl @@ -73,6 +73,15 @@ Generate backend config with images from top-level images section ) -}} {{- $_ := set $config.registry "buildTools" $buildTools -}} {{- end -}} +{{- if .Values.quotaAgent.enabled -}} + {{- $storage := get $config "storage" -}} + {{- $quota := default (dict) (get $storage "quota") -}} + {{- $_ := set $quota "enabled" true -}} + {{- $_ := set $quota "provider" "storageServer" -}} + {{- $_ := set $quota "storageServerURL" (printf "http://crater-quota-agent.%s.svc:7320" .Values.namespaces.job) -}} + {{- $_ := set $storage "quota" $quota -}} + {{- $_ := set $config "storage" $storage -}} +{{- end -}} {{- $_ := set $config "host" .Values.host -}} {{- $_ := set $config "namespaces" (dict "job" .Values.namespaces.job "image" .Values.namespaces.image) -}} {{- $config | toYaml -}} @@ -84,12 +93,18 @@ Avoid rendering full backend config into ss-config. */}} {{- define "crater.storageServerConfig" -}} {{- $backend := .Values.backendConfig -}} +{{- $storage := deepCopy $backend.storage -}} +{{- if .Values.quotaAgent.enabled -}} + {{- $quota := default (dict) (get $storage "quota") -}} + {{- $_ := set $quota "enabled" true -}} + {{- $_ := set $storage "quota" $quota -}} +{{- end -}} {{- $config := dict "host" .Values.host "port" $backend.port "namespaces" (dict "job" .Values.namespaces.job "image" .Values.namespaces.image) "postgres" $backend.postgres - "storage" $backend.storage + "storage" $storage "secrets" $backend.secrets "auth" (dict "token" $backend.auth.token @@ -101,4 +116,3 @@ Avoid rendering full backend config into ss-config. -}} {{- $config | toYaml -}} {{- end -}} - diff --git a/charts/crater/templates/quota-agent/deployment.yaml b/charts/crater/templates/quota-agent/deployment.yaml new file mode 100644 index 000000000..4617fbe39 --- /dev/null +++ b/charts/crater/templates/quota-agent/deployment.yaml @@ -0,0 +1,64 @@ +{{- if .Values.quotaAgent.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: crater-quota-agent + namespace: {{ .Values.namespaces.job }} + labels: + app: crater-quota-agent +spec: + replicas: {{ .Values.quotaAgent.replicas }} + selector: + matchLabels: + app: crater-quota-agent + template: + metadata: + labels: + app: crater-quota-agent + annotations: + checksum/auth: {{ include (print $.Template.BasePath "/quota-agent/secret.yaml") . | sha256sum }} + spec: + serviceAccountName: default + automountServiceAccountToken: false + nodeSelector: {{ .Values.nodeSelector | toYaml | nindent 8 }} + tolerations: {{ .Values.tolerations | toYaml | nindent 8 }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: quota-agent + image: {{ .Values.images.storage.repository }}:{{ .Values.images.storage.tag }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + env: + - name: CRATER_STORAGE_MODE + value: quota-agent + - name: CRATER_STORAGE_INTERNAL_TOKEN + valueFrom: + secretKeyRef: + name: crater-quota-agent-auth + key: internal-token + ports: + - name: http + containerPort: 7320 + protocol: TCP + readinessProbe: + tcpSocket: + port: http + initialDelaySeconds: 2 + periodSeconds: 5 + livenessProbe: + tcpSocket: + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + resources: + {{- toYaml .Values.quotaAgent.resources | nindent 12 }} + volumeMounts: + - name: storage + mountPath: /crater + volumes: + - name: storage + persistentVolumeClaim: + claimName: {{ required "quotaAgent.existingClaim is required when quotaAgent.enabled=true" .Values.quotaAgent.existingClaim }} +{{- end }} diff --git a/charts/crater/templates/quota-agent/secret.yaml b/charts/crater/templates/quota-agent/secret.yaml new file mode 100644 index 000000000..6f6f15e66 --- /dev/null +++ b/charts/crater/templates/quota-agent/secret.yaml @@ -0,0 +1,12 @@ +{{- if .Values.quotaAgent.enabled -}} +apiVersion: v1 +kind: Secret +metadata: + name: crater-quota-agent-auth + namespace: {{ .Values.namespaces.job }} + labels: + app: crater-quota-agent +type: Opaque +data: + internal-token: {{ printf "crater-storage-quota:%s" .Values.backendConfig.auth.token.accessTokenSecret | sha256sum | b64enc }} +{{- end }} diff --git a/charts/crater/templates/quota-agent/service.yaml b/charts/crater/templates/quota-agent/service.yaml new file mode 100644 index 000000000..c8ae3bdd4 --- /dev/null +++ b/charts/crater/templates/quota-agent/service.yaml @@ -0,0 +1,18 @@ +{{- if .Values.quotaAgent.enabled -}} +apiVersion: v1 +kind: Service +metadata: + name: crater-quota-agent + namespace: {{ .Values.namespaces.job }} + labels: + app: crater-quota-agent +spec: + type: ClusterIP + selector: + app: crater-quota-agent + ports: + - name: http + port: 7320 + targetPort: http + protocol: TCP +{{- end }} diff --git a/charts/crater/values.yaml b/charts/crater/values.yaml index 8bfdcb43d..ca05a9317 100644 --- a/charts/crater/values.yaml +++ b/charts/crater/values.yaml @@ -153,6 +153,25 @@ storage: # -- PVC name (existing or created, also used in backendConfig) pvcName: &rwPvcName "crater-rw-storage" +# -- Dedicated CephFS quota agent. The existing PVC must be a static CephFS +# volume for the same subvolume as storage.pvcName, mounted with a CephX client +# whose MDS capability includes the path-scoped "p" flag. +quotaAgent: + # -- Create the internal quota-agent Deployment and Service + # The Deployment reuses images.storage and switches behavior with CRATER_STORAGE_MODE=quota-agent. + enabled: false + # -- Pre-created static CephFS PVC using the dedicated quota CephX identity + existingClaim: "" + # -- Number of quota-agent replicas + replicas: 1 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 512Mi + # -- Node selector for all Deployments # Prevents control components from being scheduled to GPU worker nodes nodeSelector: @@ -299,6 +318,21 @@ backendConfig: # -- Persistent volume claim and path prefix configurations (Required) # All PVC names and prefix paths must be specified storage: + quota: + # -- Enable CephFS usage and quota management. Keep disabled for NFS and other storage backends. + enabled: false + # -- auto prefers storage-server and uses a Rook Ceph toolbox only as a fallback. + provider: auto + # -- Empty uses http://webdav-service..svc:7320. + storageServerURL: "" + # -- Namespace containing Rook Ceph resources and the optional toolbox. + rookNamespace: rook-ceph + # -- Optional CephFS CSI driver override; empty derives .cephfs.csi.ceph.com. + cephFSCSIDriver: "" + # -- Label selector for the optional toolbox fallback Pod. + toolboxLabelSelector: app=rook-ceph-tools + # -- Fallback filesystem name when the PV does not expose volumeAttributes.fsName. + cephFSName: cephfs # -- Path prefixes for different types of storage locations (Required) # All prefix paths must be specified prefix: diff --git a/docs/zh-CN/README.md b/docs/zh-CN/README.md index 2f44087e2..10d99c292 100644 --- a/docs/zh-CN/README.md +++ b/docs/zh-CN/README.md @@ -200,6 +200,7 @@ helm install crater oci://ghcr.io/raids-lab/crater --version - 📗 管理员指南（中文）: https://raids-lab.github.io/crater/zh/docs/admin/ - 📘 管理员指南（English）: https://raids-lab.github.io/crater/en/docs/admin/ +- 📙 [CephFS 配额管理与 quota-agent 设计说明](cephfs-quota-agent.md)：PR 架构、部署、真实 CephFS 测试与排障 ## 📁 仓库结构 diff --git a/docs/zh-CN/cephfs-quota-agent.md b/docs/zh-CN/cephfs-quota-agent.md new file mode 100644 index 000000000..c6070bda1 --- /dev/null +++ b/docs/zh-CN/cephfs-quota-agent.md @@ -0,0 +1,689 @@ +# CephFS 配额管理与 quota-agent 设计说明 + +本文面向维护 Crater 存储功能的开发者、代码评审者和集群管理员，说明当前 PR 中 CephFS 配额管理的需求、架构、数据流、配置、部署、测试、安全边界和回滚方法。 + +面向平台管理员的部署操作手册见 [CephFS 配额管理](../../website/content/docs/admin/more/db.mdx)。本文是实现与评审的权威说明，不替代面向用户的文档站内容。 + +## 1. PR 概览 + +### 1.1 要解决的问题 + +Crater 原有的 WebDAV/storage-server Pod 可以挂载 CephFS 并读取文件，但普通 Ceph CSI 客户端通常没有 MDS `p` 权限，因此会出现以下情况： + +- `ceph.dir.rbytes` 可以读取，目录用量能够显示。 +- 写入 `ceph.quota.max_bytes` 返回 `permission denied`，即使容器内进程是 `root` 也无法绕过 CephX 权限。 +- Rook toolbox 可以作为管理入口，但不是所有集群都部署 toolbox，不能把它作为唯一依赖。 +- 直接给通用 CSI 客户端增加 `p` 权限会扩大所有使用该身份的工作负载权限，风险和影响范围过大。 +- 自动扩缩容、AI 建议、目录对比等入口使管理员工作流变复杂，且不属于当前最小可用的配额管理闭环。 + +本 PR 引入独立的 `quota-agent` 运行模式，以路径受限的 CephX 身份挂载现有 CephFS 子卷，为后端提供内部目录用量与配额接口。管理员通过显式刷新获取用量，通过单次操作修改配额，并在操作日志中查看配额修改记录。 + +### 1.2 当前交付范围 + +当前对外开放并承诺的功能包括： + +- 检测当前共享 PVC 是否为受支持的 CephFS CSI 存储。 +- 按能力决定前端是否展示存储管理入口、用量和配额修改控件。 +- 读取用户、公共和账户目录的 `ceph.dir.rbytes`。 +- 管理员手动刷新全部用户目录用量，并把结果缓存到数据库。 +- 读取和修改用户目录的 `ceph.quota.max_bytes`。 +- 使用 `-1` 表示 Crater 侧的无限制配额，写入 CephFS 时转换为 `0`。 +- 记录成功和失败的配额修改操作。 +- 在创建作业前使用最近一次缓存用量检查用户是否已达到理论配额。 +- 在没有 Rook toolbox 的集群中，通过 quota-agent 完成真实 CephFS 配额管理。 + +当前产品界面和已注册 API 不提供以下功能： + +- AI 配额建议。 +- 自动扩容或自动缩容入口。 +- 定时扫描全部用户目录。 +- 目录对比。 +- Storage Index 管理界面。 + +仓库中可能仍存在早期存储治理实验代码或迁移兼容字段，它们不属于本次对外开放的配额管理契约。评审当前功能时，应以本文列出的路由、配置和界面行为为准。 + +### 1.3 主要改动模块 + +| 模块 | 作用 | 主要路径 | +| --- | --- | --- | +| 后端能力与管理 API | 检测存储能力、刷新用量、修改配额、记录日志 | `backend/internal/handler/storage.go` | +| CephFS Provider | 在 storage-server 与 toolbox 之间选择实现 | `backend/pkg/ceph/` | +| 内部客户端 | 调用 quota-agent 内部 API 并完成令牌认证 | `backend/pkg/storagequota/` | +| quota-agent | 校验路径并直接读写 CephFS xattr | `backend/internal/storage/quota*.go` | +| 用量缓存与准入 | 保存刷新结果，在创建作业前检查配额 | `backend/pkg/patrol/patrol.go`、`backend/internal/util/quota.go` | +| 数据模型与迁移 | 用户配额字段、用量缓存表及兼容迁移 | `backend/dao/model/`、`backend/cmd/gorm-gen/models/migrate.go` | +| 前端 | 能力驱动的管理员存储页、文件页用量、配额修改记录 | `frontend/src/routes/admin/storage/`、`frontend/src/components/file/` | +| Helm | 部署 quota-agent、内部 Service 和认证 Secret | `charts/crater/templates/quota-agent/` | +| 运维脚本 | 初始化专用 CephX/PV/PVC，本地无镜像测试 | `backend/hack/` | + +## 2. 用户可见行为 + +### 2.1 管理员存储管理页 + +后端能力探测成功后，管理员可以： + +1. 查看所有用户及其最近一次缓存的目录用量。 +2. 查看每条用量的刷新时间。 +3. 点击“刷新用量”，依次读取全部用户目录并更新缓存。 +4. 设置一个大于 `0` 的字节数作为配额，或设置为无限制。 +5. 查看“配额修改记录”，确认操作人、目标用户、旧值、新值、Provider、结果和错误信息。 + +首次刷新前，用户仍会正常列出，用量显示为未知或等待刷新。刷新允许部分成功，接口返回成功数、失败数和完成时间，单个目录失败不会清空该用户此前的缓存值。 + +### 2.2 普通用户文件页 + +当后端返回 `usage_readable=true` 时，文件系统页面可以读取对应目录的实时 CephFS 用量。没有为目录设置配额时，`ceph.dir.rbytes` 仍然可读，因此“显示用量”和“设置配额”是两项独立能力。 + +当能力不可用时，文件浏览、上传和下载仍按原有 storage-server/WebDAV 流程工作，只隐藏 CephFS 用量或配额相关信息。 + +### 2.3 配额达到或低于现有用量 + +CephFS 设置配额不会删除已有文件。如果管理员把配额设置为小于或等于当前用量： + +- 现有数据仍保留并可读取。 +- 目录后续写入通常会失败，应用可能收到 `ENOSPC` 或空间不足错误。 +- Crater 使用最近一次缓存用量检查新作业创建；缓存未刷新时，准入判断可能暂时落后于真实用量。 + +生产环境修改配额前，应先刷新用量，并给业务保留合理余量。 + +## 3. 总体架构 + +```mermaid +flowchart LR + UI[Crater 前端] -->|Bearer API| Backend[Crater backend] + Backend -->|内部令牌| Agent[quota-agent] + Backend -. auto 回退 .-> Toolbox[Rook toolbox] + Agent -->|专用 PVC 与 CephX| CephFS[(同一 CephFS 子卷)] + Toolbox -. 可选 .-> CephFS + Backend --> DB[(PostgreSQL)] + WebDAV[storage-server / WebDAV] -->|普通 CSI 身份| CephFS +``` + +quota-agent 复用 storage-server 镜像，但通过环境变量切换为受限模式： + +```text +CRATER_STORAGE_MODE=quota-agent +``` + +该模式只注册 `/internal/storage/*`，不提供 WebDAV、文件下载、数据集管理或用户登录接口。它不需要数据库、LDAP、镜像仓库配置或完整 backend 配置。 + +### 3.1 为什么不直接修改普通 CSI 客户端 + +CephFS 的 `p` capability 允许设置布局和配额等扩展属性。给 `client.csi-cephfs-node` 等通用身份增加 `p`，会让所有复用该身份并挂载相关文件系统的 Pod 获得更高权限。 + +quota-agent 使用单独的 CephClient，并把 MDS capability 限制在 Crater 已使用的 `subvolumePath`。这样可以把权限控制在一个组件、一份专用 PVC 和一个 CephFS 子卷内，不改变 WebDAV 或其他业务 Pod 的身份。 + +### 3.2 为什么容器内 sudo 无法解决 + +Linux `root` 只决定容器或节点上的本地进程权限。CephFS 服务端还会根据挂载使用的 CephX 客户端 capability 授权。客户端没有 `p` 时，`sudo setfattr`、以 UID 0 运行 Pod 或给容器增加普通 Linux capability 都不能获得 CephFS 配额写权限。 + +## 4. CephFS 数据语义 + +本功能使用两个 CephFS 扩展属性： + +| xattr | 含义 | 是否要求预先设置配额 | +| --- | --- | --- | +| `ceph.dir.rbytes` | 目录树当前占用字节数 | 否 | +| `ceph.quota.max_bytes` | 目录树最大可用字节数 | 是，仅在配置配额后存在有效限制 | + +Crater API 使用以下约定： + +- 正整数：具体的字节配额。 +- `-1`：无限制。 +- `0`：不接受为外部 API 输入；quota-agent 把 `-1` 转换成 CephFS 的 `0` 以取消限制。 + +示例：4 TiB 的准确字节数为 `4398046511104`。 + +```json +{ + "quota": 4398046511104 +} +``` + +## 5. Provider 与能力探测 + +### 5.1 Provider 选择 + +| Provider | 行为 | 适用场景 | +| --- | --- | --- | +| `storageServer` | 只调用 quota-agent 或兼容的 storage-server 内部接口 | 推荐的生产配置，不依赖 toolbox | +| `toolbox` | 只通过 Rook toolbox 操作 CephFS | 已有 toolbox 的兼容方案 | +| `auto` | 优先调用内部存储服务，能力不足或调用失败时回退 toolbox | 迁移期或需要回退能力的集群 | +| `disabled` | 禁用 CephFS 用量和配额管理 | NFS、其他 CSI 或不需要该功能的集群 | + +启用 Helm `quotaAgent.enabled` 后，Chart 会把 backend Provider 固定为 `storageServer`，避免生产链路意外依赖 toolbox。 + +### 5.2 后端探测顺序 + +后端不会仅根据配置开关展示页面，而是依次验证： + +1. `storage.quota.enabled` 已开启。 +2. `storage.pvc.readWriteMany` 已配置。 +3. PVC 位于 `namespaces.job` 且已经绑定 PV。 +4. PV 的 CSI Driver 与 `cephFSCSIDriver` 一致。 +5. 当前 Provider 可以读取目录用量和配额，并在临时目录完成配额写入探测。 +6. `auto` 模式下，内部服务能力不足时再探测 toolbox。 + +能力响应中的关键字段为： + +| 字段 | 含义 | +| --- | --- | +| `quota_enabled` | 配置总开关是否开启 | +| `backend` | 检测到的存储后端，支持时为 `cephfs` | +| `quota_provider` | 当前 Provider | +| `storage_server_available` | quota-agent/storage-server 内部接口是否可访问 | +| `toolbox_available` | toolbox 回退是否可用 | +| `usage_readable` | 是否可读取目录用量 | +| `quota_readable` | 是否可读取目录配额 | +| `quota_writable` | 是否可修改目录配额 | +| `reasons` | 能力不可用或降级的具体原因 | + +前端展示规则： + +- 管理员存储入口要求 `quota_enabled=true` 且 `usage_readable=true`。 +- 配额修改控件还要求 `quota_writable=true`。 +- 普通文件页仅在 `usage_readable=true` 时请求目录用量。 + +## 6. 请求与数据流 + +### 6.1 手动刷新用量 + +管理员点击刷新后，backend 会按用户依次执行： + +1. 根据用户记录和 `storage.prefix.user` 计算相对目录。 +2. 通过当前 Provider 读取 `ceph.dir.rbytes`。 +3. 把成功结果写入 `user_space_sizes`。 +4. 保留失败用户的旧缓存并累计失败数。 +5. 返回 `updated`、`failed` 和 `refreshed_at`。 + +当前没有 30 分钟定时扫描任务。顺序读取可以控制对 MDS 的瞬时压力，代价是用户很多时一次刷新会持续更久。前端在刷新期间禁用重复提交，并展示刷新完成时间。 + +### 6.2 修改配额 + +管理员提交新配额后，backend 按以下顺序处理： + +1. 校验用户名和配额值。 +2. 查找用户与当前数据库配额。 +3. 先通过 Provider 写入 CephFS `ceph.quota.max_bytes`。 +4. CephFS 成功后再更新 `users.space_quota`。 +5. 数据库更新失败时，尝试把 CephFS 回滚到旧配额。 +6. 记录 `SetStorageQuota` 成功或失败操作日志。 + +先写 CephFS 可以避免数据库展示一个实际未生效的配额。手工修改会清理旧的临时扩容状态字段，因为管理员设置值应成为新的明确基线。 + +### 6.3 作业创建检查 + +创建 Jupyter、WebIDE、PyTorch、TensorFlow、Volcano、AIJob 等作业前，backend 会读取用户配额和 `user_space_sizes` 最近一次缓存： + +- 用户被冻结时拒绝创建新作业。 +- 配额为 `-1` 或非正值时跳过容量检查。 +- 缓存用量大于或等于理论配额时拒绝创建新作业。 +- 用户、数据库或缓存记录不可用时采用 fail-open，不阻断现有作业流程。 + +该检查是平台侧的提前保护，不替代 CephFS 自身的强制配额。CephFS xattr 才是最终写入限制。 + +## 7. API 契约 + +### 7.1 对前端开放的 API + +| 方法 | 路径 | 权限 | 用途 | +| --- | --- | --- | --- | +| `GET` | `/api/v1/storage/capabilities` | 登录用户 | 查询能力 | +| `GET` | `/api/v1/storage/dirsize/*path` | 登录用户 | 查询可访问目录用量 | +| `GET` | `/api/v1/storage/my-quota` | 登录用户 | 查询本人数据库配额 | +| `GET` | `/api/v1/admin/storage/capabilities` | 管理员 | 查询管理员存储能力 | +| `GET` | `/api/v1/admin/storage/user-spaces` | 管理员 | 分页读取用户、缓存用量和配额 | +| `POST` | `/api/v1/admin/storage/user-spaces/refresh` | 管理员 | 手动刷新全部用户用量 | +| `PUT` | `/api/v1/admin/storage/user-spaces/{user}/quota` | 管理员 | 设置或取消用户配额 | + +设置配额请求： + +```json +{ + "quota": 4398046511104 +} +``` + +取消配额请求： + +```json +{ + "quota": -1 +} +``` + +### 7.2 quota-agent 内部 API + +| 方法 | 路径 | 用途 | +| --- | --- | --- | +| `GET` | `/internal/storage/capabilities` | 探测 xattr 读写能力 | +| `GET` | `/internal/storage/usage?path=` | 读取目录用量 | +| `GET` | `/internal/storage/quota?path=` | 读取目录配额 | +| `PUT` | `/internal/storage/quota` | 修改目录配额 | + +内部接口使用 `X-Crater-Internal-Token`，不应通过 Ingress 暴露。路径必须是存储根目录下的相对目录；实现会拒绝绝对路径、`..` 穿越、普通文件以及解析后逃离存储根目录的符号链接。 + +## 8. 数据库模型 + +### 8.1 用户配额字段 + +`users` 表中的相关字段包括： + +| 字段 | 作用 | +| --- | --- | +| `space_quota` | 当前管理员配置的字节配额，`-1` 表示无限制 | +| `original_space_quota` | 兼容早期临时扩容流程的原始配额 | +| `jobs_frozen` | 是否禁止该用户创建新作业 | +| `shrink_stage` | 兼容早期缩容流程的阶段 | +| `shrink_stage_updated_at` | 兼容早期缩容流程的更新时间 | + +当前界面手动修改配额时，会清理临时扩容和缩容状态。 + +### 8.2 用量缓存表 + +`user_space_sizes` 保存每个用户最近一次成功读取的目录用量及更新时间。它用于管理员列表展示和作业创建前的快速检查，不是实时计量账本。 + +数据库迁移必须同时覆盖： + +- 已有安装升级时创建字段和缓存表。 +- 新安装通过初始化 Schema 获得同样结构。 +- GORM 生成模型与手写模型字段保持一致。 + +## 9. 生产部署 + +### 9.1 前提条件 + +- Crater 共享 PVC 已经处于 `Bound`。 +- 共享 PVC 由 Rook CephFS CSI 提供，不是 NFS、RBD 或本地盘。 +- 集群安装了 Rook Operator 和 `CephClient` CRD。 +- 管理员可以创建 CephClient、Secret、PV 和 PVC。 +- backend、storage-server 镜像和 Helm Chart 来自同一版本。 + +检查现有 PVC： + +```bash +kubectl -n crater-workspace get pvc crater-rw-storage + +SOURCE_PV=$(kubectl -n crater-workspace get pvc crater-rw-storage \ + -o jsonpath='{.spec.volumeName}') + +kubectl get pv "$SOURCE_PV" \ + -o jsonpath='{.spec.csi.driver}{"\n"}{.spec.csi.volumeAttributes.fsName}{"\n"}{.spec.csi.volumeAttributes.subvolumePath}{"\n"}' +``` + +输出应包含 CephFS CSI Driver、`fsName` 和非空 `subvolumePath`。 + +### 9.2 创建专用 CephFS 身份和 PVC + +从与部署版本匹配的源码执行： + +```bash +APP_NAMESPACE=crater-workspace \ +ROOK_NAMESPACE=rook-ceph \ +SOURCE_PVC=crater-rw-storage \ +bash backend/hack/bootstrap-cephfs-quota-agent.sh +``` + +默认创建： + +| 资源 | 名称 | +| --- | --- | +| CephClient | `rook-ceph/crater-quota` | +| CSI Secret | `rook-ceph/crater-quota-csi` | +| 静态 PV | `crater-quota-storage-pv` | +| quota-agent PVC | `crater-workspace/crater-quota-storage` | + +专用静态 PV 与原 PVC 指向同一个 CephFS 子卷，不会复制已有文件。脚本会把原 PV 回收策略调整为 `Retain`，避免两个 PV 引用同一子卷时删除原 PVC 触发底层子卷回收。执行前应确认这一生命周期策略符合本集群要求。 + +常用覆盖变量： + +| 变量 | 默认值 | 作用 | +| --- | --- | --- | +| `APP_NAMESPACE` | `crater-workspace` | Crater 作业与存储命名空间 | +| `ROOK_NAMESPACE` | `rook-ceph` | Rook 资源命名空间 | +| `SOURCE_PVC` | `crater-rw-storage` | 现有 CephFS 共享 PVC | +| `QUOTA_CLIENT` | `crater-quota` | 专用 CephClient 名称 | +| `QUOTA_PV` | `crater-quota-storage-pv` | 静态 PV 名称 | +| `QUOTA_PVC` | `crater-quota-storage` | 专用 PVC 名称 | +| `CSI_SECRET` | `crater-quota-csi` | CSI Secret 名称 | + +### 9.3 启用 Helm 组件 + +在集群 values 中添加： + +```yaml +quotaAgent: + enabled: true + existingClaim: crater-quota-storage + +backendConfig: + storage: + quota: + rookNamespace: rook-ceph + cephFSCSIDriver: rook-ceph.cephfs.csi.ceph.com + cephFSName: cephfs +``` + +`quotaAgent.enabled=true` 时，Chart 自动生成以下 backend 配置： + +```yaml +backendConfig: + storage: + quota: + enabled: true + provider: storageServer + storageServerURL: http://crater-quota-agent.crater-workspace.svc:7320 +``` + +quota-agent 复用 `images.storage`，默认不需要新增镜像仓库配置。Chart 默认关闭该组件，因此不使用 CephFS 的集群升级后不会受影响。 + +文档示例使用 Chart 版本占位符，发布时需要存在与 `charts/crater/Chart.yaml` 一致的 Git tag： + +```bash +helm upgrade crater oci://ghcr.io/raids-lab/crater \ + --version \ + --values values.yaml +``` + +### 9.4 验证部署 + +```bash +kubectl -n crater-workspace get pvc crater-quota-storage +kubectl -n crater-workspace rollout status deployment/crater-quota-agent +kubectl -n crater-workspace logs deployment/crater-quota-agent +``` + +启动日志应包含 `mode=quota-agent`。随后登录管理员页面，确认能力接口返回： + +```text +usage_readable=true +quota_readable=true +quota_writable=true +``` + +## 10. 不推送镜像的真实 CephFS 开发测试 + +本地测试不要求先合并 PR、推送代码或构建新镜像。开发脚本会把当前源码交叉编译成 Linux 二进制，复制到挂载专用 PVC 的临时 Pod 中运行。 + +### 10.1 准备专用 PVC + +首次测试先执行生产部署中的初始化脚本： + +```bash +KUBECONFIG=backend/kubeconfig \ +bash backend/hack/bootstrap-cephfs-quota-agent.sh +``` + +### 10.2 启动临时 quota-agent + +```bash +KUBECONFIG=backend/kubeconfig \ +bash backend/hack/run-cephfs-quota-agent-dev.sh +``` + +脚本会： + +1. 使用仓库内 `backend/.gocache` 编译 Linux/amd64 storage-server。 +2. 创建只包含内部认证令牌的 Secret。 +3. 创建 `crater-quota-agent-dev` Pod 并挂载专用 PVC。 +4. 把本地二进制复制到 Pod 中，以 quota-agent 模式启动。 + +集群已有 storage-server 镜像只提供运行环境，实际执行的是刚编译的本地二进制。 + +### 10.3 转发端口 + +另开终端执行： + +```bash +kubectl --kubeconfig backend/kubeconfig \ + -n crater-workspace port-forward pod/crater-quota-agent-dev 7330:7320 +``` + +使用 quota-agent 开发 Pod 时，不需要再转发 `webdav-service:7320`。数据链路为“本地 backend -> `127.0.0.1:7330` -> 集群 quota-agent -> 真实 CephFS”。 + +### 10.4 配置并启动本地 backend + +本地调试配置示例： + +```yaml +storage: + quota: + enabled: true + provider: storageServer + storageServerURL: http://127.0.0.1:7330 + rookNamespace: rook-ceph + cephFSCSIDriver: rook-ceph.cephfs.csi.ceph.com +``` + +同时启动本地 backend 和 frontend，登录管理员页面完成以下验证： + +1. 存储管理入口可见。 +2. 点击刷新后可以看到真实 CephFS 用户目录用量和刷新时间。 +3. 给测试目录设置高于当前用量的临时配额。 +4. 重新打开页面，确认配额可以读回。 +5. 把配额恢复为原值或无限制。 +6. 在配额修改记录中确认成功和失败操作均可追踪。 + +### 10.5 清理临时 Pod + +```bash +KUBECONFIG=backend/kubeconfig \ +bash backend/hack/run-cephfs-quota-agent-dev.sh --cleanup +``` + +该命令只清理临时 Pod 和开发认证 Secret。专用 CephClient、PV 和 PVC 会保留，供后续测试或正式部署复用。 + +## 11. 安全设计 + +### 11.1 CephX 最小权限 + +- 不修改通用 CSI 客户端 capability。 +- 专用 CephClient 的 MDS `p` 权限限制到目标 `fsName` 和 `subvolumePath`。 +- quota-agent 只挂载 Crater 已使用的 CephFS 子卷。 + +### 11.2 服务最小暴露面 + +- quota-agent Service 为集群内 Service，不配置 Ingress。 +- Pod 设置 `automountServiceAccountToken: false`，不需要 Kubernetes API 权限。 +- quota-agent 模式不注册 WebDAV 与其他 storage-server 路由。 +- 内部令牌由 backend 登录密钥通过 SHA-256 域分离派生，Chart 只把派生令牌注入 quota-agent。 +- quota-agent 不挂载完整 backend 配置，因此无法获得数据库、LDAP 或镜像仓库凭据。 + +### 11.3 路径约束 + +内部 API 只接受相对于存储根目录的目录路径。实现会进行清理、绝对路径比较和符号链接解析，阻止路径穿越或访问挂载点外的文件。 + +## 12. 性能与运维 + +读取单个顶层目录的 `ceph.dir.rbytes` 是 MDS 元数据查询，不会像 `du` 一样遍历目录树中的每个文件。对于几十或几百个用户目录，管理员偶尔顺序刷新通常是可接受的。 + +当前设计刻意不运行 30 分钟定时任务： + +- 避免所有集群固定产生无意义的周期负载。 +- 管理员可以在配额调整前按需获取新数据。 +- 页面显示更新时间，明确缓存新鲜度。 +- 顺序执行避免大量并发 xattr 请求造成瞬时压力。 + +用户量很大时，应观察 MDS 延迟和刷新耗时，再决定是否增加批次、限速或后台任务，而不是直接提高并发。 + +## 13. 兼容性矩阵 + +| 存储/集群条件 | 用量读取 | 配额修改 | 推荐配置 | +| --- | --- | --- | --- | +| Rook CephFS，有 quota-agent | 支持 | 支持 | `quotaAgent.enabled=true` | +| Rook CephFS，无 toolbox | 支持 | 支持 | quota-agent，不需要 toolbox | +| Rook CephFS，只有 toolbox | 支持 | 取决于 toolbox 权限 | `provider: toolbox` 或迁移期 `auto` | +| Rook CephFS，普通 WebDAV CSI 无 `p` | 通常支持 | 不支持 | 部署 quota-agent | +| NFS | 不支持 CephFS xattr 用量 | 不支持 | 保持功能关闭 | +| RBD、块存储、本地盘或其他 CSI | 不支持 | 不支持 | 保持功能关闭 | + +对 NFS 和其他不支持的存储，使用默认值即可： + +```yaml +quotaAgent: + enabled: false + +backendConfig: + storage: + quota: + enabled: false +``` + +文件浏览和传输功能不受影响，前端不会展示 CephFS 配额管理入口。 + +## 14. 常见问题 + +### 14.1 `write ceph.quota.max_bytes: permission denied` + +这表示执行写入的 CephX 客户端没有目标路径的 MDS `p` 权限，不是 Go 代码、Linux 用户或 `sudo` 本身的问题。 + +确认操作经过 quota-agent，并检查： + +- Pod 挂载的是 `crater-quota-storage`，不是普通 WebDAV PVC。 +- `CephClient/crater-quota` 已生成对应 Secret。 +- MDS caps 中的 `fsName`、`subvolumePath` 和实际 PV 一致。 +- Pod 已在权限变更后重新创建并重新挂载。 + +### 14.2 quota-agent 已运行，但前端没有入口 + +依次检查： + +1. `quotaAgent.enabled` 是否为 `true`。 +2. backend 实际加载的 `storage.quota.enabled` 和 Provider。 +3. `storage.pvc.readWriteMany` 是否指向正确 PVC。 +4. PVC 是否已绑定 CephFS PV，CSI Driver 是否与配置一致。 +5. backend 是否能访问 quota-agent Service。 +6. 能力响应 `reasons` 中的具体失败信息。 + +仅看到 Pod 为 Running 不代表 xattr 权限探测已经成功。 + +### 14.3 quota-agent PVC 一直 Pending + +```bash +kubectl -n crater-workspace describe pvc crater-quota-storage +kubectl describe pv crater-quota-storage-pv +``` + +检查 PV/PVC 的 `storageClassName`、容量、访问模式、`volumeName` 和 Secret 引用是否一致。 + +### 14.4 `pods "crater-quota-agent-dev" not found` + +临时 Pod 不会永久存在。重新运行开发脚本，再确认命名空间： + +```bash +KUBECONFIG=backend/kubeconfig \ +bash backend/hack/run-cephfs-quota-agent-dev.sh + +kubectl --kubeconfig backend/kubeconfig \ + -n crater-workspace get pod crater-quota-agent-dev +``` + +### 14.5 本地端口 7330 被占用 + +关闭旧的 port-forward 进程，或使用其他本地端口，例如： + +```bash +kubectl --kubeconfig backend/kubeconfig \ + -n crater-workspace port-forward pod/crater-quota-agent-dev 7331:7320 +``` + +随后把本地配置 `storageServerURL` 改为 `http://127.0.0.1:7331`。 + +### 14.6 文件页和管理员页显示的用量不同 + +管理员列表展示 `user_space_sizes` 中最近一次手动刷新缓存；文件页可能读取当前目录的实时值。还应确认两处使用的目录前缀和用户 `space` 字段是否相同。 + +先点击管理员页“刷新用量”，比较更新时间和实际路径。如果仍不一致，再检查 `storage.prefix.user`、用户空间目录名和能力接口 Provider。 + +### 14.7 集群没有 toolbox + +不影响推荐方案。先运行初始化脚本创建专用 CephClient/PVC，再启用 quota-agent，并使用 `provider: storageServer`。toolbox 只作为可选兼容或回退手段。 + +## 15. 停用与回滚 + +仅停用功能时，把以下值恢复为 `false` 并执行 Helm upgrade： + +```yaml +quotaAgent: + enabled: false + +backendConfig: + storage: + quota: + enabled: false +``` + +停止 quota-agent 不会自动删除文件，也不会自动移除已经写入 CephFS 的目录配额。需要彻底取消限制时，应先通过页面把相关用户配额设置为无限制。 + +永久清理专用资源时： + +1. 停用并确认 quota-agent Deployment 已删除。 +2. 确认没有 Pod 使用 `crater-quota-storage`。 +3. 删除专用 PVC 和静态 PV。 +4. 最后删除专用 CSI Secret 与 CephClient。 + +不要删除 Crater 原始共享 PVC，也不要手工删除底层 CephFS 子卷。原始 PV 和专用静态 PV 可能都使用 `Retain`，清理时必须根据集群数据生命周期策略单独处理。 + +## 16. 开发验证与评审清单 + +### 16.1 自动检查 + +涉及本功能的变更至少应执行： + +```bash +cd backend +make pre-commit-check + +cd ../frontend +make pre-commit-check + +cd ../website +make pre-commit-check +``` + +若完整检查耗时过长，至少运行受影响包测试、backend 与 storage-server 构建、frontend 构建、网站构建、Helm lint/render 和 `git diff --check`，并在 PR 中明确记录未运行的项目。 + +Windows 上直接执行 Go 构建时，应按仓库开发约定使用仓库内缓存，避免在工作区留下编译产物： + +```powershell +$env:GOCACHE = (Resolve-Path backend/.gocache) +Set-Location backend +go build ./cmd/crater +go build ./cmd/storage-server +``` + +### 16.2 人工检查 + +提交 PR 前，开发者应亲自完成并记录： + +- 阅读本文与网站管理员文档，核对术语、命令、链接和版本占位符。 +- 在真实 CephFS 测试目录执行一次用量刷新、设置配额、读回配额和恢复配额。 +- 验证没有 toolbox 时 quota-agent 链路仍然工作。 +- 验证功能关闭或 NFS 场景不展示存储管理入口，文件浏览仍可使用。 +- 验证失败的配额写入不会把数据库配额误改为已生效值。 +- 验证配额修改记录包含成功和失败结果。 +- 为管理员存储页面和关键状态准备 PR 截图。 + +### 16.3 评审重点 + +- **权限范围**：CephClient capability 是否限制到正确的 `fsName` 和 `subvolumePath`。 +- **数据安全**：原 PV 的 `Retain` 调整和静态 PV 清理步骤是否符合目标集群策略。 +- **一致性**：CephFS 写入、数据库更新、失败回滚与操作日志顺序是否保持一致。 +- **能力降级**：服务不可用、驱动不匹配、功能关闭时是否隐藏入口且不影响文件功能。 +- **路径安全**：内部 API 是否继续拒绝路径穿越、文件路径和挂载点外符号链接。 +- **兼容性**：无 toolbox、NFS 和旧数据库升级场景是否有明确结果。 +- **缓存语义**：界面和作业准入是否清楚区分实时 CephFS 限制与最近一次用量缓存。 + +## 17. 上线验收标准 + +满足以下条件后，才认为完整配额管理功能可上线： + +- quota-agent 使用专用 CephClient 和专用 PVC，未扩大通用 CSI 客户端权限。 +- backend 能力接口稳定返回三个可用能力。 +- 管理员可刷新真实用量并看到更新时间。 +- 管理员可设置、读回和取消测试用户配额。 +- 配额低于用量时已有数据不丢失，新增写入按 CephFS 预期失败。 +- 配额修改成功和失败均有审计记录。 +- 无 toolbox 环境验证通过。 +- 功能关闭的非 CephFS 集群不受影响。 +- 自动检查、真实 CephFS 人工测试和文档人工阅读结果已写入 PR。 diff --git a/frontend/src/components/file/file-select-dialog.tsx b/frontend/src/components/file/file-select-dialog.tsx index 100d4cc2e..19009fae0 100644 --- a/frontend/src/components/file/file-select-dialog.tsx +++ b/frontend/src/components/file/file-select-dialog.tsx @@ -41,6 +41,7 @@ export const FileSelectDialog = ({ disabled, allowSelectFile = true, isrw = false, + isadmin = false, title, }: { value?: string @@ -48,6 +49,7 @@ export const FileSelectDialog = ({ disabled?: boolean allowSelectFile?: boolean isrw?: boolean + isadmin?: boolean title?: string }) => { const { t } = useTranslation() @@ -99,6 +101,7 @@ export const FileSelectDialog = ({ { setContent(item) }} diff --git a/frontend/src/components/file/folder-navigation.tsx b/frontend/src/components/file/folder-navigation.tsx index 9ca7c92e8..24cac3954 100644 --- a/frontend/src/components/file/folder-navigation.tsx +++ b/frontend/src/components/file/folder-navigation.tsx @@ -19,14 +19,22 @@ import { useLocation, useNavigate } from '@tanstack/react-router' import { useAtomValue } from 'jotai' import { ArrowRight, Folder, HardDrive, UserRound, UsersRound } from 'lucide-react' import { motion } from 'motion/react' -import { useMemo } from 'react' +import { useEffect, useMemo, useState } from 'react' import { useTranslation } from 'react-i18next' import { getFolderTitle } from '@/components/file/lazy-file-tree' import PageTitle from '@/components/layout/page-title' import { AccessMode, IUserContext } from '@/services/api/auth' -import { FileItem } from '@/services/api/file' +import { + DirectorySize, + FileItem, + MyQuota, + StorageCapabilities, + apiGetDirectorySize, + apiGetMyQuota, + apiGetStorageCapabilities, +} from '@/services/api/file' import { atomUserContext } from '@/utils/store' @@ -72,8 +80,81 @@ export default function FolderNavigation({ const { pathname } = useLocation() const navigate = useNavigate() const context = useAtomValue(atomUserContext) + const [userSpaceSize, setUserSpaceSize] = useState(null) + const [publicSpaceSize, setPublicSpaceSize] = useState(null) + const [accountSpaceSize, setAccountSpaceSize] = useState(null) + const [myQuota, setMyQuota] = useState(null) + const [storageCapabilities, setStorageCapabilities] = useState(null) + const [isLoading, setIsLoading] = useState(false) - // 对文件夹进行排序，公共 -> 账户 -> 用户 + // Load storage usage and the current user's quota. + useEffect(() => { + const fetchSpaceSizes = async () => { + setIsLoading(true) + try { + const capability = await apiGetStorageCapabilities().then((r) => r.data) + setStorageCapabilities(capability ?? null) + if (!capability?.usage_readable) { + setUserSpaceSize(null) + setPublicSpaceSize(null) + setAccountSpaceSize(null) + setMyQuota(null) + return + } + + const promises: Promise[] = [] + + if (context?.space) { + promises.push( + apiGetDirectorySize(`user/${context.space}`) + .then((r) => { + if (r.data) setUserSpaceSize(r.data) + }) + .catch(() => { + setUserSpaceSize(null) + }) + ) + } + + promises.push( + apiGetDirectorySize('public') + .then((r) => { + if (r.data) setPublicSpaceSize(r.data) + }) + .catch(() => { + setPublicSpaceSize(null) + }), + apiGetDirectorySize('account') + .then((r) => { + if (r.data) setAccountSpaceSize(r.data) + }) + .catch(() => { + setAccountSpaceSize(null) + }), + apiGetMyQuota() + .then((r) => { + if (r.data) setMyQuota(r.data) + }) + .catch(() => { + setMyQuota(null) + }) + ) + + await Promise.all(promises) + } catch { + setUserSpaceSize(null) + setPublicSpaceSize(null) + setAccountSpaceSize(null) + setMyQuota(null) + } finally { + setIsLoading(false) + } + } + + fetchSpaceSizes() + }, [context?.space]) + + // Keep the public, account, and user folders in a predictable order. const sortFolders = (folders: FileItem[]) => { return folders.sort((a, b) => { if (isPublicFolder(a.name)) { @@ -124,10 +205,59 @@ export default function FolderNavigation({ } const getBadgeText = (folder: string, mode: AccessMode) => { - if (isUserFolder(folder)) return t('folderNavigation.badge.private', '私有') - if (mode === AccessMode.ReadOnly) return t('folderNavigation.badge.readOnly', '只读') - if (mode === AccessMode.ReadWrite) return t('folderNavigation.badge.readWrite', '读写') - return t('folderNavigation.badge.noAccess', '无权限') + if (isUserFolder(folder)) return t('folderNavigation.badge.private') + if (mode === AccessMode.ReadOnly) return t('folderNavigation.badge.readOnly') + if (mode === AccessMode.ReadWrite) return t('folderNavigation.badge.readWrite') + return t('folderNavigation.badge.noAccess') + } + + // Format byte counts using the most readable binary unit. + const formatFileSize = (bytes: number): { size: string; unit: string } => { + if (!Number.isFinite(bytes) || bytes <= 0) return { size: '0', unit: 'B' } + const units = ['B', 'KB', 'MB', 'GB', 'TB'] + const k = 1024 + const i = Math.max(0, Math.min(units.length - 1, Math.floor(Math.log(bytes) / Math.log(k)))) + return { + size: (bytes / Math.pow(k, i)).toFixed(2), + unit: units[i], + } + } + + // quota=-1 is unlimited; null means the shared space has no independent quota. + const getSpaceMetrics = ( + spaceType: string + ): { + size: number | null + quota: number | null + sizeUnit?: string + quotaUnit?: string + formattedSize?: string + } => { + if (spaceType === 'user') { + return { + size: userSpaceSize?.size ?? null, + quota: myQuota?.space_quota ?? null, + sizeUnit: userSpaceSize?.unit, + formattedSize: userSpaceSize?.formatted, + } + } + if (spaceType === 'public') { + return { + size: publicSpaceSize?.size ?? null, + quota: null, + sizeUnit: publicSpaceSize?.unit, + formattedSize: publicSpaceSize?.formatted, + } + } + if (spaceType === 'account') { + return { + size: accountSpaceSize?.size ?? null, + quota: null, + sizeUnit: accountSpaceSize?.unit, + formattedSize: accountSpaceSize?.formatted, + } + } + return { size: null, quota: null } } const handleTitleNavigation = (name: string) => { @@ -158,7 +288,7 @@ export default function FolderNavigation({ title={t('folderNavigation.pageTitle.title')} description={t('folderNavigation.pageTitle.description')} > - +
{/* Usage Metrics */} -
-
-
- ??? - GB -
- 总 ??? GB -
+ {storageCapabilities?.usage_readable && + (() => { + const { + size, + quota, + sizeUnit, + formattedSize: apiFormattedSize, + } = getSpaceMetrics(type) + const displaySize = + typeof size === 'number' && Number.isFinite(size) ? Math.max(0, size) : null + const hasQuota = quota !== null && quota > 0 + const isUnlimited = quota === -1 + const usageRatio = + hasQuota && displaySize !== null + ? Math.min(100, (displaySize / quota!) * 100) + : null -
-
-
+ // Prefer the server-formatted size and fall back to local formatting. + let formattedSize: { size: string; unit: string } | null = null + if (apiFormattedSize) { + // Split the server-formatted value for the existing visual treatment. + const match = apiFormattedSize.match(/([\d.]+)\s*(\w+)/) + if (match) { + formattedSize = { size: match[1], unit: match[2] } + } + } else if (displaySize !== null) { + if (sizeUnit) { + formattedSize = { size: displaySize.toFixed(2), unit: sizeUnit } + } else { + formattedSize = formatFileSize(displaySize) + } + } -
- ??% 已使用 - {r.size} 个文件 -
-
+ let formattedQuota: { size: string; unit: string } | null = null + if (hasQuota && quota !== null) { + formattedQuota = formatFileSize(quota) + } + + return ( +
+
+
+ {isLoading ? ( + + {t('common.loading')} + + ) : formattedSize ? ( + <> + + {formattedSize.size} + + + {formattedSize.unit} + + + ) : ( + — + )} +
+ + {isUnlimited + ? t('storageManagement.unlimited') + : formattedQuota + ? t('folderNavigation.totalQuota', { + size: formattedQuota.size, + unit: formattedQuota.unit, + }) + : t('folderNavigation.sharedSpace')} + +
+ +
+
+
+ +
+ + {usageRatio !== null + ? t('folderNavigation.usedPercent', { + percent: usageRatio.toFixed(1), + }) + : isUnlimited + ? formattedSize + ? t('folderNavigation.usedAmount', { + size: formattedSize.size, + unit: formattedSize.unit, + }) + : '—' + : t('folderNavigation.sharedWithoutQuota')} + + + {t('folderNavigation.fileCount', { count: r.size })} + +
+
+ ) + })()} {/* Action Button */} +
+ + +
+ + + + {t('storageQuotaAudit.createdAt')} + {t('storageQuotaAudit.operator')} + {t('storageQuotaAudit.user')} + {t('storageQuotaAudit.oldQuota')} + {t('storageQuotaAudit.newQuota')} + {t('storageQuotaAudit.provider')} + {t('storageQuotaAudit.status')} + + + + {records.length === 0 ? ( + + + {auditQuery.isLoading + ? t('storageQuotaAudit.loading') + : t('storageQuotaAudit.empty')} + + + ) : ( + records.map((record) => ( + + + {new Date(record.created_at).toLocaleString()} + + {record.operator} + {record.target} + + {formatBytes( + detailNumber(record.details, 'old_quota'), + t('storageManagement.unlimited') + )} + + + {formatBytes( + detailNumber(record.details, 'new_quota'), + t('storageManagement.unlimited') + )} + + {detailString(record.details, 'provider')} + + + {record.status === 'Success' + ? t('storageQuotaAudit.success') + : t('storageQuotaAudit.failed')} + + + + )) + )} + +
+
+ + + ) +} diff --git a/frontend/src/routes/admin/storage/index.tsx b/frontend/src/routes/admin/storage/index.tsx new file mode 100644 index 000000000..d0caeb7a1 --- /dev/null +++ b/frontend/src/routes/admin/storage/index.tsx @@ -0,0 +1,391 @@ +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import { createFileRoute } from '@tanstack/react-router' +import { ColumnDef } from '@tanstack/react-table' +import { RefreshCcw } from 'lucide-react' +import { useState } from 'react' +import { useTranslation } from 'react-i18next' +import { toast } from 'sonner' + +import { Button } from '@/components/ui/button' +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card' +import { + Dialog, + DialogContent, + DialogFooter, + DialogHeader, + DialogTitle, +} from '@/components/ui/dialog' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/ui/select' + +import { DataTable } from '@/components/query-table' +import { DataTableColumnHeader } from '@/components/query-table/column-header' + +import { + PagedUserSpaces, + UserSpace, + apiAdminGetStorageCapabilities, + apiAdminGetUserSpaces, + apiAdminRefreshUserSpaceUsage, + apiAdminSetUserSpaceQuota, +} from '@/services/api/storage' +import { IResponse } from '@/services/types' + +import StorageQuotaAuditPanel from './-components/storage-governance-panel' + +export const Route = createFileRoute('/admin/storage/')({ + component: StorageManagementPage, +}) + +function getErrorMessage(error: unknown, fallback: string): string { + if (typeof error === 'object' && error !== null) { + const candidate = error as { + data?: { msg?: string } + response?: { data?: { msg?: string } } + } + return candidate.data?.msg ?? candidate.response?.data?.msg ?? fallback + } + return fallback +} + +const QUOTA_UNIT_BYTES = { + B: 1, + KB: 1024, + MB: 1024 ** 2, + GB: 1024 ** 3, + TB: 1024 ** 4, +} as const + +function quotaValueForBytes(bytes: number, unitBytes: number): number { + const rawValue = bytes / unitBytes + for (let precision = 0; precision <= 12; precision += 1) { + const candidate = Number(rawValue.toFixed(precision)) + if (Math.round(candidate * unitBytes) === bytes) return candidate + } + return rawValue +} + +function normalizeQuotaDisplay(value: number, unit: string): { value: number; unit: string } { + if (!Number.isFinite(value) || value < 0 || unit === 'unlimited') { + return { value: Math.max(0, value || 0), unit } + } + + const unitBytes = QUOTA_UNIT_BYTES[unit as keyof typeof QUOTA_UNIT_BYTES] + if (!unitBytes) return { value, unit } + + const bytes = Math.round(value * unitBytes) + const orderedUnits: Array = ['TB', 'GB', 'MB', 'KB', 'B'] + + const exactUnit = orderedUnits.find((candidate) => { + const candidateBytes = QUOTA_UNIT_BYTES[candidate] + return bytes >= candidateBytes && bytes % candidateBytes === 0 + }) + if (exactUnit) { + return { + value: bytes / QUOTA_UNIT_BYTES[exactUnit], + unit: exactUnit, + } + } + + const fallbackUnit = orderedUnits.find((candidate) => bytes >= QUOTA_UNIT_BYTES[candidate]) ?? 'B' + return { + value: quotaValueForBytes(bytes, QUOTA_UNIT_BYTES[fallbackUnit]), + unit: fallbackUnit, + } +} + +export default function StorageManagementPage() { + const { t } = useTranslation() + const queryClient = useQueryClient() + + // Quota confirmation dialog state. + const [isQuotaDialogOpen, setIsQuotaDialogOpen] = useState(false) + const [selectedUser, setSelectedUser] = useState(null) + const [quotaValue, setQuotaValue] = useState(0) + const [quotaUnit, setQuotaUnit] = useState('GB') + + const storageCapabilitiesQuery = useQuery({ + queryKey: ['admin', 'storage', 'capabilities'], + queryFn: () => apiAdminGetStorageCapabilities().then((res) => res.data), + staleTime: 60 * 1000, + }) + const storageCapabilities = storageCapabilitiesQuery.data + const usageAvailable = + !!storageCapabilities?.quota_enabled && !!storageCapabilities?.usage_readable + const quotaManagementAvailable = usageAvailable && !!storageCapabilities?.quota_writable + + // Load cached usage for all user spaces. + const userSpacesQuery = useQuery({ + queryKey: ['admin', 'user-spaces'], + queryFn: () => + apiAdminGetUserSpaces(1, 1000).then((res: IResponse) => res.data.items), + enabled: usageAvailable, + staleTime: 5 * 60 * 1000, + }) + + // Apply a quota only after explicit confirmation in the dialog. + const setQuotaMutation = useMutation({ + mutationFn: ({ user, quota }: { user: string; quota: number }) => + apiAdminSetUserSpaceQuota(user, quota), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ['admin', 'user-spaces'] }) + queryClient.invalidateQueries({ queryKey: ['admin', 'storage-quota-audit'] }) + toast.success(t('storageManagement.setQuotaSuccess')) + setIsQuotaDialogOpen(false) + }, + onError: (error: unknown) => { + toast.error(getErrorMessage(error, t('storageManagement.setQuotaError'))) + }, + }) + + const refreshUsageMutation = useMutation({ + mutationFn: apiAdminRefreshUserSpaceUsage, + onSuccess: (response) => { + queryClient.invalidateQueries({ queryKey: ['admin', 'user-spaces'] }) + toast.success( + t('storageManagement.refreshSuccess', { + updated: response.data.updated, + failed: response.data.failed, + }) + ) + }, + onError: (error: unknown) => { + toast.error(getErrorMessage(error, t('storageManagement.refreshError'))) + }, + }) + + const convertToBytes = (value: number, unit: string): number => { + switch (unit) { + case 'B': + return Math.round(value) + case 'KB': + return Math.round(value * 1024) + case 'MB': + return Math.round(value * 1024 * 1024) + case 'GB': + return Math.round(value * 1024 * 1024 * 1024) + case 'TB': + return Math.round(value * 1024 * 1024 * 1024 * 1024) + default: + return Math.round(value) + } + } + + const alignQuotaInput = () => { + if (quotaUnit === 'unlimited') return + const normalized = normalizeQuotaDisplay(quotaValue, quotaUnit) + setQuotaValue(normalized.value) + setQuotaUnit(normalized.unit) + } + + const handleSetQuota = () => { + if (!selectedUser) return + alignQuotaInput() + const normalized = normalizeQuotaDisplay(quotaValue, quotaUnit) + const quotaInBytes = + normalized.unit === 'unlimited' ? -1 : convertToBytes(normalized.value, normalized.unit) + if (quotaInBytes !== -1 && (!Number.isSafeInteger(quotaInBytes) || quotaInBytes <= 0)) { + toast.error(t('storageManagement.invalidQuota')) + return + } + setQuotaMutation.mutate({ user: selectedUser.user, quota: quotaInBytes }) + } + + const openSetQuotaDialog = (user: UserSpace) => { + setSelectedUser(user) + if (user.quota === -1) { + setQuotaValue(0) + setQuotaUnit('unlimited') + } else { + const normalized = normalizeQuotaDisplay(user.quota, 'B') + setQuotaValue(normalized.value) + setQuotaUnit(normalized.unit) + } + setIsQuotaDialogOpen(true) + } + + const usageColumns: ColumnDef[] = [ + { + accessorKey: 'user', + header: ({ column }) => ( + + ), + }, + { + accessorKey: 'size', + header: ({ column }) => ( + + ), + cell: ({ row }) => + row.original.size < 0 ? t('storageManagement.usagePending') : row.original.formatted, + }, + { + accessorKey: 'updated_at', + header: ({ column }) => ( + + ), + cell: ({ row }) => + row.original.updated_at + ? new Date(row.original.updated_at).toLocaleString() + : t('storageManagement.neverRefreshed'), + }, + ] + + const quotaColumns: ColumnDef[] = [ + { + accessorKey: 'quota', + header: ({ column }) => ( + + ), + cell: ({ row }) => { + const { quota, quota_formatted } = row.original + return quota === -1 ? t('storageManagement.unlimited') : quota_formatted + }, + }, + { + accessorKey: 'usage_ratio', + header: ({ column }) => ( + + ), + cell: ({ row }) => { + const { size, quota } = row.original + if (size < 0 || quota <= 0) return '-' + const ratio = (size / quota) * 100 + const color = + ratio >= 90 + ? 'text-red-500 font-semibold' + : ratio >= 70 + ? 'text-yellow-500' + : 'text-green-600' + return {ratio.toFixed(1)}% + }, + }, + { + accessorKey: 'actions', + header: t('storageManagement.actions'), + cell: ({ row }) => { + const user = row.original + return ( +
+ +
+ ) + }, + }, + ] + const columns = quotaManagementAvailable ? [...usageColumns, ...quotaColumns] : usageColumns + + return ( + <> + {usageAvailable ? ( + + + + ) : ( + + + {t('navigation.storageManagement')} + {t('storageManagement.unavailable')} + + + {storageCapabilitiesQuery.isLoading + ? t('storageManagement.detecting') + : storageCapabilities?.reasons?.join('; ') || t('storageManagement.requirements')} + + + )} + {quotaManagementAvailable && } + + {quotaManagementAvailable && ( + + + + + {t('storageManagement.setQuotaFor', { user: selectedUser?.user })} + + +
+
+ + setQuotaValue(Number(e.target.value))} + onBlur={alignQuotaInput} + disabled={quotaUnit === 'unlimited'} + /> + +
+
+ {t('storageManagement.currentUsage', { + usage: + selectedUser && selectedUser.size >= 0 + ? selectedUser.formatted + : t('storageManagement.usagePending'), + })} +
+
+ + + + + + )} + + ) +} diff --git a/frontend/src/routes/admin/storage/route.tsx b/frontend/src/routes/admin/storage/route.tsx new file mode 100644 index 000000000..e76006476 --- /dev/null +++ b/frontend/src/routes/admin/storage/route.tsx @@ -0,0 +1,7 @@ +import { createFileRoute } from '@tanstack/react-router' + +import StorageManagementPage from './index' + +export const Route = createFileRoute('/admin/storage')({ + component: StorageManagementPage, +}) diff --git a/frontend/src/services/api/file.ts b/frontend/src/services/api/file.ts index a96dad99c..e8bf52d5c 100644 --- a/frontend/src/services/api/file.ts +++ b/frontend/src/services/api/file.ts @@ -56,3 +56,46 @@ export const apiGetDatasetFiles = (datasetID: number, path: string) => apiGet>( path === '' ? `ss/dataset/${datasetID}` : `ss/dataset/${datasetID}/${path.replace(/^\//, '')}` ) + +export interface DirectorySize { + path: string + size: number + unit: string + formatted: string +} + +export const apiGetDirectorySize = (path: string) => + apiGet>( + `v1/storage/dirsize/${path + .replace(/^\//, '') + .split('/') + .map((segment) => encodeURIComponent(segment)) + .join('/')}` + ) + +export interface MyQuota { + space_quota: number + space_quota_formatted: string +} + +export const apiGetMyQuota = () => apiGet>('v1/storage/my-quota') + +export interface StorageCapabilities { + backend: string + configured: boolean + quota_enabled: boolean + pvc_name: string + pvc_namespace?: string + pv_name?: string + csi_driver?: string + quota_provider: 'auto' | 'storageServer' | 'toolbox' | 'disabled' + storage_server_available: boolean + toolbox_available: boolean + usage_readable: boolean + quota_readable: boolean + quota_writable: boolean + reasons?: string[] +} + +export const apiGetStorageCapabilities = () => + apiGet>('v1/storage/capabilities') diff --git a/frontend/src/services/api/storage.ts b/frontend/src/services/api/storage.ts new file mode 100644 index 000000000..c54338ea8 --- /dev/null +++ b/frontend/src/services/api/storage.ts @@ -0,0 +1,84 @@ +import { apiGet, apiPost, apiPut } from '@/services/client' +import { IResponse } from '@/services/types' + +export interface UserSpace { + user: string + size: number + quota: number + unit: string + formatted: string + updated_at?: string | null + quota_formatted: string + is_expanded: boolean + jobs_frozen: boolean + shrink_stage?: string + original_quota?: number + original_quota_formatted?: string +} + +export interface PagedUserSpaces { + items: UserSpace[] + total: number + page: number + pageSize: number + totalPages: number +} + +export interface StorageCapabilities { + backend: string + configured: boolean + quota_enabled: boolean + pvc_name: string + pvc_namespace?: string + pv_name?: string + csi_driver?: string + quota_provider: 'auto' | 'storageServer' | 'toolbox' | 'disabled' + storage_server_available: boolean + toolbox_available: boolean + usage_readable: boolean + quota_readable: boolean + quota_writable: boolean + reasons?: string[] +} + +export const apiAdminGetStorageCapabilities = (): Promise> => + apiGet>('v1/admin/storage/capabilities') + +export const apiAdminGetUserSpaces = ( + page: number = 1, + pageSize: number = 10 +): Promise> => + apiGet>( + `v1/admin/storage/user-spaces?page=${page}&pageSize=${pageSize}` + ) + +export interface RefreshStorageUsageResponse { + updated: number + failed: number + refreshed_at: string +} + +export const apiAdminRefreshUserSpaceUsage = (): Promise> => + apiPost>( + 'v1/admin/storage/user-spaces/refresh', + undefined, + { timeout: false } + ) + +export interface SetQuotaRequest { + quota: number +} + +export interface SetQuotaResponse { + user: string + quota: number + unit: string + quota_formatted: string + ceph_quota_set: boolean +} + +export const apiAdminSetUserSpaceQuota = ( + user: string, + quota: number +): Promise> => + apiPut>(`v1/admin/storage/user-spaces/${user}/quota`, { quota }) diff --git a/frontend/src/services/api/vcjob.ts b/frontend/src/services/api/vcjob.ts index 212fe41a6..8892d79ec 100644 --- a/frontend/src/services/api/vcjob.ts +++ b/frontend/src/services/api/vcjob.ts @@ -728,3 +728,7 @@ export type GetCronjobConfigStatusResp = Record apiV1Post>('admin/operations/cronjob/config/status', param) + +// 执行巡检任务 +export const apiAdminExecutePatrolJob = (jobName: string) => + apiV1Post>('admin/operations/cronjob/execute', { jobName }) diff --git a/frontend/src/services/client.ts b/frontend/src/services/client.ts index 7ef4095f6..f66441017 100644 --- a/frontend/src/services/client.ts +++ b/frontend/src/services/client.ts @@ -412,8 +412,8 @@ export const apiV1Delete = (url: string, json?: unknown) => export const apiGet = (url: string, options?: Options) => apiRequest(() => apiClient.get(url, options).json()) -export const apiPost = (url: string, json?: unknown) => - apiRequest(() => apiClient.post(url, { json }).json()) +export const apiPost = (url: string, json?: unknown, options?: Options) => + apiRequest(() => apiClient.post(url, { json, ...options }).json()) export const apiPut = (url: string, json?: unknown) => apiRequest(() => apiClient.put(url, { json }).json()) diff --git a/frontend/src/utils/file-size.ts b/frontend/src/utils/file-size.ts index eb1fc98bd..b68e693d2 100644 --- a/frontend/src/utils/file-size.ts +++ b/frontend/src/utils/file-size.ts @@ -37,7 +37,7 @@ export const isFileSizeExceeded = (fileSize: number): boolean => { * @returns 格式化后的文件大小字符串 */ export const formatFileSize = (bytes: number): string => { - if (bytes === 0) return '0 B' + if (!Number.isFinite(bytes) || bytes <= 0) return '0 B' const k = 1024 const sizes = ['B', 'KB', 'MB', 'GB', 'TB'] diff --git a/frontend/src/utils/formatter.ts b/frontend/src/utils/formatter.ts index 4e72916ff..23c1e98d6 100644 --- a/frontend/src/utils/formatter.ts +++ b/frontend/src/utils/formatter.ts @@ -24,8 +24,7 @@ export const shortestImageName = (imageName: string): string => { } export function formatBytes(bytes: number, decimals: number = 2): string { - if (bytes === 0) return '0 B' - if (bytes < 0) return '-' + formatBytes(-bytes, decimals) + if (!Number.isFinite(bytes) || bytes <= 0) return '0 B' const k = 1024 const sizes = ['B', 'KB', 'MB', 'GB', 'TB'] diff --git a/website/content/docs/admin/more/db.mdx b/website/content/docs/admin/more/db.mdx index 873914ae3..3ac04b86e 100644 --- a/website/content/docs/admin/more/db.mdx +++ b/website/content/docs/admin/more/db.mdx @@ -1,4 +1,278 @@ --- -title: "高可用数据库配置" -description: "提供高可用的数据库配置" +title: "CephFS 配额管理" +description: "在 Rook CephFS 集群中部署 quota-agent，读取目录用量并安全地管理用户目录配额。" --- + +## 功能概览 + +Crater 可以读取 CephFS 用户目录的实际用量，并由管理员设置或取消目录配额。该功能使用以下 CephFS 扩展属性： + +- `ceph.dir.rbytes`：目录当前实际用量，无需预先设置配额即可读取。 +- `ceph.quota.max_bytes`：目录最大可用字节数，写入 `0` 表示取消配额限制。 + +配额管理默认关闭，仅支持由 Rook CephFS CSI 提供的共享存储。NFS、本地盘和其他 CSI 存储仍可正常用于文件浏览、上传和下载，但不会显示配额管理入口。 + + +Crater 不会通过定时任务扫描所有用户目录。管理员在存储管理页面点击“刷新用量”后，后端才会逐个读取目录用量，并记录每条数据的刷新时间。 + + +## 实现方式 + +quota-agent 复用 Crater 的 storage-server 镜像，并通过 `CRATER_STORAGE_MODE=quota-agent` 切换到受限运行模式。请求链路如下： + +1. 管理员通过 Crater 后端刷新用量或修改配额。 +2. 后端通过集群内部 Service 调用 quota-agent。 +3. quota-agent 在挂载的 CephFS 目录上读取或写入扩展属性。 +4. 后端缓存目录用量，并将配额修改结果写入操作日志。 + +quota-agent 只注册内部配额接口，不提供 WebDAV、文件下载或用户登录接口。它使用独立的 CephX 客户端，并且 `p` 权限仅限制在 Crater 使用的 CephFS 子卷路径，不会扩大普通业务 Pod 使用的 CSI 身份权限。 + +内部接口由后端登录密钥派生的令牌保护，quota-agent 不需要数据库、LDAP、镜像仓库或完整后端配置。此方案不依赖 Rook toolbox。 + +## 前提条件 + +启用前请确认： + +- Crater 的共享存储 PVC 已经创建并处于 `Bound` 状态。 +- 共享 PVC 由 Rook CephFS CSI 提供，而不是 NFS 或块存储。 +- 集群已经安装 Rook Operator 和 `CephClient` CRD。 +- 执行初始化脚本的管理员可以创建 `CephClient`、Secret、PV 和 PVC，并可以修改原始 PV 的回收策略。 +- backend、storage-server 和 Helm Chart 使用包含 quota-agent 功能的同一版本。 +- 管理终端已经安装 `bash`、`kubectl`、`grep` 和 `helm`。 + +首先检查共享 PVC。以下示例使用 Chart 的默认 PVC 名称；如果集群修改过 `storage.pvcName`，请替换 `crater-rw-storage`： + +```bash +kubectl -n crater-workspace get pvc crater-rw-storage + +SOURCE_PV=$(kubectl -n crater-workspace get pvc crater-rw-storage \ + -o jsonpath='{.spec.volumeName}') + +kubectl get pv "$SOURCE_PV" \ + -o jsonpath='{.spec.csi.driver}{"\n"}{.spec.csi.volumeAttributes.fsName}{"\n"}{.spec.csi.volumeAttributes.subvolumePath}{"\n"}' +``` + +CSI Driver 应类似 `rook-ceph.cephfs.csi.ceph.com`，并且 `fsName` 与 `subvolumePath` 均不为空。 + +## 创建专用 CephFS 身份 + +初始化脚本会读取现有 PVC 对应的 CephFS 子卷信息，然后创建： + +- 一个仅对该子卷路径拥有 MDS `p` 权限的 `CephClient`； +- 一个供 CephFS CSI 挂载使用的 Secret； +- 一个指向同一 CephFS 子卷的静态 PV； +- 一个供 quota-agent 挂载的专用 PVC。 + + +脚本会把原始动态 PV 的 `persistentVolumeReclaimPolicy` 修改为 `Retain`。这是为了避免静态 PV 仍指向同一子卷时，删除原始 PVC 导致底层 CephFS 子卷被提前回收。执行前请确认该策略符合集群的数据生命周期要求。 + + +从与已部署 Chart 对应的 Crater 源码版本执行脚本。已有同版本源码时可以跳过克隆步骤。 + + + +```bash +git clone --branch "v" --depth 1 \ + https://github.com/raids-lab/crater.git +cd crater + +APP_NAMESPACE=crater-workspace \ +ROOK_NAMESPACE=rook-ceph \ +SOURCE_PVC=crater-rw-storage \ +bash backend/hack/bootstrap-cephfs-quota-agent.sh +``` + +默认会创建以下资源： + +| 资源 | 默认名称 | +| --- | --- | +| CephClient | `rook-ceph/crater-quota` | +| CSI Secret | `rook-ceph/crater-quota-csi` | +| 静态 PV | `crater-quota-storage-pv` | +| quota-agent PVC | `crater-workspace/crater-quota-storage` | + +脚本支持通过环境变量适配不同集群： + +| 环境变量 | 默认值 | 用途 | +| --- | --- | --- | +| `APP_NAMESPACE` | `crater-workspace` | Crater 作业与存储所在命名空间 | +| `ROOK_NAMESPACE` | `rook-ceph` | Rook Operator 与 CephClient 所在命名空间 | +| `SOURCE_PVC` | `crater-rw-storage` | 已绑定的 Crater CephFS 共享 PVC | +| `QUOTA_CLIENT` | `crater-quota` | 专用 CephClient 名称 | +| `QUOTA_PV` | `crater-quota-storage-pv` | 静态 PV 名称 | +| `QUOTA_PVC` | `crater-quota-storage` | quota-agent 使用的 PVC 名称 | +| `CSI_SECRET` | `crater-quota-csi` | CSI 挂载凭证 Secret 名称 | + +脚本不会输出 CephX 密钥，也不会修改 `client.csi-cephfs-node` 等现有 CSI 客户端权限。 + +## 启用 Helm 组件 + +在集群使用的 values 文件中加入： + +```yaml +quotaAgent: + enabled: true + existingClaim: crater-quota-storage + +backendConfig: + storage: + quota: + rookNamespace: rook-ceph + cephFSCSIDriver: rook-ceph.cephfs.csi.ceph.com + cephFSName: cephfs +``` + +当 `quotaAgent.enabled` 为 `true` 时，Chart 会自动完成以下配置，无需重复填写： + +```yaml +backendConfig: + storage: + quota: + enabled: true + provider: storageServer + storageServerURL: http://crater-quota-agent.crater-workspace.svc:7320 +``` + +quota-agent 复用 `images.storage`，因此请确保该镜像标签与此次发布匹配。Rook 不在 `rook-ceph` 命名空间，或 CSI Driver 名称不同的集群，必须覆盖对应字段。 + +使用现有的完整 values 文件升级 Crater： + + + +```bash +helm upgrade crater oci://ghcr.io/raids-lab/crater \ + --version \ + --values values.yaml +``` + +升级后 Chart 会在 `namespaces.job` 指定的命名空间创建： + +- `Deployment/crater-quota-agent` +- `Service/crater-quota-agent` +- `Secret/crater-quota-agent-auth` + +默认配置中的 `quotaAgent.enabled: false` 和 `backendConfig.storage.quota.enabled: false` 保证未启用此功能的 NFS 或其他存储集群不受影响。 + +## 验证部署 + +确认专用 PVC 和 quota-agent 正常： + +```bash +kubectl -n crater-workspace get pvc crater-quota-storage +kubectl -n crater-workspace rollout status deployment/crater-quota-agent +kubectl -n crater-workspace logs deployment/crater-quota-agent +``` + +启动日志应包含： + +```text +mode=quota-agent +``` + +登录 Crater 管理员界面并进入“存储管理”。后端能力探测成功后，应满足： + +- `usage_readable=true` +- `quota_readable=true` +- `quota_writable=true` + +此时页面会显示配额修改控件和“刷新用量”按钮。首次刷新前，用户仍会正常列出，但用量显示为等待刷新。刷新完成后，每条用量会显示对应的刷新时间。 + +可以使用一个测试用户完成端到端验证： + +1. 点击“刷新用量”，确认用户目录用量可以读取。 +2. 给测试用户设置一个高于当前用量的临时测试配额。 +3. 再次刷新或重新打开页面，确认配额值可以读回。 +4. 测试完成后，在页面中取消配额或恢复原配额。 +5. 在配额修改记录中确认成功或失败操作均已记录。 + + +不要给用量接近上限的生产用户设置过小的测试配额。CephFS 不会删除已经存在的数据，但目录达到或超过上限后，新的写入会失败，应用通常会收到空间不足错误。 + + +## 配置参考 + +| 配置项 | 默认值 | 说明 | +| --- | --- | --- | +| `quotaAgent.enabled` | `false` | 是否创建 quota-agent Deployment 和 Service | +| `quotaAgent.existingClaim` | 空 | quota-agent 挂载的专用静态 CephFS PVC；启用时必填 | +| `quotaAgent.replicas` | `1` | quota-agent 副本数 | +| `backendConfig.storage.quota.enabled` | `false` | 配额功能总开关；启用 quota-agent 时由 Chart 自动打开 | +| `backendConfig.storage.quota.provider` | `auto` | 可选 `auto`、`storageServer`、`toolbox` 或 `disabled` | +| `backendConfig.storage.quota.storageServerURL` | 空 | 内部配额服务地址；启用 quota-agent 时由 Chart 自动设置 | +| `backendConfig.storage.quota.rookNamespace` | `rook-ceph` | Rook 资源与可选 toolbox 所在命名空间 | +| `backendConfig.storage.quota.cephFSCSIDriver` | 空 | CephFS CSI Driver；为空时根据 Rook 命名空间推导 | +| `backendConfig.storage.quota.toolboxLabelSelector` | `app=rook-ceph-tools` | 仅在选择 toolbox 回退方案时使用 | +| `backendConfig.storage.quota.cephFSName` | `cephfs` | PV 未提供 `fsName` 时使用的回退文件系统名称 | + +使用 quota-agent 时，主要 Provider 是 `storageServer`，不需要部署 toolbox。`toolboxLabelSelector` 仅用于显式选择 toolbox 或 `auto` 回退模式的集群。 + +## 常见问题 + +### 脚本提示不是受支持的 CephFS CSI 卷 + +检查共享 PVC 对应 PV 的 `.spec.csi.driver`、`volumeAttributes.fsName` 和 `volumeAttributes.subvolumePath`。NFS PVC、块设备 PVC 或缺少子卷路径的 PV 不能使用该初始化脚本。 + +### 脚本提示 CephClient CRD 不存在 + +确认 Rook Operator 已安装并正常运行： + +```bash +kubectl api-resources --api-group=ceph.rook.io +kubectl -n rook-ceph get pods +``` + +输出中应包含 `cephclients.ceph.rook.io`。如果 Rook 使用其他命名空间，请同时修改命令和 `ROOK_NAMESPACE`。 + +### quota-agent PVC 一直处于 Pending + +检查静态 PV 与 PVC 的 `storageClassName`、容量、访问模式和 `volumeName` 是否一致，并查看事件： + +```bash +kubectl -n crater-workspace describe pvc crater-quota-storage +kubectl describe pv crater-quota-storage-pv +``` + +### quota-agent 已启动但页面没有配额入口 + +依次检查： + +1. Helm 渲染后的 `quotaAgent.enabled` 是否为 `true`。 +2. 后端配置中的共享 PVC 名称是否正确。 +3. 共享 PV 的 CSI Driver 是否与 `cephFSCSIDriver` 一致。 +4. backend 是否可以访问 `crater-quota-agent` Service。 +5. quota-agent 日志中是否存在读取或写入扩展属性失败。 + +前端只在后端能力探测确认 CephFS 配额可用后展示入口，仅创建 Pod 并不代表权限已经验证成功。 + +### 返回 permission denied + +确认 quota-agent 挂载的是专用 `crater-quota-storage` PVC，而不是普通 WebDAV PVC；然后检查 `CephClient/crater-quota` 的 MDS caps 是否包含目标 `fsName`、`subvolumePath` 和 `p` 权限。不要通过扩大普通 CSI 节点客户端权限来绕过问题。 + +### 集群使用 NFS + +保持以下默认配置即可： + +```yaml +quotaAgent: + enabled: false + +backendConfig: + storage: + quota: + enabled: false +``` + +文件浏览和上传下载不会受影响，前端也不会显示 CephFS 配额管理页面。 + +## 停用与清理 + +只停用功能时，将 `quotaAgent.enabled` 和 `backendConfig.storage.quota.enabled` 设为 `false`，然后执行正常的 Helm 升级。Chart 会停止创建 quota-agent 工作负载，原有文件和配额扩展属性不会被自动删除。 + +如果还要永久删除专用 CephClient、Secret、PVC 和 PV，请遵循以下顺序： + +1. 先停用并确认 quota-agent Deployment 已删除。 +2. 确认没有其他 Pod 使用 `crater-quota-storage`。 +3. 删除专用 PVC 和静态 PV。 +4. 最后删除专用 CSI Secret 与 CephClient。 + +专用静态 PV 和原始动态 PV 的回收策略均可能为 `Retain`。清理专用资源时不要删除 Crater 原始共享 PVC，也不要手动删除底层 CephFS 子卷，否则可能造成业务数据丢失。