From 766925a1f2427a16e1c98aeb01b0525ade8a0c9f Mon Sep 17 00:00:00 2001 From: unknown <2362787603@qq.com> Date: Tue, 16 Jun 2026 00:24:04 +0800 Subject: [PATCH 1/6] feat(storage): add storage governance and metadata management --- .gitattributes | 9 + .gitignore | 44 + backend/cmd/crater/helper/config.go | 1 + backend/dao/model/storage_decision.go | 62 + backend/dao/model/storage_index.go | 256 + backend/dao/model/system_config.go | 26 +- backend/dao/model/user_space_size.go | 26 + backend/go.mod | 1 + backend/go.sum | 2 + .../hack/storage-index-experiment-toolbox.sh | 82 + backend/internal/handler/aijob/new.go | 12 + .../internal/handler/operations/cronjob.go | 95 + .../internal/handler/operations/operations.go | 1 + backend/internal/handler/spjob/spjob.go | 5 + backend/internal/handler/storage.go | 1085 +++ backend/internal/handler/system_config.go | 116 +- backend/internal/handler/vcjob/custom.go | 6 + backend/internal/handler/vcjob/jupyter.go | 6 + backend/internal/handler/vcjob/pytorch.go | 7 + backend/internal/handler/vcjob/tensorflow.go | 6 + backend/internal/handler/vcjob/webide.go | 6 + backend/internal/service/config_service.go | 219 +- backend/internal/util/quota.go | 88 + backend/pkg/ceph/ceph.go | 658 ++ backend/pkg/cronjob/manger.go | 106 + backend/pkg/cronjob/manger_test.go | 4 +- backend/pkg/llm/direct_decision.go | 525 ++ backend/pkg/llm/direct_decision_test.go | 118 + backend/pkg/llm/llm_tools.go | 1008 +++ backend/pkg/llm/provider.go | 207 + backend/pkg/llm/skill.go | 46 + backend/pkg/llm/skills/README.md | 19 + .../skills/storage-governance-agent/SKILL.md | 22 + backend/pkg/monitor/helper.go | 36 + backend/pkg/monitor/interface.go | 10 + backend/pkg/patrol/patrol.go | 595 +- backend/pkg/storagegovernance/policy.go | 282 + backend/pkg/storagegovernance/policy_test.go | 112 + backend/pkg/storagegovernance/query.go | 175 + backend/pkg/storagegovernance/replay.go | 88 + backend/pkg/storagegovernance/service.go | 589 ++ backend/pkg/storagegovernance/types.go | 95 + backend/pkg/storageindex/experiment.go | 187 + backend/pkg/storageindex/service.go | 6922 +++++++++++++++++ backend/pkg/storageindex/service_test.go | 870 +++ .../components/file/file-select-dialog.tsx | 3 + .../src/components/file/folder-navigation.tsx | 229 +- .../src/components/file/lazy-file-tree.tsx | 22 +- frontend/src/components/ui/collapsible.tsx | 7 + .../src/i18n/locales/enUS/translation.json | 35 + .../src/i18n/locales/zhCN/translation.json | 50 + frontend/src/index.css | 42 + frontend/src/routeTree.gen.ts | 49 + .../cronjobs/-components/cronjob-card.tsx | 91 +- frontend/src/routes/admin/cronjobs/index.tsx | 33 +- .../admin/more/-components/llm-settings.tsx | 3 +- .../-components/storage-decision-settings.tsx | 252 + frontend/src/routes/admin/more/index.tsx | 256 +- frontend/src/routes/admin/route.tsx | 6 + .../storage-directory-compare-panel.tsx | 374 + .../-components/storage-governance-panel.tsx | 701 ++ .../-components/storage-index-panel.tsx | 578 ++ frontend/src/routes/admin/storage/index.tsx | 634 ++ frontend/src/routes/admin/storage/route.tsx | 7 + frontend/src/services/api/file.ts | 86 +- frontend/src/services/api/storage.ts | 570 ++ frontend/src/services/api/system-config.ts | 24 + frontend/src/services/api/vcjob.ts | 4 + frontend/src/services/client.ts | 4 +- frontend/src/utils/file-size.ts | 2 +- frontend/src/utils/format.ts | 17 + frontend/src/utils/formatter.ts | 3 +- 72 files changed, 18715 insertions(+), 202 deletions(-) create mode 100644 .gitattributes create mode 100644 backend/dao/model/storage_decision.go create mode 100644 backend/dao/model/storage_index.go create mode 100644 backend/dao/model/user_space_size.go create mode 100644 backend/hack/storage-index-experiment-toolbox.sh create mode 100644 backend/internal/handler/storage.go create mode 100644 backend/internal/util/quota.go create mode 100644 backend/pkg/ceph/ceph.go create mode 100644 backend/pkg/llm/direct_decision.go create mode 100644 backend/pkg/llm/direct_decision_test.go create mode 100644 backend/pkg/llm/llm_tools.go create mode 100644 backend/pkg/llm/provider.go create mode 100644 backend/pkg/llm/skill.go create mode 100644 backend/pkg/llm/skills/README.md create mode 100644 backend/pkg/llm/skills/storage-governance-agent/SKILL.md create mode 100644 backend/pkg/storagegovernance/policy.go create mode 100644 backend/pkg/storagegovernance/policy_test.go create mode 100644 backend/pkg/storagegovernance/query.go create mode 100644 backend/pkg/storagegovernance/replay.go create mode 100644 backend/pkg/storagegovernance/service.go create mode 100644 backend/pkg/storagegovernance/types.go create mode 100644 backend/pkg/storageindex/experiment.go create mode 100644 backend/pkg/storageindex/service.go create mode 100644 backend/pkg/storageindex/service_test.go create mode 100644 frontend/src/routes/admin/more/-components/storage-decision-settings.tsx create mode 100644 frontend/src/routes/admin/storage/-components/storage-directory-compare-panel.tsx create mode 100644 frontend/src/routes/admin/storage/-components/storage-governance-panel.tsx create mode 100644 frontend/src/routes/admin/storage/-components/storage-index-panel.tsx create mode 100644 frontend/src/routes/admin/storage/index.tsx create mode 100644 frontend/src/routes/admin/storage/route.tsx create mode 100644 frontend/src/services/api/storage.ts create mode 100644 frontend/src/utils/format.ts diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..8bd5c6ce3 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,9 @@ +* text=auto eol=lf +*.png -text +*.jpg -text +*.jpeg -text +*.gif -text +*.webp -text +*.pdf -text +*.zip -text +backend/crater -text diff --git a/.gitignore b/.gitignore index 83c85888e..e6c7d6bb7 100644 --- a/.gitignore +++ b/.gitignore @@ -79,3 +79,47 @@ kubeconfig* # Claude Code .claude/ +storage/etc/config.yaml +.gocache/ +.gomodcache/ + +# Local build/test scratch +.gocache*/ +.gotmp*/ +backend/crater + +# Python cache +__pycache__/ +*.pyc +*.pyo + +# Temporary experiment artifacts +tmp_* +rewrite_section3*.py +storage-governance-datasets.zip +storage-governance-qwen25-7b.zip + +# Editor/tool backup files with random numeric suffixes +*.go.[0-9]* + +# Local workspace-only directories and assets +/.claude/ +/.gocache-test/ +/.gopath/ +/_external/ +/datasets/ +/models/ +/models_compare/ +/offline-tools/ +/offline-tools.zip +/graduation-thesis-latex/ + +# Local document/build logs +/texput.log + +# Local documentation drafts +/docs/zh-CN/framework-architecture-and-workflow.md +/docs/zh-CN/storage-governance-requirements/ +/docs/zh-CN/storage-management.md +/docs/zh-CN/storage-metadata-index-policy-engine.md +/docs/zh-CN/storage-tiering-policy-satisfier.md diff --git a/backend/cmd/crater/helper/config.go b/backend/cmd/crater/helper/config.go index 7b548f9d1..8868f4036 100644 --- a/backend/cmd/crater/helper/config.go +++ b/backend/cmd/crater/helper/config.go @@ -106,6 +106,7 @@ func (ci *ConfigInitializer) SetupManagerDependencies(registerConfig *handler.Re registerConfig.CronJobManager = cronjob.NewCronJobManager( registerConfig.Client, registerConfig.KubeClient, + registerConfig.KubeConfig, registerConfig.PrometheusClient, registerConfig.GpuAnalysisService, registerConfig.BillingService, diff --git a/backend/dao/model/storage_decision.go b/backend/dao/model/storage_decision.go new file mode 100644 index 000000000..a8fba911b --- /dev/null +++ b/backend/dao/model/storage_decision.go @@ -0,0 +1,62 @@ +package model + +import ( + "time" + + "gorm.io/datatypes" + "gorm.io/gorm" +) + +type StorageDecisionStatus string + +const ( + StorageDecisionStatusPending StorageDecisionStatus = "pending" + StorageDecisionStatusRunning StorageDecisionStatus = "running" + StorageDecisionStatusDone StorageDecisionStatus = "done" + StorageDecisionStatusError StorageDecisionStatus = "error" +) + +type StorageDecisionSource string + +const ( + StorageDecisionSourceManual StorageDecisionSource = "manual" + StorageDecisionSourcePatrol StorageDecisionSource = "patrol" + StorageDecisionSourceReplay StorageDecisionSource = "replay" +) + +// StorageDecisionRecord stores the full decision trail of a storage governance run. +// It keeps the input snapshot, the raw LLM proposal, and the final decision after +// applying platform safety constraints so the result can be audited and replayed. +type StorageDecisionRecord struct { + gorm.Model + JobID string `gorm:"type:varchar(64);not null;uniqueIndex;comment:决策任务ID" json:"jobId"` + UserID uint `gorm:"index;comment:用户ID" json:"userId"` + Username string `gorm:"type:varchar(64);not null;index;comment:用户名" json:"username"` + Source StorageDecisionSource `gorm:"type:varchar(32);not null;index;comment:触发来源" json:"source"` + Status StorageDecisionStatus `gorm:"type:varchar(32);not null;index;default:pending;comment:决策状态" json:"status"` + TriggerReason string `gorm:"type:text;comment:触发原因" json:"triggerReason"` + + Snapshot datatypes.JSON `gorm:"type:jsonb;comment:决策时的输入快照" json:"snapshot"` + RawDecision datatypes.JSON `gorm:"type:jsonb;comment:LLM原始决策" json:"rawDecision"` + FinalDecision datatypes.JSON `gorm:"type:jsonb;comment:约束校验后的最终决策" json:"finalDecision"` + ConstraintResult datatypes.JSON `gorm:"type:jsonb;comment:安全约束评估结果" json:"constraintResult"` + + RawAllowExpand bool `gorm:"not null;default:false;comment:LLM原始是否允许扩容" json:"rawAllowExpand"` + RawExpandBytes int64 `gorm:"not null;default:0;comment:LLM原始建议扩容量" json:"rawExpandBytes"` + RawFreezeNewJobs bool `gorm:"not null;default:false;comment:LLM原始是否冻结新作业" json:"rawFreezeNewJobs"` + FinalAllowExpand bool `gorm:"not null;default:false;comment:最终是否允许扩容" json:"finalAllowExpand"` + FinalExpandBytes int64 `gorm:"not null;default:0;comment:最终扩容量" json:"finalExpandBytes"` + FinalFreezeNewJobs bool `gorm:"not null;default:false;comment:最终是否冻结新作业" json:"finalFreezeNewJobs"` + ConstraintAdjusted bool `gorm:"not null;default:false;comment:安全约束是否调整了决策" json:"constraintAdjusted"` + ConstraintBlocked bool `gorm:"not null;default:false;comment:安全约束是否阻断了扩容" json:"constraintBlocked"` + AppliedAction string `gorm:"type:varchar(64);comment:最终动作摘要" json:"appliedAction"` + ErrorMessage string `gorm:"type:text;comment:错误信息" json:"errorMessage"` + StartedAt *time.Time `gorm:"comment:开始时间" json:"startedAt"` + FinishedAt *time.Time `gorm:"comment:结束时间" json:"finishedAt"` + LatencyMs int64 `gorm:"not null;default:0;comment:决策耗时毫秒" json:"latencyMs"` + ConstraintVersion string `gorm:"type:varchar(64);comment:约束策略版本" json:"constraintVersion"` +} + +func (StorageDecisionRecord) TableName() string { + return "storage_decision_records" +} diff --git a/backend/dao/model/storage_index.go b/backend/dao/model/storage_index.go new file mode 100644 index 000000000..dfb6b9574 --- /dev/null +++ b/backend/dao/model/storage_index.go @@ -0,0 +1,256 @@ +//nolint:lll // GORM tags encode storage index schema metadata inline with each field. +package model + +import "time" + +type StorageIndexWorkspaceType string + +const ( + StorageIndexWorkspaceTypeUser StorageIndexWorkspaceType = "user" + StorageIndexWorkspaceTypeAccount StorageIndexWorkspaceType = "account" + StorageIndexWorkspaceTypePublic StorageIndexWorkspaceType = "public" +) + +type StorageIndexScanStatus string + +const ( + StorageIndexScanStatusPending StorageIndexScanStatus = "pending" + StorageIndexScanStatusRunning StorageIndexScanStatus = "running" + StorageIndexScanStatusDone StorageIndexScanStatus = "done" + StorageIndexScanStatusError StorageIndexScanStatus = "error" +) + +type StorageIndexScanMode string + +const ( + StorageIndexScanModeFull StorageIndexScanMode = "full" + StorageIndexScanModeDailyRefresh StorageIndexScanMode = "daily_refresh" +) + +type StorageIndexEntryType string + +const ( + StorageIndexEntryTypeFile StorageIndexEntryType = "file" + StorageIndexEntryTypeDir StorageIndexEntryType = "dir" + StorageIndexEntryTypeSymlink StorageIndexEntryType = "symlink" + StorageIndexEntryTypeOther StorageIndexEntryType = "other" +) + +type StorageIndexRedundancyTargetType string + +const ( + StorageIndexRedundancyTargetTypeFile StorageIndexRedundancyTargetType = "file" + StorageIndexRedundancyTargetTypeDirectory StorageIndexRedundancyTargetType = "directory" +) + +type StorageIndexVerificationStatus string + +const ( + StorageIndexVerificationStatusSuspected StorageIndexVerificationStatus = "suspected" + StorageIndexVerificationStatusVerified StorageIndexVerificationStatus = "verified" +) + +// StorageIndexScanJob records a metadata indexing run for a workspace. +type StorageIndexScanJob struct { + ID uint `gorm:"primaryKey" json:"id"` + ScanID string `gorm:"type:varchar(64);not null;uniqueIndex;comment:扫描任务ID" json:"scanId"` + WorkspaceType StorageIndexWorkspaceType `gorm:"type:varchar(16);not null;index;comment:工作空间类型" json:"workspaceType"` + WorkspaceName string `gorm:"type:varchar(128);not null;index;comment:工作空间名称" json:"workspaceName"` + LogicalPath string `gorm:"type:text;not null;comment:逻辑扫描路径" json:"logicalPath"` + SnapshotName string `gorm:"type:varchar(128);comment:扫描使用的快照名称" json:"snapshotName"` + MaterializedSnapshotName string `gorm:"type:varchar(160);comment:实际物化快照目录名称" json:"materializedSnapshotName"` + ScanRoot string `gorm:"type:text;comment:实际扫描根路径" json:"scanRoot"` + TriggerSource string `gorm:"type:varchar(32);not null;default:manual;comment:触发来源" json:"triggerSource"` + ScanMode StorageIndexScanMode `gorm:"type:varchar(32);not null;default:full;comment:扫描模式" json:"scanMode"` + BaseScanID string `gorm:"type:varchar(64);index;comment:差异比对基线扫描ID" json:"baseScanId"` + DiffMethod string `gorm:"type:varchar(32);comment:差异计算方式" json:"diffMethod"` + Status StorageIndexScanStatus `gorm:"type:varchar(16);not null;index;default:pending;comment:扫描状态" json:"status"` + EntryCount int64 `gorm:"not null;default:0;comment:入库条目数" json:"entryCount"` + FileCount int64 `gorm:"not null;default:0;comment:文件条目数" json:"fileCount"` + DirectoryCount int64 `gorm:"not null;default:0;comment:目录条目数" json:"directoryCount"` + TotalSizeBytes int64 `gorm:"not null;default:0;comment:工作空间总大小" json:"totalSizeBytes"` + ChangedPathCount int64 `gorm:"not null;default:0;comment:与基线相比的变化目录数" json:"changedPathCount"` + RedundancyCount int64 `gorm:"not null;default:0;comment:冗余命中数" json:"redundancyCount"` + RedundancyBytes int64 `gorm:"not null;default:0;comment:冗余空间估算字节数" json:"redundancyBytes"` + ErrorMessage string `gorm:"type:text;comment:错误信息" json:"errorMessage"` + StartedAt *time.Time `gorm:"comment:开始时间" json:"startedAt"` + FinishedAt *time.Time `gorm:"comment:结束时间" json:"finishedAt"` + LatencyMs int64 `gorm:"not null;default:0;comment:耗时毫秒" json:"latencyMs"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} + +func (StorageIndexScanJob) TableName() string { + return "storage_index_scan_jobs" +} + +// StorageIndexEntry stores the latest indexed path metadata for a workspace. +type StorageIndexEntry struct { + ID uint `gorm:"primaryKey" json:"id"` + WorkspaceType StorageIndexWorkspaceType `gorm:"type:varchar(16);not null;index;comment:工作空间类型" json:"workspaceType"` + WorkspaceName string `gorm:"type:varchar(128);not null;index;comment:工作空间名称" json:"workspaceName"` + ScanID string `gorm:"type:varchar(64);not null;index;comment:来源扫描任务ID" json:"scanId"` + LogicalPath string `gorm:"type:text;not null;comment:逻辑路径" json:"logicalPath"` + RelativePath string `gorm:"type:text;not null;comment:相对路径" json:"relativePath"` + ParentPath string `gorm:"type:text;comment:父路径" json:"parentPath"` + Name string `gorm:"type:varchar(512);not null;index;comment:对象名称" json:"name"` + EntryType StorageIndexEntryType `gorm:"type:varchar(16);not null;index;comment:对象类型" json:"entryType"` + SizeBytes int64 `gorm:"not null;default:0;comment:对象大小" json:"sizeBytes"` + OwnerUID int64 `gorm:"not null;default:0;comment:属主UID" json:"ownerUid"` + OwnerGID int64 `gorm:"not null;default:0;comment:属组GID" json:"ownerGid"` + Mode string `gorm:"type:varchar(16);comment:权限位" json:"mode"` + LinkCount int64 `gorm:"not null;default:0;comment:链接数" json:"linkCount"` + ModifiedAt *time.Time `gorm:"comment:mtime" json:"modifiedAt"` + ChangedAt *time.Time `gorm:"comment:ctime" json:"changedAt"` + AccessedAt *time.Time `gorm:"comment:atime" json:"accessedAt"` + IsTopLevel bool `gorm:"not null;default:false;index;comment:是否根目录直系子节点" json:"isTopLevel"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} + +func (StorageIndexEntry) TableName() string { + return "storage_index_entries" +} + +// StorageIndexDirectoryMetric stores aggregated directory metrics for a workspace. +type StorageIndexDirectoryMetric struct { + ID uint `gorm:"primaryKey" json:"id"` + WorkspaceType StorageIndexWorkspaceType `gorm:"type:varchar(16);not null;index;comment:工作空间类型" json:"workspaceType"` + WorkspaceName string `gorm:"type:varchar(128);not null;index;comment:工作空间名称" json:"workspaceName"` + ScanID string `gorm:"type:varchar(64);not null;index;comment:来源扫描任务ID" json:"scanId"` + Path string `gorm:"type:text;not null;comment:目录路径" json:"path"` + ParentPath string `gorm:"type:text;comment:父目录路径" json:"parentPath"` + Name string `gorm:"type:varchar(512);not null;index;comment:目录名" json:"name"` + Depth int `gorm:"not null;default:0;comment:目录深度" json:"depth"` + IsTopLevel bool `gorm:"not null;default:false;index;comment:是否根目录直系子目录" json:"isTopLevel"` + FileCount int64 `gorm:"not null;default:0;comment:子树文件数" json:"fileCount"` + DirectoryCount int64 `gorm:"not null;default:0;comment:子树目录数" json:"directoryCount"` + TotalSizeBytes int64 `gorm:"not null;default:0;comment:子树累计大小" json:"totalSizeBytes"` + LatestGrowth int64 `gorm:"not null;default:0;comment:最近增量字节数" json:"latestGrowth"` + ImmediateChildDirCount int64 `gorm:"not null;default:0;comment:直接子目录数" json:"immediateChildDirCount"` + ImmediateChildFileCount int64 `gorm:"not null;default:0;comment:直接子文件数" json:"immediateChildFileCount"` + LatestModifiedAt *time.Time `gorm:"comment:目录最近修改时间" json:"latestModifiedAt"` + Signature string `gorm:"type:varchar(128);index;comment:目录签名" json:"signature"` + CategoryHint string `gorm:"type:varchar(64);index;comment:目录类别提示" json:"categoryHint"` + CandidateScore float64 `gorm:"not null;default:0;comment:候选目录评分" json:"candidateScore"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} + +func (StorageIndexDirectoryMetric) TableName() string { + return "storage_index_directory_metrics" +} + +type StorageIndexCandidateStatus string + +const ( + StorageIndexCandidateStatusSuspected StorageIndexCandidateStatus = "suspected" + StorageIndexCandidateStatusVerified StorageIndexCandidateStatus = "verified" + StorageIndexCandidateStatusRejected StorageIndexCandidateStatus = "rejected" +) + +type StorageIndexCandidate struct { + ID uint `gorm:"primaryKey" json:"id"` + WorkspaceType StorageIndexWorkspaceType `gorm:"type:varchar(16);not null;index;comment:工作空间类型" json:"workspaceType"` + WorkspaceName string `gorm:"type:varchar(128);not null;index;comment:工作空间名称" json:"workspaceName"` + ScanID string `gorm:"type:varchar(64);not null;index;comment:来源扫描任务ID" json:"scanId"` + CandidateType string `gorm:"type:varchar(32);index;comment:候选目录类型" json:"candidateType"` + TargetPath string `gorm:"type:text;not null;comment:候选目录路径" json:"targetPath"` + PublicPath string `gorm:"type:text;comment:匹配的公共空间目录路径" json:"publicPath"` + Evidence string `gorm:"type:text;comment:候选依据" json:"evidence"` + CandidateScore float64 `gorm:"not null;default:0;comment:候选评分" json:"candidateScore"` + Status StorageIndexCandidateStatus `gorm:"type:varchar(32);not null;default:suspected;index;comment:候选状态" json:"status"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} + +func (StorageIndexCandidate) TableName() string { + return "storage_index_candidates" +} + +type StorageIndexCandidateFile struct { + ID uint `gorm:"primaryKey" json:"id"` + WorkspaceType StorageIndexWorkspaceType `gorm:"type:varchar(16);not null;index;comment:工作空间类型" json:"workspaceType"` + WorkspaceName string `gorm:"type:varchar(128);not null;index;comment:工作空间名称" json:"workspaceName"` + ScanID string `gorm:"type:varchar(64);not null;index;comment:来源扫描任务ID" json:"scanId"` + CandidatePath string `gorm:"type:text;not null;comment:候选目录路径" json:"candidatePath"` + FilePath string `gorm:"type:text;not null;comment:候选文件路径" json:"filePath"` + FileName string `gorm:"type:varchar(512);index;comment:候选文件名" json:"fileName"` + RelativePath string `gorm:"type:text;comment:候选文件相对路径" json:"relativePath"` + SizeBytes int64 `gorm:"not null;default:0;comment:候选文件大小" json:"sizeBytes"` + MatchedPublicFile string `gorm:"type:text;comment:匹配的公共文件路径" json:"matchedPublicFile"` + HashAlgorithm string `gorm:"type:varchar(32);comment:哈希算法" json:"hashAlgorithm"` + FileHash string `gorm:"type:varchar(128);comment:候选文件哈希" json:"fileHash"` + VerificationStatus StorageIndexVerificationStatus `gorm:"type:varchar(32);not null;default:suspected;index;comment:校验状态" json:"verificationStatus"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} + +func (StorageIndexCandidateFile) TableName() string { + return "storage_index_candidate_files" +} + +type StorageIndexPublicRootBaseline struct { + ID uint `gorm:"primaryKey" json:"id"` + ScanID string `gorm:"type:varchar(64);not null;index;comment:来源扫描任务ID" json:"scanId"` + ResourceName string `gorm:"type:varchar(512);not null;index;comment:公共资源名称" json:"resourceName"` + LogicalPath string `gorm:"type:text;not null;comment:公共资源逻辑路径" json:"logicalPath"` + RootHash string `gorm:"type:varchar(128);index;comment:公共资源根目录哈希" json:"rootHash"` + Category string `gorm:"type:varchar(64);index;comment:公共资源类别" json:"category"` + TotalSizeBytes int64 `gorm:"not null;default:0;comment:资源总大小" json:"totalSizeBytes"` + KeyFileCount int64 `gorm:"not null;default:0;comment:关键文件数量" json:"keyFileCount"` + Signature string `gorm:"type:varchar(128);index;comment:资源目录签名" json:"signature"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} + +func (StorageIndexPublicRootBaseline) TableName() string { + return "storage_index_public_root_baseline" +} + +type StorageIndexPublicFileBaseline struct { + ID uint `gorm:"primaryKey" json:"id"` + ScanID string `gorm:"type:varchar(64);not null;index;comment:来源扫描任务ID" json:"scanId"` + PublicRootPath string `gorm:"type:text;not null;index;comment:公共基线根目录" json:"publicRootPath"` + PublicRootHash string `gorm:"type:varchar(128);index;comment:公共基线根目录哈希" json:"publicRootHash"` + FilePath string `gorm:"type:text;not null;comment:公共文件路径" json:"filePath"` + FileName string `gorm:"type:varchar(512);index;comment:公共文件名" json:"fileName"` + RelativePath string `gorm:"type:text;index;comment:相对公共根目录的路径" json:"relativePath"` + SizeBytes int64 `gorm:"not null;default:0;comment:文件大小" json:"sizeBytes"` + MatchKey string `gorm:"type:varchar(1024);index;comment:快速匹配键" json:"matchKey"` + MatchKeyHash string `gorm:"type:varchar(128);index;comment:快速匹配键哈希" json:"matchKeyHash"` + HashAlgorithm string `gorm:"type:varchar(32);comment:哈希算法" json:"hashAlgorithm"` + FileHash string `gorm:"type:varchar(128);comment:文件哈希" json:"fileHash"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} + +func (StorageIndexPublicFileBaseline) TableName() string { + return "storage_index_public_file_baseline" +} + +// StorageIndexRedundancyHit stores redundancy findings against the public baseline. +type StorageIndexRedundancyHit struct { + ID uint `gorm:"primaryKey" json:"id"` + WorkspaceType StorageIndexWorkspaceType `gorm:"type:varchar(16);not null;index;comment:工作空间类型" json:"workspaceType"` + WorkspaceName string `gorm:"type:varchar(128);not null;index;comment:工作空间名称" json:"workspaceName"` + ScanID string `gorm:"type:varchar(64);not null;index;comment:来源扫描任务ID" json:"scanId"` + TargetType StorageIndexRedundancyTargetType `gorm:"type:varchar(16);not null;index;comment:冗余对象类型" json:"targetType"` + TargetPath string `gorm:"type:text;not null;comment:工作空间中的冗余路径" json:"targetPath"` + PublicPath string `gorm:"type:text;not null;comment:公共空间基线路径" json:"publicPath"` + MatchKey string `gorm:"type:varchar(512);index;comment:匹配键" json:"matchKey"` + Evidence string `gorm:"type:text;comment:检测依据" json:"evidence"` + Confidence string `gorm:"type:varchar(32);comment:置信级别" json:"confidence"` + VerificationStatus StorageIndexVerificationStatus `gorm:"type:varchar(32);not null;default:suspected;index;comment:校验状态" json:"verificationStatus"` + VerificationMode string `gorm:"type:varchar(32);comment:校验方式" json:"verificationMode"` + HashAlgorithm string `gorm:"type:varchar(32);comment:哈希算法" json:"hashAlgorithm"` + TargetHash string `gorm:"type:varchar(128);comment:工作空间对象哈希" json:"targetHash"` + PublicHash string `gorm:"type:varchar(128);comment:公共空间对象哈希" json:"publicHash"` + EstimatedBytes int64 `gorm:"not null;default:0;comment:估算冗余空间大小" json:"estimatedBytes"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} + +func (StorageIndexRedundancyHit) TableName() string { + return "storage_index_redundancy_hits" +} diff --git a/backend/dao/model/system_config.go b/backend/dao/model/system_config.go index 3224e005c..4860deca0 100644 --- a/backend/dao/model/system_config.go +++ b/backend/dao/model/system_config.go @@ -1,16 +1,14 @@ -// 请将此文件保存为 dao/model/system_config.go - package model -// SystemConfig 用于存储系统级别的键值对配置 +// SystemConfig stores system-wide key-value configuration. type SystemConfig struct { - Key string `gorm:"primarykey;size:100;comment:配置项的键"` - Value string `gorm:"type:text;comment:配置项的值"` + Key string `gorm:"primarykey;size:100;comment:配置项键"` + Value string `gorm:"type:text;comment:配置项值"` } const ( - // LLM 相关配置键 - ConfigKeyLLMBaseURL = "LLM_API_BASE_URL" // 例如: https://api.openai.com/v1 + // Generic LLM configuration keys. + ConfigKeyLLMBaseURL = "LLM_API_BASE_URL" // e.g. https://api.openai.com/v1 ConfigKeyLLMAPIKey = "LLM_API_KEY" // #nosec G101 ConfigKeyLLMModelName = "LLM_MODEL_NAME" @@ -40,13 +38,25 @@ const ( ConfigKeyModelDownloadBandwidth = "POD_BANDWIDTH_MODEL_DOWNLOAD" ConfigKeyJobIngressBandwidth = "POD_BANDWIDTH_JOB_INGRESS" ConfigKeyJobEgressBandwidth = "POD_BANDWIDTH_JOB_EGRESS" + + // Storage decision keys. + ConfigKeyStorageDecisionMode = "STORAGE_DECISION_MODE" + ConfigKeyStorageDecisionConfigSource = "STORAGE_DECISION_CONFIG_SOURCE" + ConfigKeyStorageDirectModelBaseURL = "STORAGE_DIRECT_MODEL_BASE_URL" + ConfigKeyStorageDirectModelAPIKey = "STORAGE_DIRECT_MODEL_API_KEY" // #nosec G101 + ConfigKeyStorageDirectModelName = "STORAGE_DIRECT_MODEL_NAME" ) -// DefaultConfigKeys 定义了系统启动时必须存在的键 +// DefaultConfigKeys defines keys that must exist after startup. var DefaultConfigKeys = []string{ ConfigKeyLLMBaseURL, ConfigKeyLLMAPIKey, ConfigKeyLLMModelName, + ConfigKeyStorageDecisionMode, + ConfigKeyStorageDecisionConfigSource, + ConfigKeyStorageDirectModelBaseURL, + ConfigKeyStorageDirectModelAPIKey, + ConfigKeyStorageDirectModelName, ConfigKeyEnableGpuAnalysis, ConfigKeyEnableBillingFeature, ConfigKeyEnableBillingActive, diff --git a/backend/dao/model/user_space_size.go b/backend/dao/model/user_space_size.go new file mode 100644 index 000000000..b6c469bc8 --- /dev/null +++ b/backend/dao/model/user_space_size.go @@ -0,0 +1,26 @@ +package model + +import ( + "time" +) + +// UserSpaceSize 用户空间大小模型 +type UserSpaceSize struct { + ID uint `gorm:"primaryKey" json:"id"` + UserID uint `gorm:"index" json:"user_id"` + User User `gorm:"foreignKey:UserID" json:"user"` + Username string `gorm:"size:64;not null;uniqueIndex" json:"username"` + Size int64 `json:"size"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +// TenantUsageHistory 租户存储使用历史 +type TenantUsageHistory struct { + ID uint `gorm:"primaryKey" json:"id"` + TenantID uint `gorm:"index" json:"tenant_id"` + UsageBytes int64 `json:"usage_bytes"` + RecordedAt time.Time `json:"recorded_at"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} diff --git a/backend/go.mod b/backend/go.mod index 09d21c0c8..10b9b5771 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -18,6 +18,7 @@ require ( github.com/prometheus/common v0.65.0 github.com/robfig/cron/v3 v3.0.0 github.com/samber/lo v1.51.0 + github.com/sashabaranov/go-openai v1.41.2 github.com/smartystreets/goconvey v1.8.1 github.com/swaggo/files v1.0.1 github.com/swaggo/gin-swagger v1.6.0 diff --git a/backend/go.sum b/backend/go.sum index 23e0eb304..b906fbec5 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -269,6 +269,8 @@ github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/samber/lo v1.51.0 h1:kysRYLbHy/MB7kQZf5DSN50JHmMsNEdeY24VzJFu7wI= github.com/samber/lo v1.51.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0= +github.com/sashabaranov/go-openai v1.41.2 h1:vfPRBZNMpnqu8ELsclWcAvF19lDNgh1t6TVfFFOPiSM= +github.com/sashabaranov/go-openai v1.41.2/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= diff --git a/backend/hack/storage-index-experiment-toolbox.sh b/backend/hack/storage-index-experiment-toolbox.sh new file mode 100644 index 000000000..de52e9286 --- /dev/null +++ b/backend/hack/storage-index-experiment-toolbox.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Usage: +# bash storage-index-experiment-toolbox.sh /mnt/mycephfs/.../user-space-root +# +# Notes: +# - ROOT must be the actual CephFS path visible inside rook-ceph-tools. +# - Recursive stats come from CephFS virtual xattrs: +# ceph.dir.rfiles / ceph.dir.rsubdirs / ceph.dir.rentries / ceph.dir.rbytes +# - find is only used to enumerate top-level directory names for applying the +# same allow/block filter as the service implementation. + +ROOT="${1:-}" +if [[ -z "$ROOT" ]]; then + echo "Usage: bash storage-index-experiment-toolbox.sh " >&2 + exit 1 +fi + +if [[ ! -d "$ROOT" ]]; then + echo "error: ROOT is not a directory: $ROOT" >&2 + exit 1 +fi + +SKIP_RE='^(conda|\.conda|miniconda3|anaconda3|mambaforge|\.mamba|micromamba|venv|\.venv|\.git|\.svn|\.hg|\.idea|\.vscode|\.vscode-server|\.ipynb_checkpoints|\.npm|\.yarn|\.pnpm-store|\.cargo|\.m2|\.gradle|\.pytest_cache|\.mypy_cache|__pycache__)$' + +get_dir_stat() { + local attr="$1" + local target="$2" + getfattr --only-values -n "$attr" "$target" 2>/dev/null || echo 0 +} + +TOTAL_BYTES="$(get_dir_stat ceph.dir.rbytes "$ROOT")" +TOTAL_FILES="$(get_dir_stat ceph.dir.rfiles "$ROOT")" +TOTAL_DIRS="$(get_dir_stat ceph.dir.rsubdirs "$ROOT")" +TOTAL_ENTRIES="$(get_dir_stat ceph.dir.rentries "$ROOT")" + +mapfile -t TOP_LEVEL_DIRS < <(find "$ROOT" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | sort) + +SELECTED=() +SKIPPED=() +for name in "${TOP_LEVEL_DIRS[@]}"; do + if [[ "$name" =~ $SKIP_RE ]]; then + SKIPPED+=("$name") + else + SELECTED+=("$name") + fi +done + +FILTERED_BYTES=0 +FILTERED_FILES=0 +FILTERED_DIRS=0 +FILTERED_ENTRIES=0 + +for name in "${SELECTED[@]}"; do + subtree="$ROOT/$name" + bytes="$(get_dir_stat ceph.dir.rbytes "$subtree")" + files="$(get_dir_stat ceph.dir.rfiles "$subtree")" + subdirs="$(get_dir_stat ceph.dir.rsubdirs "$subtree")" + entries="$(get_dir_stat ceph.dir.rentries "$subtree")" + FILTERED_BYTES=$((FILTERED_BYTES + bytes)) + FILTERED_FILES=$((FILTERED_FILES + files)) + # rsubdirs excludes the subtree root itself, while the filtered scan result + # counts each selected top-level directory as an indexed directory node. + FILTERED_DIRS=$((FILTERED_DIRS + subdirs + 1)) + FILTERED_ENTRIES=$((FILTERED_ENTRIES + entries + 1)) +done + +echo "workspace_root=$ROOT" +echo "total_bytes=$TOTAL_BYTES" +echo "total_file_count=$TOTAL_FILES" +echo "total_directory_count=$TOTAL_DIRS" +echo "total_entry_count=$TOTAL_ENTRIES" +echo "filtered_bytes=$FILTERED_BYTES" +echo "filtered_file_count=$FILTERED_FILES" +echo "filtered_directory_count=$FILTERED_DIRS" +echo "filtered_entry_count=$FILTERED_ENTRIES" +echo "top_level_candidate_dir_count=${#TOP_LEVEL_DIRS[@]}" +echo "selected_top_level_dir_count=${#SELECTED[@]}" +echo "skipped_top_level_dir_count=${#SKIPPED[@]}" +echo "selected_top_level_dir_names=$(IFS=,; echo "${SELECTED[*]-}")" +echo "skipped_top_level_dir_names=$(IFS=,; echo "${SKIPPED[*]-}")" diff --git a/backend/internal/handler/aijob/new.go b/backend/internal/handler/aijob/new.go index 9513b12d8..ffdddfd95 100644 --- a/backend/internal/handler/aijob/new.go +++ b/backend/internal/handler/aijob/new.go @@ -139,6 +139,12 @@ func (mgr *AIJobMgr) CreateJupyterJob(c *gin.Context) { taskModel.PodTemplate = datatypes.NewJSONType(podSpec) taskModel.Owner = token.Username + + if err := interutil.CheckStorageQuota(token.Username); err != nil { + resputil.Error(c, err.Error(), resputil.NotSpecified) + return + } + err = mgr.taskService.Create(taskModel) if err != nil { resputil.Error(c, fmt.Sprintf("create task failed, err %v", err), resputil.NotSpecified) @@ -205,6 +211,12 @@ func (mgr *AIJobMgr) CreateCustom(c *gin.Context) { taskModel.PodTemplate = datatypes.NewJSONType(podSpec) taskModel.Owner = token.Username + + if err := interutil.CheckStorageQuota(token.Username); err != nil { + resputil.Error(c, err.Error(), resputil.NotSpecified) + return + } + err = mgr.taskService.Create(taskModel) if err != nil { resputil.Error(c, fmt.Sprintf("create task failed, err %v", err), resputil.NotSpecified) diff --git a/backend/internal/handler/operations/cronjob.go b/backend/internal/handler/operations/cronjob.go index 372b456c5..af9604982 100644 --- a/backend/internal/handler/operations/cronjob.go +++ b/backend/internal/handler/operations/cronjob.go @@ -1,18 +1,22 @@ package operations import ( + "context" "encoding/json" "fmt" "time" "github.com/gin-gonic/gin" "github.com/samber/lo" + "gorm.io/datatypes" "k8s.io/klog/v2" "k8s.io/utils/ptr" "github.com/raids-lab/crater/dao/model" + "github.com/raids-lab/crater/dao/query" "github.com/raids-lab/crater/internal/resputil" "github.com/raids-lab/crater/pkg/patrol" + "github.com/raids-lab/crater/pkg/util" ) // UpdateCronjobConfig godoc @@ -289,3 +293,94 @@ func (cm *OperationsMgr) GetLastCronjobRecord(c *gin.Context) { resputil.Success(c, records) } + +// ExecutePatrolJob godoc +// +// @Summary Execute patrol job +// @Description Execute a patrol job immediately +// @Tags Operations +// @Accept json +// @Produce json +// @Security Bearer +// @Param jobName body string true "Job name" +// @Success 200 {object} resputil.Response[any] "Success" +// @Failure 400 {object} resputil.Response[any] "Request parameter error" +// @Failure 500 {object} resputil.Response[any] "Other errors" +// @Router /v1/operations/cronjob/execute [post] +func (mgr *OperationsMgr) ExecutePatrolJob(c *gin.Context) { + var req struct { + JobName string `json:"jobName" binding:"required"` + } + if err := c.ShouldBindJSON(&req); err != nil { + resputil.Error(c, err.Error(), resputil.InvalidRequest) + return + } + + // 获取巡检函数 + var f util.AnyFunc + var err error + switch req.JobName { + case patrol.UPDATE_USER_SPACE_SIZE: + f, err = patrol.GetPatrolFunc(req.JobName, mgr.cronJobManager.GetPatrolClients(), nil) + case patrol.ANALYZE_STORAGE_ALERTS: + f, err = patrol.GetPatrolFunc(req.JobName, mgr.cronJobManager.GetPatrolClients(), nil) + case patrol.TRIGGER_GPU_ANALYSIS_JOB: + f, err = patrol.GetPatrolFunc(req.JobName, mgr.cronJobManager.GetPatrolClients(), nil) + case patrol.REFRESH_PUBLIC_STORAGE_INDEX: + f, err = patrol.GetPatrolFunc(req.JobName, mgr.cronJobManager.GetPatrolClients(), nil) + case patrol.REFRESH_USER_STORAGE_INDEX: + f, err = patrol.GetPatrolFunc(req.JobName, mgr.cronJobManager.GetPatrolClients(), nil) + default: + resputil.Error(c, "Unsupported patrol job: "+req.JobName, resputil.InvalidRequest) + return + } + + if err != nil { + resputil.Error(c, err.Error(), resputil.ServiceError) + return + } + + // 异步执行巡检任务 + go func() { + defer func() { + if r := recover(); r != nil { + klog.Errorf("ExecutePatrolJob: panic in patrol job %s: %v", req.JobName, r) + } + }() + + ctx := context.Background() + executeTime := time.Now() + jobResult, err := f(ctx) + + status := model.CronJobRecordStatusSuccess + if err != nil { + status = model.CronJobRecordStatusFailed + klog.Errorf("ExecutePatrolJob: patrol job %s failed: %v", req.JobName, err) + } else { + klog.Infof("ExecutePatrolJob: patrol job %s completed: %v", req.JobName, jobResult) + } + + // 保存执行记录 + rec := &model.CronJobRecord{ + Name: req.JobName, + ExecuteTime: executeTime, + Message: fmt.Sprintf("manual execute patrol job: %s", req.JobName), + Status: status, + } + + if jobResult != nil { + if data, err := json.Marshal(jobResult); err != nil { + klog.Errorf("ExecutePatrolJob: failed to marshal job result: %v", err) + } else { + rec.JobData = datatypes.JSON(data) + } + } + + if err := query.GetDB().WithContext(ctx).Model(rec).Create(rec).Error; err != nil { + klog.Errorf("ExecutePatrolJob: failed to create record: %v", err) + } + }() + + // 立即返回成功响应 + resputil.Success(c, "任务已开始执行，请稍后在执行记录中查看结果") +} diff --git a/backend/internal/handler/operations/operations.go b/backend/internal/handler/operations/operations.go index 4226b608a..2b40b37ae 100644 --- a/backend/internal/handler/operations/operations.go +++ b/backend/internal/handler/operations/operations.go @@ -71,6 +71,7 @@ func (mgr *OperationsMgr) RegisterAdmin(g *gin.RouterGroup) { g.POST("/cronjob/record/time", mgr.GetCronjobRecordTimeRange) g.POST("/cronjob/record/list", mgr.GetCronjobRecords) g.POST("/cronjob/record/delete", mgr.DeleteCronjobRecords) + g.POST("/cronjob/execute", mgr.ExecutePatrolJob) } func (cm *OperationsMgr) StopCron() { diff --git a/backend/internal/handler/spjob/spjob.go b/backend/internal/handler/spjob/spjob.go index 231fb98e3..b5a394765 100644 --- a/backend/internal/handler/spjob/spjob.go +++ b/backend/internal/handler/spjob/spjob.go @@ -128,6 +128,11 @@ func (mgr *SparseJobMgr) Create(c *gin.Context) { return } + if err := util.CheckStorageQuota(token.Username); err != nil { + resputil.Error(c, err.Error(), resputil.NotSpecified) + return + } + volumes, volumeMounts, err := vcjob.GenerateVolumeMounts(c, req.VolumeMounts, token) if err != nil { resputil.Error(c, err.Error(), resputil.NotSpecified) diff --git a/backend/internal/handler/storage.go b/backend/internal/handler/storage.go new file mode 100644 index 000000000..9d8618f52 --- /dev/null +++ b/backend/internal/handler/storage.go @@ -0,0 +1,1085 @@ +package handler + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + "github.com/gin-gonic/gin" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + + "github.com/raids-lab/crater/dao/model" + "github.com/raids-lab/crater/dao/query" + "github.com/raids-lab/crater/internal/resputil" + "github.com/raids-lab/crater/internal/util" + "github.com/raids-lab/crater/pkg/ceph" + "github.com/raids-lab/crater/pkg/config" + "github.com/raids-lab/crater/pkg/monitor" + "github.com/raids-lab/crater/pkg/patrol" + "github.com/raids-lab/crater/pkg/storagegovernance" + "github.com/raids-lab/crater/pkg/storageindex" +) + +// ---- LLM 任务状态存储 ---- + +//nolint:gochecknoinits // This is the standard way to register a gin handler. +func init() { + Registers = append(Registers, NewStorageMgr) +} + +type StorageMgr struct { + name string + kubeClient kubernetes.Interface + kubeConfig *rest.Config + promClient monitor.PrometheusInterface +} + +// AutoScaleRequest 自动扩缩容请求 +type AutoScaleRequest struct { + MinQuota int64 `json:"min_quota" binding:"required,min=-1"` // 最小配额，-1 表示无限制 + MaxQuota int64 `json:"max_quota" binding:"required,min=-1"` // 最大配额，-1 表示无限制 + ScaleUpRatio float64 `json:"scale_up_ratio" binding:"required,min=1"` // 扩容比例，如 1.5 表示扩容到当前使用的 1.5 倍 + ScaleDownRatio float64 `json:"scale_down_ratio" binding:"required,min=0.1,max=1"` // 缩容比例，如 0.8 表示缩容到当前使用的 0.8 倍 +} + +func NewStorageMgr(conf *RegisterConfig) Manager { + return &StorageMgr{ + name: "storage", + kubeClient: conf.KubeClient, + kubeConfig: conf.KubeConfig, + promClient: conf.PrometheusClient, + } +} + +func (mgr *StorageMgr) GetName() string { return mgr.name } + +func (mgr *StorageMgr) RegisterPublic(_ *gin.RouterGroup) {} + +func (mgr *StorageMgr) RegisterProtected(g *gin.RouterGroup) { + g.GET("/dirsize/*path", mgr.GetDirectorySize) + g.GET("/my-quota", mgr.GetMyQuota) +} + +func (mgr *StorageMgr) RegisterAdmin(g *gin.RouterGroup) { + g.GET("/user-spaces", mgr.GetAllUserSpaceSizes) + g.PUT("/user-spaces/:user/quota", mgr.SetUserSpaceQuota) + g.POST("/user-spaces/:user/autoscale", mgr.AutoScaleUserSpaceQuota) + g.POST("/auto-shrink", mgr.RunAutoShrink) + g.POST("/user-spaces/:user/apply-expansion", mgr.ApplyExpansion) + g.POST("/user-spaces/:user/freeze-jobs", mgr.FreezeJobs) + g.POST("/user-spaces/:user/revert-expansion", mgr.RevertExpansion) + g.POST("/user-spaces/:user/unfreeze-jobs", mgr.UnfreezeJobs) + g.GET("/decisions", mgr.ListStorageDecisions) + g.POST("/decisions/replay", mgr.ReplayStorageDecisions) + g.GET("/decisions/:job_id", mgr.GetStorageDecision) + g.POST("/user-spaces/:user/llm-decision", mgr.TriggerLLMDecision) + g.GET("/user-spaces/:user/llm-decision/:job_id", mgr.GetLLMDecisionStatus) + g.POST("/index/scan", mgr.TriggerMetadataIndexScan) + g.GET("/index/scans/:scan_id", mgr.GetMetadataIndexScan) + g.GET("/index/workspaces/:workspace_type/:workspace_name/overview", mgr.GetMetadataWorkspaceOverview) + g.GET("/index/workspaces/:workspace_type/:workspace_name/redundancy-hits", mgr.ListMetadataWorkspaceRedundancyHits) + g.GET("/index/workspaces/:workspace_type/:workspace_name/candidates", mgr.ListMetadataWorkspaceCandidates) + g.GET("/index/workspaces/:workspace_type/:workspace_name/candidate-files", mgr.ListMetadataWorkspaceCandidateFiles) + g.POST("/index/compare-folders", mgr.CompareMetadataFolders) + g.GET("/index/compare-folders/:job_id", mgr.GetMetadataFolderCompareJob) +} + +// GetDirectorySize godoc +// +// @Summary Get directory size in CephFS +// @Description Get the size of a directory in CephFS using getfattr command +// @Tags Storage +// @Accept json +// @Produce json +// @Security Bearer +// @Param path path string true "Directory path" +// @Success 200 {object} resputil.Response[any] "Success" +// @Failure 400 {object} resputil.Response[any] "Request parameter error" +// @Failure 500 {object} resputil.Response[any] "Other errors" +// @Router /v1/storage/dirsize/{path} [get] +func (mgr *StorageMgr) GetDirectorySize(c *gin.Context) { + // 1. 获取路径参数 + path := strings.TrimPrefix(c.Request.URL.Path, "/api/v1/storage/dirsize/") + if path == "" { + resputil.BadRequestError(c, "路径不能为空") + return + } + + // 2. 确保路径以 / 开头 + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + + // 3. 执行 Ceph 命令获取目录大小 + cfg := config.GetConfig() + prefixConfig := ceph.StoragePrefixConfig{ + User: cfg.Storage.Prefix.User, + Account: cfg.Storage.Prefix.Account, + Public: cfg.Storage.Prefix.Public, + } + size, err := ceph.GetCephDirectorySize(mgr.kubeClient, mgr.kubeConfig, "rook-ceph", path, prefixConfig) + if err != nil { + klog.Warningf("GetDirectorySize: failed to get size for %q, returning unknown sentinel: %v", path, err) + size = -1 + } + + // 4. 返回结果 + resputil.Success(c, gin.H{ + "path": path, + "size": size, + "unit": "bytes", + "formatted": formatSize(size), + }) +} + +// GetMyQuota godoc +// +// @Summary Get current user's storage quota +// @Description Get the storage quota for the currently authenticated user +// @Tags Storage +// @Produce json +// @Security Bearer +// @Success 200 {object} resputil.Response[any] "Success" +// @Failure 500 {object} resputil.Response[any] "Other errors" +// @Router /v1/storage/my-quota [get] +func (mgr *StorageMgr) GetMyQuota(c *gin.Context) { + token := util.GetToken(c) + + var row struct { + SpaceQuota int64 `gorm:"column:space_quota"` + } + if err := query.GetDB().Raw( + "SELECT space_quota FROM users WHERE id = ? AND deleted_at IS NULL", token.UserID, + ).Scan(&row).Error; err != nil { + resputil.Error(c, fmt.Sprintf("获取配额失败: %v", err), resputil.NotSpecified) + return + } + + resputil.Success(c, gin.H{ + "space_quota": row.SpaceQuota, + "space_quota_formatted": formatSize(row.SpaceQuota), + }) +} + +// GetAllUserSpaceSizes godoc +// +// @Summary Get all user space sizes +// @Description Get the size of all user spaces from database +// @Tags Storage +// @Accept json +// @Produce json +// @Security Bearer +// @Param page query int false "Page number" +// @Param pageSize query int false "Page size" +// @Success 200 {object} resputil.Response[any] "Success" +// @Failure 500 {object} resputil.Response[any] "Other errors" +// @Router /v1/storage/admin/user-spaces [get] +func (mgr *StorageMgr) GetAllUserSpaceSizes(c *gin.Context) { + // 1. 获取分页参数 + page, _ := strconv.Atoi(c.DefaultQuery("page", "1")) + pageSize, _ := strconv.Atoi(c.DefaultQuery("pageSize", "10")) + + // 2. 从数据库中获取用户空间大小和配额 + type UserSpaceInfo struct { + model.UserSpaceSize + Username string `json:"username"` + SpaceQuota int64 `json:"space_quota"` + OriginalSpaceQuota *int64 `json:"original_space_quota"` + JobsFrozen bool `json:"jobs_frozen"` + ShrinkStage string `json:"shrink_stage"` + } + + var userSpaceInfos []UserSpaceInfo + var total int64 + + db := query.GetDB() + + // 计算总数 + if err := db.Model(&model.UserSpaceSize{}).Count(&total).Error; err != nil { + resputil.Error(c, fmt.Sprintf("获取用户空间大小总数失败: %v", err), resputil.NotSpecified) + return + } + + // 计算分页偏移量 + offset := (page - 1) * pageSize + + // 获取分页数据，关联 User 表获取 SpaceQuota 和 OriginalSpaceQuota + if err := db.Table("user_space_sizes"). + Select( + "user_space_sizes.*, " + + "users.space_quota as space_quota, " + + "users.original_space_quota as original_space_quota, " + + "users.name as username, " + + "users.jobs_frozen as jobs_frozen, " + + "users.shrink_stage as shrink_stage", + ). + Joins("LEFT JOIN users ON user_space_sizes.user_id = users.id"). + Offset(offset).Limit(pageSize). + Find(&userSpaceInfos).Error; err != nil { + resputil.Error(c, fmt.Sprintf("获取用户空间大小失败: %v", err), resputil.NotSpecified) + return + } + + // 3. 格式化结果 + formattedUserSpaces := make([]map[string]any, 0, len(userSpaceInfos)) + for i := range userSpaceInfos { + info := userSpaceInfos[i] + item := map[string]any{ + "user": info.Username, + "size": info.Size, + "quota": info.SpaceQuota, + "unit": "bytes", + "formatted": formatSize(info.Size), + "quota_formatted": formatSize(info.SpaceQuota), + "is_expanded": info.OriginalSpaceQuota != nil, + "jobs_frozen": info.JobsFrozen, + "shrink_stage": info.ShrinkStage, + } + if info.OriginalSpaceQuota != nil { + item["original_quota"] = *info.OriginalSpaceQuota + item["original_quota_formatted"] = formatSize(*info.OriginalSpaceQuota) + } + formattedUserSpaces = append(formattedUserSpaces, item) + } + + // 4. 返回结果（包含分页信息） + resputil.Success(c, gin.H{ + "items": formattedUserSpaces, + "total": total, + "page": page, + "pageSize": pageSize, + "totalPages": (int(total) + pageSize - 1) / pageSize, + }) +} + +// SetUserSpaceQuota godoc +// +// @Summary Set user space quota +// @Description Set the space quota for a user +// @Tags Storage +// @Accept json +// @Produce json +// @Security Bearer +// @Param user path string true "Username" +// @Param quota body int64 true "Space quota in bytes, -1 for unlimited" +// @Success 200 {object} resputil.Response[any] "Success" +// @Failure 400 {object} resputil.Response[any] "Request parameter error" +// @Failure 404 {object} resputil.Response[any] "User not found" +// @Failure 500 {object} resputil.Response[any] "Other errors" +// @Router /v1/storage/admin/user-spaces/{user}/quota [put] +func (mgr *StorageMgr) SetUserSpaceQuota(c *gin.Context) { + // 1. 获取用户名参数 + user := c.Param("user") + if user == "" { + resputil.BadRequestError(c, "用户名不能为空") + return + } + + // 2. 解析请求体 + type QuotaRequest struct { + Quota int64 `json:"quota" binding:"required"` + } + var req QuotaRequest + if err := c.ShouldBindJSON(&req); err != nil { + resputil.BadRequestError(c, "请求体格式错误: "+err.Error()) + return + } + + // 3. 验证配额值 + if req.Quota < -1 { + resputil.BadRequestError(c, "配额值不能小于 -1") + return + } + + // 4. 获取用户信息（含临时扩容状态） + db := query.GetDB() + var userRow struct { + model.User + OriginalSpaceQuota *int64 `gorm:"column:original_space_quota"` + } + if err := db.Model(&model.User{}). + Select("users.*, users.original_space_quota"). + Where("name = ?", user). + First(&userRow).Error; err != nil { + resputil.Error(c, "用户不存在", resputil.NotSpecified) + return + } + userInfo := userRow.User + + // 5. 更新理论配额 + // 临时扩容期间：只更新 original_space_quota（理论配额），保持 space_quota（现配额）不变 + // 无临时扩容：更新 space_quota（理论配额即现配额） + isExpanded := userRow.OriginalSpaceQuota != nil + if isExpanded { + if err := db.Exec("UPDATE users SET original_space_quota = ? WHERE name = ? AND deleted_at IS NULL", req.Quota, user).Error; err != nil { + resputil.Error(c, fmt.Sprintf("更新理论配额失败: %v", err), resputil.NotSpecified) + return + } + } else { + if err := db.Model(&model.User{}).Where("name = ?", user).Update("space_quota", req.Quota).Error; err != nil { + resputil.Error(c, fmt.Sprintf("更新理论配额失败: %v", err), resputil.NotSpecified) + return + } + } + + // 6. 同步 CephFS 配额 + // 临时扩容期间 Ceph 配额维持现配额不变；无扩容时才同步新值 + cfg := config.GetConfig() + prefixConfig := ceph.StoragePrefixConfig{ + User: cfg.Storage.Prefix.User, + Account: cfg.Storage.Prefix.Account, + Public: cfg.Storage.Prefix.Public, + } + userPath := fmt.Sprintf("/user/%s", userInfo.Space) + + var err error + if !isExpanded { + err = ceph.SetCephDirectoryQuota(mgr.kubeClient, mgr.kubeConfig, "rook-ceph", userPath, prefixConfig, req.Quota) + if err != nil { + klog.Errorf("SetUserSpaceQuota: 设置用户 %s Ceph 配额失败: %v", user, err) + } + } + + // 7. 返回结果 + resputil.Success(c, gin.H{ + "user": user, + "quota": req.Quota, + "unit": "bytes", + "quota_formatted": formatSize(req.Quota), + "ceph_quota_set": err == nil, + "ceph_quota_error": err, + }) +} + +// AutoScaleUserSpaceQuota godoc +// +// @Summary Auto scale user space quota +// @Description Auto scale the space quota for a user based on current usage +// @Tags Storage +// @Accept json +// @Produce json +// @Security Bearer +// @Param user path string true "Username" +// @Param body body AutoScaleRequest true "Auto scale configuration" +// @Success 200 {object} resputil.Response[any] "Success" +// @Failure 400 {object} resputil.Response[any] "Request parameter error" +// @Failure 404 {object} resputil.Response[any] "User not found" +// @Failure 500 {object} resputil.Response[any] "Other errors" +// @Router /v1/storage/admin/user-spaces/{user}/autoscale [post] +func (mgr *StorageMgr) AutoScaleUserSpaceQuota(c *gin.Context) { + // 1. 获取用户名参数 + user := c.Param("user") + if user == "" { + resputil.BadRequestError(c, "用户名不能为空") + return + } + + // 2. 解析请求体 + var req AutoScaleRequest + if err := c.ShouldBindJSON(&req); err != nil { + resputil.BadRequestError(c, "请求体格式错误: "+err.Error()) + return + } + + // 3. 获取用户信息和当前使用空间大小 + db := query.GetDB() + var userInfo model.User + if err := db.Where("name = ?", user).First(&userInfo).Error; err != nil { + resputil.Error(c, "用户不存在", resputil.NotSpecified) + return + } + + var userSpaceSize model.UserSpaceSize + if err := db.Where("user_id = ?", userInfo.ID).First(&userSpaceSize).Error; err != nil { + resputil.Error(c, fmt.Sprintf("获取用户空间使用情况失败: %v", err), resputil.NotSpecified) + return + } + + // 4. 计算新的配额 + currentUsage := userSpaceSize.Size + newQuota := int64(float64(currentUsage) * req.ScaleUpRatio) + + // 应用最小和最大配额限制 + if req.MinQuota != -1 && newQuota < req.MinQuota { + newQuota = req.MinQuota + } + if req.MaxQuota != -1 && newQuota > req.MaxQuota { + newQuota = req.MaxQuota + } + + // 5. 更新用户配额 + if err := db.Model(&model.User{}).Where("name = ?", user).Update("space_quota", newQuota).Error; err != nil { + resputil.Error(c, fmt.Sprintf("更新用户配额失败: %v", err), resputil.NotSpecified) + return + } + + // 6. 实际设置 CephFS 目录配额 + cfg := config.GetConfig() + prefixConfig := ceph.StoragePrefixConfig{ + User: cfg.Storage.Prefix.User, + Account: cfg.Storage.Prefix.Account, + Public: cfg.Storage.Prefix.Public, + } + + // 构建用户空间路径 + userPath := fmt.Sprintf("/user/%s", userInfo.Space) + + // 调用 SetCephDirectoryQuota 设置实际配额 + cephErr := ceph.SetCephDirectoryQuota(mgr.kubeClient, mgr.kubeConfig, "rook-ceph", userPath, prefixConfig, newQuota) + if cephErr != nil { + // 记录错误但不影响响应，确保数据库更新成功 + klog.Errorf("AutoScaleUserSpaceQuota: 设置用户 %s Ceph 配额失败: %v", user, cephErr) + } + + // 7. 返回结果 + resputil.Success(c, gin.H{ + "user": user, + "current_usage": currentUsage, + "new_quota": newQuota, + "unit": "bytes", + "current_usage_formatted": formatSize(currentUsage), + "new_quota_formatted": formatSize(newQuota), + "ceph_quota_set": cephErr == nil, + "ceph_quota_error": cephErr, + }) +} + +// RunAutoShrink triggers one manual scan that shrinks users currently in temporary +// expansion state back to their original quota when it is safe to do so. +func (mgr *StorageMgr) RunAutoShrink(c *gin.Context) { + result, err := patrol.RunAutoShrinkStorageExpansions(c.Request.Context(), &patrol.Clients{ + KubeClient: mgr.kubeClient, + KubeConfig: mgr.kubeConfig, + PromClient: mgr.promClient, + }) + if err != nil { + resputil.Error(c, fmt.Sprintf("自动缩容执行失败：%v", err), resputil.NotSpecified) + return + } + + resputil.Success(c, gin.H{ + "message": result, + }) +} + +// ApplyExpansion godoc +// +// @Summary Apply temporary storage expansion for a user +// @Description Save the current quota as original and set an expanded quota +// @Tags Storage +// @Accept json +// @Produce json +// @Security Bearer +// @Param user path string true "Username" +// @Param body body object true "expand_bytes: bytes to add on top of current quota" +// @Success 200 {object} resputil.Response[any] "Success" +// @Failure 400 {object} resputil.Response[any] "Request parameter error" +// @Failure 500 {object} resputil.Response[any] "Other errors" +// @Router /v1/storage/admin/user-spaces/{user}/apply-expansion [post] +func (mgr *StorageMgr) ApplyExpansion(c *gin.Context) { + user := c.Param("user") + if user == "" { + resputil.BadRequestError(c, "用户名不能为空") + return + } + + var req struct { + ExpandBytes int64 `json:"expand_bytes" binding:"required,min=1"` + FreezeNewJobs bool `json:"freeze_new_jobs"` + DecisionJobID string `json:"decision_job_id"` + } + if err := c.ShouldBindJSON(&req); err != nil { + resputil.BadRequestError(c, "请求体格式错误: "+err.Error()) + return + } + + db := query.GetDB() + + // 查询当前配额和原始配额 + var row struct { + SpaceQuota int64 `gorm:"column:space_quota"` + OriginalSpaceQuota *int64 `gorm:"column:original_space_quota"` + } + if err := db.Raw( + "SELECT space_quota, original_space_quota FROM users WHERE name = ? AND deleted_at IS NULL", + user, + ).Scan(&row).Error; err != nil { + resputil.Error(c, "用户不存在", resputil.NotSpecified) + return + } + + if row.OriginalSpaceQuota != nil { + resputil.BadRequestError(c, "该用户已存在临时扩容，请先还原后再扩容") + return + } + + newQuota := row.SpaceQuota + req.ExpandBytes + + // 保存原始配额，并更新为新配额，同时设置 jobs_frozen + if err := db.Exec( + "UPDATE users "+ + "SET original_space_quota = space_quota, space_quota = ?, jobs_frozen = ?, "+ + "shrink_stage = ?, shrink_stage_updated_at = NOW() "+ + "WHERE name = ? AND deleted_at IS NULL", + newQuota, req.FreezeNewJobs, "expanded", user, + ).Error; err != nil { + resputil.Error(c, fmt.Sprintf("更新配额失败: %v", err), resputil.NotSpecified) + return + } + + // 同步到 CephFS + var userInfo model.User + if err := db.Where("name = ?", user).First(&userInfo).Error; err == nil { + cfg := config.GetConfig() + prefixConfig := ceph.StoragePrefixConfig{ + User: cfg.Storage.Prefix.User, + Account: cfg.Storage.Prefix.Account, + Public: cfg.Storage.Prefix.Public, + } + if cephErr := ceph.SetCephDirectoryQuota( + mgr.kubeClient, + mgr.kubeConfig, + "rook-ceph", + fmt.Sprintf("/user/%s", userInfo.Space), + prefixConfig, + newQuota, + ); cephErr != nil { + klog.Errorf("ApplyExpansion: 设置用户 %s Ceph 配额失败: %v", user, cephErr) + } + } + if req.DecisionJobID != "" { + action := "manual_expand" + if req.FreezeNewJobs { + action = "manual_expand_and_freeze" + } + _ = storagegovernance.MarkDecisionExecution(c.Request.Context(), req.DecisionJobID, action, nil) + } + + resputil.Success(c, gin.H{ + "user": user, + "original_quota": row.SpaceQuota, + "new_quota": newQuota, + "original_quota_formatted": formatSize(row.SpaceQuota), + "new_quota_formatted": formatSize(newQuota), + "jobs_frozen": req.FreezeNewJobs, + }) +} + +// RevertExpansion godoc +// +// @Summary Revert temporary storage expansion for a user +// @Description Restore the user's quota to the original value before expansion +// @Tags Storage +// @Accept json +// @Produce json +// @Security Bearer +// @Param user path string true "Username" +// @Success 200 {object} resputil.Response[any] "Success" +// @Failure 400 {object} resputil.Response[any] "Request parameter error" +// @Failure 500 {object} resputil.Response[any] "Other errors" +// @Router /v1/storage/admin/user-spaces/{user}/revert-expansion [post] +func (mgr *StorageMgr) RevertExpansion(c *gin.Context) { + user := c.Param("user") + if user == "" { + resputil.BadRequestError(c, "用户名不能为空") + return + } + + db := query.GetDB() + + var row struct { + SpaceQuota int64 `gorm:"column:space_quota"` + OriginalSpaceQuota *int64 `gorm:"column:original_space_quota"` + } + if err := db.Raw( + "SELECT space_quota, original_space_quota FROM users WHERE name = ? AND deleted_at IS NULL", + user, + ).Scan(&row).Error; err != nil { + resputil.Error(c, "用户不存在", resputil.NotSpecified) + return + } + + if row.OriginalSpaceQuota == nil { + resputil.BadRequestError(c, "该用户当前没有临时扩容，无需还原") + return + } + + originalQuota := *row.OriginalSpaceQuota + + // 查询用户 ID 和当前实际用量，决定是否同时解冻 + var userIDRow struct { + ID uint `gorm:"column:id"` + } + db.Raw("SELECT id FROM users WHERE name = ? AND deleted_at IS NULL", user).Scan(&userIDRow) + + var currentSize int64 + var spaceSize model.UserSpaceSize + if err := db.Where("user_id = ?", userIDRow.ID).First(&spaceSize).Error; err == nil { + currentSize = spaceSize.Size + } + + // 只有还原后的理论配额大于当前用量时才自动解冻；否则保持冻结状态 + shouldUnfreeze := originalQuota <= 0 || currentSize < originalQuota + if shouldUnfreeze { + if err := db.Exec( + "UPDATE users "+ + "SET space_quota = ?, original_space_quota = NULL, jobs_frozen = false, "+ + "shrink_stage = NULL, shrink_stage_updated_at = NULL "+ + "WHERE name = ? AND deleted_at IS NULL", + originalQuota, + user, + ).Error; err != nil { + resputil.Error(c, fmt.Sprintf("还原配额失败: %v", err), resputil.NotSpecified) + return + } + } else { + // 仅还原配额，不解冻（用量仍超出理论配额） + if err := db.Exec( + "UPDATE users "+ + "SET space_quota = ?, original_space_quota = NULL, "+ + "shrink_stage = NULL, shrink_stage_updated_at = NULL "+ + "WHERE name = ? AND deleted_at IS NULL", + originalQuota, + user, + ).Error; err != nil { + resputil.Error(c, fmt.Sprintf("还原配额失败: %v", err), resputil.NotSpecified) + return + } + } + + // 同步到 CephFS + var userInfo model.User + if err := db.Where("name = ?", user).First(&userInfo).Error; err == nil { + cfg := config.GetConfig() + prefixConfig := ceph.StoragePrefixConfig{ + User: cfg.Storage.Prefix.User, + Account: cfg.Storage.Prefix.Account, + Public: cfg.Storage.Prefix.Public, + } + if cephErr := ceph.SetCephDirectoryQuota( + mgr.kubeClient, + mgr.kubeConfig, + "rook-ceph", + fmt.Sprintf("/user/%s", userInfo.Space), + prefixConfig, + originalQuota, + ); cephErr != nil { + klog.Errorf("RevertExpansion: 设置用户 %s Ceph 配额失败: %v", user, cephErr) + } + } + + resputil.Success(c, gin.H{ + "user": user, + "reverted_quota": originalQuota, + "reverted_quota_formatted": formatSize(originalQuota), + "jobs_unfrozen": shouldUnfreeze, + }) +} + +// UnfreezeJobs godoc +// +// @Summary Manually unfreeze job creation for a user +// @Description Clear the jobs_frozen flag, allowing the user to create new jobs again +// @Tags Storage +// @Produce json +// @Security Bearer +// @Param user path string true "Username" +// @Success 200 {object} resputil.Response[any] "Success" +// @Failure 400 {object} resputil.Response[any] "Request parameter error" +// @Failure 500 {object} resputil.Response[any] "Other errors" +// @Router /v1/storage/admin/user-spaces/{user}/unfreeze-jobs [post] +func (mgr *StorageMgr) UnfreezeJobs(c *gin.Context) { + user := c.Param("user") + if user == "" { + resputil.BadRequestError(c, "用户名不能为空") + return + } + + db := query.GetDB() + if err := db.Exec("UPDATE users SET jobs_frozen = false WHERE name = ? AND deleted_at IS NULL", user).Error; err != nil { + resputil.Error(c, fmt.Sprintf("解冻失败: %v", err), resputil.NotSpecified) + return + } + + resputil.Success(c, gin.H{"user": user, "jobs_frozen": false}) +} + +// FreezeJobs manually freezes job creation for a user and optionally binds the action to a decision record. +func (mgr *StorageMgr) FreezeJobs(c *gin.Context) { + user := c.Param("user") + if user == "" { + resputil.BadRequestError(c, "用户名不能为空") + return + } + + var req struct { + DecisionJobID string `json:"decision_job_id"` + } + if err := c.ShouldBindJSON(&req); err != nil && err.Error() != "EOF" { + resputil.BadRequestError(c, "请求体格式错误: "+err.Error()) + return + } + + db := query.GetDB() + if err := db.Exec("UPDATE users SET jobs_frozen = true WHERE name = ? AND deleted_at IS NULL", user).Error; err != nil { + if req.DecisionJobID != "" { + _ = storagegovernance.MarkDecisionExecution(c.Request.Context(), req.DecisionJobID, "manual_freeze_failed", err) + } + resputil.Error(c, fmt.Sprintf("冻结失败: %v", err), resputil.NotSpecified) + return + } + + if req.DecisionJobID != "" { + _ = storagegovernance.MarkDecisionExecution(c.Request.Context(), req.DecisionJobID, "manual_freeze", nil) + } + + resputil.Success(c, gin.H{"user": user, "jobs_frozen": true}) +} + +// TriggerLLMDecision godoc +// +// @Summary Trigger LLM storage expansion decision for a user +// @Description Calls Claude agent to analyze whether a user needs temporary storage expansion +// @Tags Storage +// @Accept json +// @Produce json +// @Security Bearer +// @Param user path string true "Username" +// @Success 200 {object} resputil.Response[any] "Success" +// @Failure 500 {object} resputil.Response[any] "Other errors" +// @Router /v1/storage/admin/user-spaces/{user}/llm-decision [post] +// TriggerLLMDecision 异步启动 LLM 分析，立即返回 job_id +func (mgr *StorageMgr) TriggerLLMDecision(c *gin.Context) { + user := c.Param("user") + if user == "" { + resputil.BadRequestError(c, "用户名不能为空") + return + } + + engine := storagegovernance.NewEngine( + mgr.kubeClient, + mgr.kubeConfig, + mgr.promClient, + storagegovernance.DefaultConstraintConfig(), + ) + jobID, err := engine.StartAsyncDecision(context.Background(), storagegovernance.DecisionRequest{ + Username: user, + Source: model.StorageDecisionSourceManual, + TriggerReason: "manual llm decision request", + }) + if err != nil { + klog.Errorf("TriggerLLMDecision: user=%s err=%v", user, err) + resputil.Error(c, fmt.Sprintf("鍚姩 LLM 鍒嗘瀽澶辫触: %v", err), resputil.NotSpecified) + return + } + + resputil.Success(c, gin.H{"job_id": jobID}) +} + +// GetLLMDecisionStatus 查询 LLM 分析任务状态 +func (mgr *StorageMgr) GetLLMDecisionStatus(c *gin.Context) { + jobID := c.Param("job_id") + + job, err := storagegovernance.GetDecisionStatus(c.Request.Context(), jobID) + + if err != nil { + resputil.Error(c, "任务不存在", resputil.NotSpecified) + return + } + + resputil.Success(c, job) +} + +// formatSize 格式化大小为人类可读格式 +// ListStorageDecisions returns paginated persisted storage decision records. +func (mgr *StorageMgr) ListStorageDecisions(c *gin.Context) { + page, _ := strconv.Atoi(c.DefaultQuery("page", "1")) + pageSize, _ := strconv.Atoi(c.DefaultQuery("pageSize", "20")) + + result, err := storagegovernance.ListDecisionRecords( + c.Request.Context(), + page, + pageSize, + c.Query("user"), + c.Query("status"), + c.Query("source"), + ) + if err != nil { + resputil.Error(c, fmt.Sprintf("failed to list storage decisions: %v", err), resputil.NotSpecified) + return + } + + resputil.Success(c, result) +} + +// GetStorageDecision returns one persisted storage decision record with full details. +func (mgr *StorageMgr) GetStorageDecision(c *gin.Context) { + jobID := c.Param("job_id") + if jobID == "" { + resputil.BadRequestError(c, "job_id cannot be empty") + return + } + + result, err := storagegovernance.GetDecisionRecord(c.Request.Context(), jobID) + if err != nil { + resputil.Error(c, fmt.Sprintf("failed to get storage decision: %v", err), resputil.NotSpecified) + return + } + + resputil.Success(c, result) +} + +// ReplayStorageDecisions re-evaluates stored decisions under the current or overridden safety policy. +func (mgr *StorageMgr) ReplayStorageDecisions(c *gin.Context) { + var req struct { + Limit int `json:"limit"` + MaxExpandRatio *float64 `json:"max_expand_ratio"` + MaxExpandBytes *int64 `json:"max_expand_bytes"` + MinPlatformReservedRatio *float64 `json:"min_platform_reserved_ratio"` + MinPlatformReservedBytes *int64 `json:"min_platform_reserved_bytes"` + ExpansionCooldownHours *int `json:"expansion_cooldown_hours"` + ForceFreezeWhenOverQuota *bool `json:"force_freeze_when_over_quota"` + } + if err := c.ShouldBindJSON(&req); err != nil && err.Error() != "EOF" { + resputil.BadRequestError(c, "invalid replay request: "+err.Error()) + return + } + + cfg := storagegovernance.DefaultConstraintConfig() + if req.MaxExpandRatio != nil { + cfg.MaxExpandRatio = *req.MaxExpandRatio + } + if req.MaxExpandBytes != nil { + cfg.MaxExpandBytes = *req.MaxExpandBytes + } + if req.MinPlatformReservedRatio != nil { + cfg.MinPlatformReservedRatio = *req.MinPlatformReservedRatio + } + if req.MinPlatformReservedBytes != nil { + cfg.MinPlatformReservedBytes = *req.MinPlatformReservedBytes + } + if req.ExpansionCooldownHours != nil { + cfg.ExpansionCooldown = time.Duration(*req.ExpansionCooldownHours) * time.Hour + } + if req.ForceFreezeWhenOverQuota != nil { + cfg.ForceFreezeWhenOverQuota = *req.ForceFreezeWhenOverQuota + } + + summary, err := storagegovernance.ReplayStoredDecisions(c.Request.Context(), cfg, req.Limit) + if err != nil { + resputil.Error(c, fmt.Sprintf("failed to replay storage decisions: %v", err), resputil.NotSpecified) + return + } + + resputil.Success(c, summary) +} + +func (mgr *StorageMgr) metadataIndexService() *storageindex.Service { + return storageindex.NewService(mgr.kubeClient, mgr.kubeConfig) +} + +// TriggerMetadataIndexScan starts an asynchronous metadata indexing run for one workspace. +func (mgr *StorageMgr) TriggerMetadataIndexScan(c *gin.Context) { + var req struct { + WorkspaceType string `json:"workspace_type" binding:"required"` + WorkspaceName string `json:"workspace_name"` + } + if err := c.ShouldBindJSON(&req); err != nil { + resputil.BadRequestError(c, "invalid metadata scan request: "+err.Error()) + return + } + + scanID, err := mgr.metadataIndexService().StartFullScan(c.Request.Context(), storageindex.StartScanRequest{ + WorkspaceType: model.StorageIndexWorkspaceType(strings.TrimSpace(req.WorkspaceType)), + WorkspaceName: strings.TrimSpace(req.WorkspaceName), + TriggerSource: "manual", + ScanMode: model.StorageIndexScanModeDailyRefresh, + }) + if err != nil { + resputil.Error(c, fmt.Sprintf("failed to start metadata scan: %v", err), resputil.NotSpecified) + return + } + + resputil.Success(c, gin.H{ + "scan_id": scanID, + "workspace_type": req.WorkspaceType, + "workspace_name": req.WorkspaceName, + }) +} + +// GetMetadataIndexScan returns the current status of a metadata indexing job. +func (mgr *StorageMgr) GetMetadataIndexScan(c *gin.Context) { + scanID := strings.TrimSpace(c.Param("scan_id")) + if scanID == "" { + resputil.BadRequestError(c, "scan_id cannot be empty") + return + } + + job, err := mgr.metadataIndexService().GetScanJob(c.Request.Context(), scanID) + if err != nil { + resputil.Error(c, fmt.Sprintf("failed to query metadata scan: %v", err), resputil.NotSpecified) + return + } + + resputil.Success(c, job) +} + +// GetMetadataWorkspaceOverview returns the latest indexed overview for a workspace. +func (mgr *StorageMgr) GetMetadataWorkspaceOverview(c *gin.Context) { + workspaceType := model.StorageIndexWorkspaceType(strings.TrimSpace(c.Param("workspace_type"))) + workspaceName := strings.TrimSpace(c.Param("workspace_name")) + + overview, err := mgr.metadataIndexService().GetWorkspaceOverview(c.Request.Context(), workspaceType, workspaceName) + if err != nil { + resputil.Error(c, fmt.Sprintf("failed to query metadata overview: %v", err), resputil.NotSpecified) + return + } + + resputil.Success(c, overview) +} + +// ListMetadataWorkspaceRedundancyHits returns redundancy hits detected against the public baseline. +func (mgr *StorageMgr) ListMetadataWorkspaceRedundancyHits(c *gin.Context) { + mgr.listStorageIndexPage(c, "redundancy hits", func( + ctx context.Context, + workspaceType model.StorageIndexWorkspaceType, + workspaceName string, + page int, + pageSize int, + ) (any, int64, error) { + return mgr.metadataIndexService().ListRedundancyHits(ctx, workspaceType, workspaceName, page, pageSize) + }) +} + +func (mgr *StorageMgr) ListMetadataWorkspaceCandidates(c *gin.Context) { + mgr.listStorageIndexPage(c, "candidates", func( + ctx context.Context, + workspaceType model.StorageIndexWorkspaceType, + workspaceName string, + page int, + pageSize int, + ) (any, int64, error) { + return mgr.metadataIndexService().ListCandidates(ctx, workspaceType, workspaceName, page, pageSize) + }) +} + +func (mgr *StorageMgr) listStorageIndexPage( + c *gin.Context, + resourceName string, + queryPage func(context.Context, model.StorageIndexWorkspaceType, string, int, int) (any, int64, error), +) { + workspaceType := model.StorageIndexWorkspaceType(strings.TrimSpace(c.Param("workspace_type"))) + workspaceName := strings.TrimSpace(c.Param("workspace_name")) + page, _ := strconv.Atoi(c.DefaultQuery("page", "1")) + pageSize, _ := strconv.Atoi(c.DefaultQuery("pageSize", "50")) + + items, total, err := queryPage(c.Request.Context(), workspaceType, workspaceName, page, pageSize) + if err != nil { + resputil.Error(c, fmt.Sprintf("failed to query %s: %v", resourceName, err), resputil.NotSpecified) + return + } + + successStorageIndexPage(c, items, total, page, pageSize) +} + +func (mgr *StorageMgr) ListMetadataWorkspaceCandidateFiles(c *gin.Context) { + workspaceType := model.StorageIndexWorkspaceType(strings.TrimSpace(c.Param("workspace_type"))) + workspaceName := strings.TrimSpace(c.Param("workspace_name")) + candidatePath := strings.TrimSpace(c.Query("candidate_path")) + page, _ := strconv.Atoi(c.DefaultQuery("page", "1")) + pageSize, _ := strconv.Atoi(c.DefaultQuery("pageSize", "200")) + + items, total, err := mgr.metadataIndexService().ListCandidateFiles( + c.Request.Context(), + workspaceType, + workspaceName, + candidatePath, + page, + pageSize, + ) + if err != nil { + resputil.Error(c, fmt.Sprintf("failed to query candidate files: %v", err), resputil.NotSpecified) + return + } + + resputil.Success(c, gin.H{ + "items": items, + "total": total, + "page": page, + "pageSize": pageSize, + "totalPages": (int(total) + pageSize - 1) / pageSize, + }) +} + +func (mgr *StorageMgr) CompareMetadataFolders(c *gin.Context) { + var req struct { + LeftPath string `json:"left_path" binding:"required"` + RightPath string `json:"right_path" binding:"required"` + CompareType string `json:"compare_type"` + CompareMode string `json:"compare_mode"` + } + if err := c.ShouldBindJSON(&req); err != nil { + resputil.BadRequestError(c, "invalid compare folder request: "+err.Error()) + return + } + + jobID, err := mgr.metadataIndexService().StartCompareDirectories( + c.Request.Context(), + strings.TrimSpace(req.LeftPath), + strings.TrimSpace(req.RightPath), + strings.TrimSpace(req.CompareType), + strings.TrimSpace(req.CompareMode), + ) + if err != nil { + resputil.Error(c, fmt.Sprintf("failed to start compare job: %v", err), resputil.NotSpecified) + return + } + + resputil.Success(c, gin.H{"job_id": jobID}) +} + +func (mgr *StorageMgr) GetMetadataFolderCompareJob(c *gin.Context) { + jobID := strings.TrimSpace(c.Param("job_id")) + if jobID == "" { + resputil.BadRequestError(c, "job_id cannot be empty") + return + } + + job, err := mgr.metadataIndexService().GetCompareDirectoryJob(jobID) + if err != nil { + resputil.Error(c, fmt.Sprintf("failed to query compare job: %v", err), resputil.NotSpecified) + return + } + + resputil.Success(c, job) +} + +func formatSize(bytes int64) string { + const unit = 1024 + if bytes <= 0 { + return "0 B" + } + if bytes < unit { + return fmt.Sprintf("%d B", bytes) + } + div, exp := int64(unit), 0 + for n := bytes / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp]) +} + +func successStorageIndexPage(c *gin.Context, items any, total int64, page, pageSize int) { + resputil.Success(c, gin.H{ + "items": items, + "total": total, + "page": page, + "pageSize": pageSize, + "totalPages": (int(total) + pageSize - 1) / pageSize, + }) +} diff --git a/backend/internal/handler/system_config.go b/backend/internal/handler/system_config.go index b0e9173fc..7ec218953 100644 --- a/backend/internal/handler/system_config.go +++ b/backend/internal/handler/system_config.go @@ -58,9 +58,14 @@ func (mgr *SystemConfigMgr) RegisterAdmin(g *gin.RouterGroup) { // 路由组: /v1/admin/system-config g.GET("/llm", mgr.GetLLMConfig) g.PUT("/llm", mgr.UpdateLLMConfig) - // 新增：重置 LLM 配置 + // 重置平台通用 LLM 配置 g.DELETE("/llm", mgr.ResetLLMConfig) + // 存储决策模型配置 + g.GET("/storage-decision", mgr.GetStorageDecisionConfig) + g.PUT("/storage-decision", mgr.UpdateStorageDecisionConfig) + g.DELETE("/storage-decision", mgr.ResetStorageDecisionConfig) + g.GET("/gpu-analysis", mgr.GetGpuAnalysisStatus) g.PUT("/gpu-analysis", mgr.SetGpuAnalysisStatus) g.GET("/prequeue", mgr.GetPrequeueConfig) @@ -89,7 +94,24 @@ type UpdateLLMConfigReq struct { BaseURL string `json:"baseUrl" binding:"required"` APIKey string `json:"apiKey"` ModelName string `json:"modelName" binding:"required"` - Validate bool `json:"validate"` // 是否立即校验连接 + Validate bool `json:"validate"` +} + +type StorageDecisionConfigResp struct { + DecisionMode string `json:"decisionMode"` + ConfigSource string `json:"configSource"` + BaseURL string `json:"baseUrl"` + APIKey string `json:"apiKey"` + ModelName string `json:"modelName"` +} + +type UpdateStorageDecisionConfigReq struct { + DecisionMode string `json:"decisionMode"` + ConfigSource string `json:"configSource"` + BaseURL string `json:"baseUrl"` + APIKey string `json:"apiKey"` + ModelName string `json:"modelName"` + Validate bool `json:"validate"` } type GpuAnalysisStatusResp struct { @@ -237,7 +259,7 @@ func (mgr *SystemConfigMgr) GetLLMConfig(c *gin.Context) { // UpdateLLMConfig godoc // @Summary 更新 LLM 配置 -// @Description 更新 LLM 的连接信息。如果 validate 为 true，会尝试连接 /check 接口，失败则不保存。 +// @Description 更新 LLM 的连接信息。如果 validate 为 true，会尝试连接 /models 接口，失败则不保存。 // @Tags SystemConfig // @Accept json // @Produce json @@ -291,6 +313,94 @@ func (mgr *SystemConfigMgr) ResetLLMConfig(c *gin.Context) { resputil.Success(c, "LLM configuration reset successfully") } +// GetStorageDecisionConfig godoc +// @Summary 获取存储决策模型配置 +// @Description 获取当前系统配置的存储决策模式、配置来源与自定义模型连接信息。出于安全考虑，API Key 可能会被脱敏显示。 +// @Tags SystemConfig +// @Produce json +// @Security Bearer +// @Success 200 {object} resputil.Response[StorageDecisionConfigResp] "配置信息" +// @Failure 500 {object} resputil.Response[any] "服务器错误" +// @Router /v1/admin/system-config/storage-decision [get] +func (mgr *SystemConfigMgr) GetStorageDecisionConfig(c *gin.Context) { + cfg, err := mgr.service.GetStorageDecisionConfig(c.Request.Context()) + if err != nil { + resputil.Error(c, err.Error(), resputil.ServiceError) + return + } + + // 响应给前端时，将 Key 替换为固定掩码 + displayKey := "" + if cfg.APIKey != "" { + displayKey = service.MaskedAPIKeyPlaceholder + } + + resputil.Success(c, StorageDecisionConfigResp{ + DecisionMode: cfg.DecisionMode, + ConfigSource: cfg.ConfigSource, + BaseURL: cfg.BaseURL, + APIKey: displayKey, + ModelName: cfg.ModelName, + }) +} + +// UpdateStorageDecisionConfig godoc +// @Summary 更新存储决策模型配置 +// @Description 更新存储决策模式、配置来源以及自定义模型连接信息。如果 validate 为 true，会按当前来源校验连接。 +// @Tags SystemConfig +// @Accept json +// @Produce json +// @Security Bearer +// @Param data body UpdateStorageDecisionConfigReq true "配置信息" +// @Success 200 {object} resputil.Response[string] "更新成功" +// @Failure 400 {object} resputil.Response[any] "参数错误或校验失败" +// @Router /v1/admin/system-config/storage-decision [put] +func (mgr *SystemConfigMgr) UpdateStorageDecisionConfig(c *gin.Context) { + var req UpdateStorageDecisionConfigReq + if err := c.ShouldBindJSON(&req); err != nil { + resputil.BadRequestError(c, err.Error()) + return + } + + serviceCfg := &service.StorageDecisionConfig{ + DecisionMode: req.DecisionMode, + ConfigSource: req.ConfigSource, + BaseURL: req.BaseURL, + APIKey: req.APIKey, + ModelName: req.ModelName, + } + + err := mgr.service.UpdateStorageDecisionConfig(c.Request.Context(), serviceCfg, req.Validate) + if err != nil { + if strings.Contains(err.Error(), "validation failed") { + resputil.Error(c, "Storage decision connection check failed. Please verify your settings.", resputil.BusinessLogicError) + return + } + resputil.Error(c, err.Error(), resputil.ServiceError) + return + } + + resputil.Success(c, "Storage decision configuration updated successfully") +} + +// ResetStorageDecisionConfig godoc +// @Summary 重置存储决策模型配置 +// @Description 重置存储决策模式与自定义模型连接配置，不影响平台通用 LLM 配置 +// @Tags SystemConfig +// @Produce json +// @Security Bearer +// @Success 200 {object} resputil.Response[string] "重置成功" +// @Failure 500 {object} resputil.Response[any] "服务器错误" +// @Router /v1/admin/system-config/storage-decision [delete] +func (mgr *SystemConfigMgr) ResetStorageDecisionConfig(c *gin.Context) { + err := mgr.service.ResetStorageDecisionConfig(c.Request.Context()) + if err != nil { + resputil.Error(c, err.Error(), resputil.ServiceError) + return + } + resputil.Success(c, "Storage decision configuration reset successfully") +} + // GetGpuAnalysisStatus godoc // @Summary 获取 GPU 分析功能开关状态 // @Description 查询当前系统是否开启了自动 GPU 资源滥用检测 diff --git a/backend/internal/handler/vcjob/custom.go b/backend/internal/handler/vcjob/custom.go index 7635bbb1f..162ba106f 100644 --- a/backend/internal/handler/vcjob/custom.go +++ b/backend/internal/handler/vcjob/custom.go @@ -60,6 +60,12 @@ func (mgr *VolcanojobMgr) CreateTrainingJob(c *gin.Context) { resputil.Error(c, err.Error(), resputil.ServiceError) return } + + if err := util.CheckStorageQuota(token.Username); err != nil { + resputil.Error(c, err.Error(), resputil.NotSpecified) + return + } + if !mgr.preCheckCreateJob(c, token, scheduleType, false) { return } diff --git a/backend/internal/handler/vcjob/jupyter.go b/backend/internal/handler/vcjob/jupyter.go index 40e59a07c..6d117d2e0 100644 --- a/backend/internal/handler/vcjob/jupyter.go +++ b/backend/internal/handler/vcjob/jupyter.go @@ -70,6 +70,12 @@ func (mgr *VolcanojobMgr) CreateJupyterJob(c *gin.Context) { resputil.Error(c, err.Error(), resputil.ServiceError) return } + + if err := util.CheckStorageQuota(token.Username); err != nil { + resputil.Error(c, err.Error(), resputil.NotSpecified) + return + } + if !mgr.preCheckCreateJob(c, token, scheduleType, true) { return } diff --git a/backend/internal/handler/vcjob/pytorch.go b/backend/internal/handler/vcjob/pytorch.go index b5ba5b523..57c9797fa 100644 --- a/backend/internal/handler/vcjob/pytorch.go +++ b/backend/internal/handler/vcjob/pytorch.go @@ -17,6 +17,7 @@ import ( "github.com/raids-lab/crater/pkg/vcqueue" ) +//nolint:gocyclo // Job assembly coordinates validation, queues, and task generation in one request flow. func (mgr *VolcanojobMgr) CreatePytorchJob(c *gin.Context) { token := util.GetToken(c) @@ -30,6 +31,12 @@ func (mgr *VolcanojobMgr) CreatePytorchJob(c *gin.Context) { resputil.BadRequestError(c, err.Error()) return } + + if err := util.CheckStorageQuota(token.Username); err != nil { + resputil.Error(c, err.Error(), resputil.NotSpecified) + return + } + if !mgr.preCheckCreateJob(c, token, scheduleType, false) { return } diff --git a/backend/internal/handler/vcjob/tensorflow.go b/backend/internal/handler/vcjob/tensorflow.go index 0e86c96cb..55cb828ac 100644 --- a/backend/internal/handler/vcjob/tensorflow.go +++ b/backend/internal/handler/vcjob/tensorflow.go @@ -66,6 +66,12 @@ func (mgr *VolcanojobMgr) CreateTensorflowJob(c *gin.Context) { resputil.BadRequestError(c, err.Error()) return } + + if err := util.CheckStorageQuota(token.Username); err != nil { + resputil.Error(c, err.Error(), resputil.NotSpecified) + return + } + if !mgr.preCheckCreateJob(c, token, scheduleType, false) { return } diff --git a/backend/internal/handler/vcjob/webide.go b/backend/internal/handler/vcjob/webide.go index 5aa1014ec..b1ba696ab 100644 --- a/backend/internal/handler/vcjob/webide.go +++ b/backend/internal/handler/vcjob/webide.go @@ -55,6 +55,12 @@ func (mgr *VolcanojobMgr) CreateWebIDEJob(c *gin.Context) { resputil.Error(c, err.Error(), resputil.ServiceError) return } + + if err := util.CheckStorageQuota(token.Username); err != nil { + resputil.Error(c, err.Error(), resputil.NotSpecified) + return + } + if !mgr.preCheckCreateJob(c, token, scheduleType, true) { return } diff --git a/backend/internal/service/config_service.go b/backend/internal/service/config_service.go index ae73d7d86..930f468a4 100644 --- a/backend/internal/service/config_service.go +++ b/backend/internal/service/config_service.go @@ -28,6 +28,12 @@ import ( // 定义掩码常量 const MaskedAPIKeyPlaceholder = "********************************************" +const DefaultStorageDirectModelBaseURL = "http://192.168.5.68:30186/v1" + +const ( + StorageDecisionConfigSourcePlatform = "platform" + StorageDecisionConfigSourceCustom = "custom" +) const ( DefaultModelDownloadMaxConcurrent = 5 @@ -52,6 +58,14 @@ type ModelDownloadLimitConfig struct { WhitelistUserIDs []uint } +type StorageDecisionConfig struct { + DecisionMode string + ConfigSource string + BaseURL string + APIKey string + ModelName string +} + // cleanBaseURL 内部辅助：清理 URL 结尾的斜杠 func (c *LLMConfig) cleanBaseURL() string { return strings.TrimSuffix(strings.TrimSpace(c.BaseURL), "/") @@ -131,6 +145,12 @@ func defaultSystemConfigValue(key string) string { return FormatBillingAmountConfigValue(defaultBillingIssueAmount) case model.ConfigKeyBillingDefaultIssuePeriodMinute: return "43200" + case model.ConfigKeyStorageDecisionMode: + return "agent" + case model.ConfigKeyStorageDecisionConfigSource: + return StorageDecisionConfigSourcePlatform + case model.ConfigKeyStorageDirectModelBaseURL: + return DefaultStorageDirectModelBaseURL default: return "" } @@ -267,7 +287,12 @@ func (s *ConfigService) updateConfigs(ctx context.Context, updates map[string]st // GetLLMConfig 从数据库按需读取最新配置 func (s *ConfigService) GetLLMConfig(ctx context.Context) (*LLMConfig, error) { - configMap, err := s.getConfigs(ctx, model.ConfigKeyLLMBaseURL, model.ConfigKeyLLMAPIKey, model.ConfigKeyLLMModelName) + configMap, err := s.getConfigs( + ctx, + model.ConfigKeyLLMBaseURL, + model.ConfigKeyLLMAPIKey, + model.ConfigKeyLLMModelName, + ) if err != nil { return nil, err } @@ -293,6 +318,43 @@ func (s *ConfigService) GetLLMConfig(ctx context.Context) (*LLMConfig, error) { }, nil } +func (s *ConfigService) GetStorageDecisionConfig(ctx context.Context) (*StorageDecisionConfig, error) { + configMap, err := s.getConfigs( + ctx, + model.ConfigKeyStorageDecisionMode, + model.ConfigKeyStorageDecisionConfigSource, + model.ConfigKeyStorageDirectModelBaseURL, + model.ConfigKeyStorageDirectModelAPIKey, + model.ConfigKeyStorageDirectModelName, + ) + if err != nil { + return nil, err + } + + directEncryptedKey := configMap[model.ConfigKeyStorageDirectModelAPIKey] + directPlainKey := "" + if directEncryptedKey != "" { + decrypted, decryptErr := crypto.Decrypt(directEncryptedKey) + if decryptErr != nil { + klog.Errorf("Failed to decrypt direct API Key: %v, assuming plain text or empty", decryptErr) + directPlainKey = directEncryptedKey + } else { + directPlainKey = decrypted + } + } + + return &StorageDecisionConfig{ + DecisionMode: normalizeDecisionMode(configMap[model.ConfigKeyStorageDecisionMode]), + ConfigSource: normalizeStorageDecisionConfigSource( + configMap[model.ConfigKeyStorageDecisionConfigSource], + configMap[model.ConfigKeyStorageDirectModelName], + ), + BaseURL: configMap[model.ConfigKeyStorageDirectModelBaseURL], + APIKey: directPlainKey, + ModelName: configMap[model.ConfigKeyStorageDirectModelName], + }, nil +} + // CheckLLMConnection 使用 /models 接口进行校验，并验证 ModelName 是否存在 func (s *ConfigService) CheckLLMConnection(ctx context.Context, cfg *LLMConfig) error { checkURL := cfg.GetCheckURL() @@ -358,6 +420,52 @@ func (s *ConfigService) CheckLLMConnection(ctx context.Context, cfg *LLMConfig) return nil } +func normalizeDecisionMode(mode string) string { + switch strings.TrimSpace(strings.ToLower(mode)) { + case "direct": + return "direct" + default: + return "agent" + } +} + +func normalizeStorageDecisionConfigSource(source, modelName string) string { + switch strings.TrimSpace(strings.ToLower(source)) { + case StorageDecisionConfigSourceCustom: + return StorageDecisionConfigSourceCustom + case StorageDecisionConfigSourcePlatform: + return StorageDecisionConfigSourcePlatform + default: + if strings.TrimSpace(modelName) != "" { + return StorageDecisionConfigSourceCustom + } + return StorageDecisionConfigSourcePlatform + } +} + +func (s *ConfigService) CheckStorageDecisionConnection( + ctx context.Context, + llmCfg *LLMConfig, + cfg *StorageDecisionConfig, +) error { + if normalizeStorageDecisionConfigSource(cfg.ConfigSource, cfg.ModelName) == StorageDecisionConfigSourceCustom { + customCfg := &LLMConfig{ + BaseURL: cfg.BaseURL, + APIKey: cfg.APIKey, + ModelName: cfg.ModelName, + } + return s.CheckLLMConnection(ctx, customCfg) + } + if llmCfg == nil { + var err error + llmCfg, err = s.GetLLMConfig(ctx) + if err != nil { + return err + } + } + return s.CheckLLMConnection(ctx, llmCfg) +} + // SetGpuAnalysisEnabled 设置GPU分析功能的开关，并同步创建或更新定时任务的状态 func (s *ConfigService) SetGpuAnalysisEnabled(c *gin.Context, enable bool) error { var ctx = c.Request.Context() @@ -473,28 +581,30 @@ func (s *ConfigService) ResetLLMConfig(ctx context.Context) error { }) } -// UpdateLLMConfig 更新配置 -func (s *ConfigService) UpdateLLMConfig(ctx context.Context, reqCfg *LLMConfig, validate bool) error { - finalKeyToSave := "" - - if reqCfg.APIKey == MaskedAPIKeyPlaceholder { - oldConfigRaw, err := s.getConfigs(ctx, model.ConfigKeyLLMAPIKey) - if err == nil { - finalKeyToSave = oldConfigRaw[model.ConfigKeyLLMAPIKey] +func (s *ConfigService) ResetStorageDecisionConfig(ctx context.Context) error { + updates := map[string]string{ + model.ConfigKeyStorageDecisionMode: "agent", + model.ConfigKeyStorageDecisionConfigSource: StorageDecisionConfigSourcePlatform, + model.ConfigKeyStorageDirectModelBaseURL: DefaultStorageDirectModelBaseURL, + model.ConfigKeyStorageDirectModelAPIKey: "", + model.ConfigKeyStorageDirectModelName: "", + } - if validate { - plainKey, err := crypto.Decrypt(finalKeyToSave) - if err == nil { - reqCfg.APIKey = plainKey - } + return s.q.Transaction(func(tx *query.Query) error { + for k, v := range updates { + if _, err := tx.SystemConfig.WithContext(ctx).Where(tx.SystemConfig.Key.Eq(k)).Update(tx.SystemConfig.Value, v); err != nil { + return err } } - } else { - encrypted, err := crypto.Encrypt(reqCfg.APIKey) - if err != nil { - return fmt.Errorf("failed to encrypt api key: %w", err) - } - finalKeyToSave = encrypted + return nil + }) +} + +// UpdateLLMConfig 更新配置 +func (s *ConfigService) UpdateLLMConfig(ctx context.Context, reqCfg *LLMConfig, validate bool) error { + finalLLMKeyToSave, err := s.prepareSecretForSave(ctx, model.ConfigKeyLLMAPIKey, reqCfg.APIKey) + if err != nil { + return err } if validate { @@ -505,7 +615,7 @@ func (s *ConfigService) UpdateLLMConfig(ctx context.Context, reqCfg *LLMConfig, updates := map[string]any{ model.ConfigKeyLLMBaseURL: reqCfg.BaseURL, - model.ConfigKeyLLMAPIKey: finalKeyToSave, + model.ConfigKeyLLMAPIKey: finalLLMKeyToSave, model.ConfigKeyLLMModelName: reqCfg.ModelName, } @@ -519,6 +629,73 @@ func (s *ConfigService) UpdateLLMConfig(ctx context.Context, reqCfg *LLMConfig, }) } +func (s *ConfigService) UpdateStorageDecisionConfig( + ctx context.Context, + reqCfg *StorageDecisionConfig, + validate bool, +) error { + reqCfg.DecisionMode = normalizeDecisionMode(reqCfg.DecisionMode) + reqCfg.ConfigSource = normalizeStorageDecisionConfigSource(reqCfg.ConfigSource, reqCfg.ModelName) + + finalKeyToSave, err := s.prepareSecretForSave(ctx, model.ConfigKeyStorageDirectModelAPIKey, reqCfg.APIKey) + if err != nil { + return err + } + + if validate { + llmCfg, getErr := s.GetLLMConfig(ctx) + if getErr != nil { + return getErr + } + if err := s.CheckStorageDecisionConnection(ctx, llmCfg, reqCfg); err != nil { + return fmt.Errorf("validation failed: %w", err) + } + } + + updates := map[string]any{ + model.ConfigKeyStorageDecisionMode: reqCfg.DecisionMode, + model.ConfigKeyStorageDecisionConfigSource: reqCfg.ConfigSource, + model.ConfigKeyStorageDirectModelBaseURL: reqCfg.BaseURL, + model.ConfigKeyStorageDirectModelAPIKey: finalKeyToSave, + model.ConfigKeyStorageDirectModelName: reqCfg.ModelName, + } + + return s.q.Transaction(func(tx *query.Query) error { + for k, v := range updates { + if _, err := tx.SystemConfig.WithContext(ctx).Where(tx.SystemConfig.Key.Eq(k)).Update(tx.SystemConfig.Value, v); err != nil { + return err + } + } + return nil + }) +} + +func (s *ConfigService) prepareSecretForSave( + ctx context.Context, + configKey string, + targetValue string, +) (string, error) { + var currentValue string + + if targetValue == MaskedAPIKeyPlaceholder { + oldConfigRaw, err := s.getConfigs(ctx, configKey) + if err == nil { + currentValue = oldConfigRaw[configKey] + } + return currentValue, nil + } + + if targetValue == "" { + return "", nil + } + + encrypted, err := crypto.Encrypt(targetValue) + if err != nil { + return "", fmt.Errorf("failed to encrypt api key: %w", err) + } + return encrypted, nil +} + // getConfigs 辅助方法 func (s *ConfigService) getConfigs(ctx context.Context, keys ...string) (map[string]string, error) { sc := s.q.SystemConfig diff --git a/backend/internal/util/quota.go b/backend/internal/util/quota.go new file mode 100644 index 000000000..40927d5b2 --- /dev/null +++ b/backend/internal/util/quota.go @@ -0,0 +1,88 @@ +//nolint:lll,mnd // Quota checks keep SQL and percentage thresholds inline for operational clarity. +package util + +import ( + "fmt" + + "k8s.io/klog/v2" + + "github.com/raids-lab/crater/dao/model" + "github.com/raids-lab/crater/dao/query" +) + +// CheckStorageQuota 检查用户存储是否超过理论配额，或作业是否被管理员冻结。 +// 任一条件成立时返回非 nil 错误，调用方应拒绝创建新作业。 +func CheckStorageQuota(username string) error { + db := query.GetDB() + + // 步骤 1：查用户 ID 和 space_quota + var baseRow struct { + ID uint `gorm:"column:id"` + SpaceQuota int64 `gorm:"column:space_quota"` + } + if err := db.Raw( + "SELECT id, space_quota FROM users WHERE name = ? AND deleted_at IS NULL", + username, + ).Scan(&baseRow).Error; err != nil || baseRow.ID == 0 { + klog.Warningf("CheckStorageQuota: user %q not found or query error, skip. err=%v id=%d", username, err, baseRow.ID) + return nil + } + + // 步骤 2：尝试获取 jobs_frozen + var frozenRow struct { + JobsFrozen bool `gorm:"column:jobs_frozen"` + } + if err := db.Raw("SELECT jobs_frozen FROM users WHERE id = ?", baseRow.ID).Scan(&frozenRow).Error; err == nil && frozenRow.JobsFrozen { + klog.Infof("CheckStorageQuota: user=%q jobs_frozen=true, blocking job creation", username) + return fmt.Errorf("管理员已暂停您的新作业创建权限，请联系管理员") + } + + theoreticalQuota := baseRow.SpaceQuota + + // 步骤 3：尝试获取 original_space_quota（临时扩容时才有值） + var origRow struct { + OriginalSpaceQuota *int64 `gorm:"column:original_space_quota"` + } + if err := db.Raw("SELECT original_space_quota FROM users WHERE id = ?", baseRow.ID).Scan(&origRow).Error; err == nil && origRow.OriginalSpaceQuota != nil { + theoreticalQuota = *origRow.OriginalSpaceQuota + } + + klog.Infof("CheckStorageQuota: user=%q id=%d space_quota=%d original_space_quota=%v theoretical=%d", + username, baseRow.ID, baseRow.SpaceQuota, origRow.OriginalSpaceQuota, theoreticalQuota) + + // -1 = 无限制，0 = 未设置，均跳过 + if theoreticalQuota <= 0 { + klog.Infof("CheckStorageQuota: user=%q quota=%d (unlimited/unset), skip", username, theoreticalQuota) + return nil + } + + // 步骤 4：从 user_space_sizes 取最近一次记录的用量 + var usage model.UserSpaceSize + if err := db.Where("user_id = ?", baseRow.ID).First(&usage).Error; err != nil { + klog.Warningf("CheckStorageQuota: user=%q no user_space_sizes record (err=%v), skip", username, err) + return nil + } + + klog.Infof("CheckStorageQuota: user=%q size=%d theoretical=%d (%.1f%%)", + username, usage.Size, theoreticalQuota, float64(usage.Size)/float64(theoreticalQuota)*100) + + if usage.Size >= theoreticalQuota { + return fmt.Errorf("存储空间已超过理论配额（已用 %s / 配额 %s），禁止创建新作业", + FormatStorageSize(usage.Size), FormatStorageSize(theoreticalQuota)) + } + return nil +} + +// FormatStorageSize 将字节数格式化为人类可读的字符串。 +func FormatStorageSize(bytes int64) string { + const unit = 1024 + if bytes < unit { + return fmt.Sprintf("%d B", bytes) + } + div, exp := int64(unit), 0 + for n := bytes / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp]) +} diff --git a/backend/pkg/ceph/ceph.go b/backend/pkg/ceph/ceph.go new file mode 100644 index 000000000..830f1a3df --- /dev/null +++ b/backend/pkg/ceph/ceph.go @@ -0,0 +1,658 @@ +//nolint:errorlint,gocritic,gocyclo,mnd,staticcheck // Ceph path/capacity discovery is infra-heavy and intentionally centralized. +package ceph + +import ( + "context" + "fmt" + "io" + "path" + "strconv" + "strings" + "sync" + + "net/http" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" + + cfgpkg "github.com/raids-lab/crater/pkg/config" +) + +const parentDir = ".." +const UnknownSizeBytes int64 = -1 + +func AvailableBytes(totalBytes, usedBytes int64) int64 { + if totalBytes < 0 || usedBytes < 0 { + return UnknownSizeBytes + } + return totalBytes - usedBytes +} + +var ( + // cephMountPathCache 缓存 CephFS 挂载路径 + cephMountPathCache string + // cephMountPathOnce 确保只初始化一次 + cephMountPathOnce sync.Once +) + +func sharedStoragePVCName() string { + storagePVCName := "crater-storage" + if cfg := cfgpkg.GetConfig(); cfg != nil { + if value := strings.TrimSpace(cfg.Storage.PVC.ReadWriteMany); value != "" { + storagePVCName = value + } + } + return storagePVCName +} + +// FindCephToolboxPod 查找 Rook-Ceph Toolbox Pod +func FindCephToolboxPod(clientset kubernetes.Interface, namespace string) (*corev1.Pod, error) { + pods, err := clientset.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: "app=rook-ceph-tools", + }) + if err != nil { + return nil, fmt.Errorf("列出 Pod 失败: %v", err) + } + + for _, pod := range pods.Items { + if pod.Status.Phase == corev1.PodRunning { + return &pod, nil + } + } + + return nil, fmt.Errorf("未找到运行中的 Rook-Ceph Toolbox Pod") +} + +// ExecInPod 在指定 Pod 中执行命令 +func ExecInPod(clientset kubernetes.Interface, config *rest.Config, pod *corev1.Pod, command []string) (string, error) { + var stdout, stderr strings.Builder + err := ExecInPodStream(clientset, config, pod, command, &stdout, &stderr) + if err != nil { + return "", fmt.Errorf("执行命令失败: %v, stderr: %s", err, stderr.String()) + } + + return stdout.String(), nil +} + +// ExecInPodStream 在指定 Pod 中执行命令，并将 stdout/stderr 流式写入给定 writer。 +func ExecInPodStream( + clientset kubernetes.Interface, + config *rest.Config, + pod *corev1.Pod, + command []string, + stdout io.Writer, + stderr io.Writer, +) error { + req := clientset.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(pod.Name). + Namespace(pod.Namespace). + SubResource("exec") + + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + return fmt.Errorf("添加 scheme 失败: %v", err) + } + + parameterCodec := runtime.NewParameterCodec(scheme) + req.VersionedParams(&corev1.PodExecOptions{ + Command: command, + Stdin: false, + Stdout: true, + Stderr: true, + TTY: false, + }, parameterCodec) + + exec, err := remotecommand.NewSPDYExecutor(config, http.MethodPost, req.URL()) + if err != nil { + return fmt.Errorf("创建 executor 失败: %v", err) + } + + if stdout == nil { + stdout = io.Discard + } + if stderr == nil { + stderr = io.Discard + } + + err = exec.Stream(remotecommand.StreamOptions{ + Stdout: stdout, + Stderr: stderr, + }) + if err != nil { + return err + } + + return nil +} + +// StoragePrefixConfig 存储路径前缀配置 +type StoragePrefixConfig struct { + User string + Account string + Public string +} + +// ResolveCephFSPath 将逻辑路径解析为 CephFS 中的实际路径 +// 逻辑路径格式: /user/{space}/... 或 /public/... 或 /account/{space}/... +// CephFS 实际路径格式: /mnt/mycephfs/volumes/csi/{pvc}/{prefix}/... +// 该函数通过在 toolbox pod 中执行 ls 命令自动查找 PVC 挂载路径 +func ResolveCephFSPath( + clientset kubernetes.Interface, + config *rest.Config, + namespace, logicalPath string, + prefixConfig StoragePrefixConfig, +) (string, error) { + toolboxPod, err := FindCephToolboxPod(clientset, namespace) + if err != nil { + return "", err + } + + trimmedPath := strings.TrimLeft(logicalPath, "/") + parts := strings.SplitN(trimmedPath, "/", 3) + + if len(parts) < 1 { + return "", fmt.Errorf("invalid path format: %s", logicalPath) + } + + var storagePrefix string + var remainingPath string + + switch parts[0] { + case "user": + if len(parts) < 2 { + return "", fmt.Errorf("user path must include space name: %s", logicalPath) + } + storagePrefix = prefixConfig.User + remainingPath = strings.Join(parts[1:], "/") + case "account": + storagePrefix = prefixConfig.Account + if len(parts) > 1 { + remainingPath = strings.Join(parts[1:], "/") + } + case "public": + storagePrefix = prefixConfig.Public + if len(parts) > 1 { + remainingPath = strings.Join(parts[1:], "/") + } + default: + return "", fmt.Errorf("unknown path type: %s", parts[0]) + } + + cephMountPath, err := findCephMountPath(clientset, config, toolboxPod) + if err != nil { + return "", err + } + + var fullPath string + if remainingPath != "" { + // 确保路径使用正斜杠，不包含 ./ 或 ../ + // 构建干净的路径 + var pathParts []string + + // 添加存储前缀 + if storagePrefix != "" { + pathParts = append(pathParts, storagePrefix) + } + + // 添加剩余路径部分 + if remainingPath != "" { + // 分割剩余路径并清理 + remainingParts := strings.Split(remainingPath, "/") + for _, part := range remainingParts { + if part != "" && part != "." { + if part == parentDir { + // 处理上级目录 + if len(pathParts) > 0 { + pathParts = pathParts[:len(pathParts)-1] + } + } else { + pathParts = append(pathParts, part) + } + } + } + } + + // 构建最终路径 + if len(pathParts) > 0 { + cleanPath := strings.Join(pathParts, "/") + fullPath = fmt.Sprintf("%s/%s", cephMountPath, cleanPath) + } else { + fullPath = cephMountPath + } + } else { + fullPath = fmt.Sprintf("%s/%s", cephMountPath, storagePrefix) + } + + // 确保最终路径使用正斜杠 + fullPath = strings.ReplaceAll(fullPath, "\\", "/") + // 清理路径，移除所有 ./ 和 ../ + fullPath = path.Clean(fullPath) + + return fullPath, nil +} + +// findCephMountPath 在 toolbox pod 中自动查找 CephFS 的 PVC 挂载路径 +// 通过 Kubernetes API 获取名为 crater-storage 的 PVC 信息，然后构建路径 +func findCephMountPath(clientset kubernetes.Interface, config *rest.Config, pod *corev1.Pod) (string, error) { + var err error + + // 使用 sync.Once 确保只执行一次路径发现 + cephMountPathOnce.Do(func() { + fmt.Println("=== 开始发现 CephFS PVC 路径 ===") + cephMountPathCache, err = discoverCephMountPath(clientset, config, pod) + if err != nil { + fmt.Printf("=== 路径发现失败: %v ===\n", err) + } else { + fmt.Printf("=== 路径发现成功: %s ===\n", cephMountPathCache) + } + }) + + if err != nil { + return "", err + } + + return cephMountPathCache, nil +} + +// discoverCephMountPath 实际执行路径发现 +func discoverCephMountPath(clientset kubernetes.Interface, config *rest.Config, pod *corev1.Pod) (string, error) { + storagePVCName := sharedStoragePVCName() + + // 查找共享存储 PVC + fmt.Printf("1. 查找名为 %s 的 PVC...\n", storagePVCName) + craterStoragePVC, err := clientset.CoreV1().PersistentVolumeClaims("").List(context.TODO(), metav1.ListOptions{ + FieldSelector: "metadata.name=" + storagePVCName, + }) + if err != nil { + fmt.Printf("查找 %s PVC 失败: %v\n", storagePVCName, err) + return "", fmt.Errorf("查找 %s PVC 失败: %v", storagePVCName, err) + } + + if len(craterStoragePVC.Items) == 0 { + fmt.Printf("未找到名为 %s 的 PVC\n", storagePVCName) + return "", fmt.Errorf("未找到名为 %s 的 PVC", storagePVCName) + } + + pvc := craterStoragePVC.Items[0] + pvName := pvc.Spec.VolumeName + if pvName == "" { + fmt.Printf("%s PVC 未绑定到 PV\n", storagePVCName) + return "", fmt.Errorf("%s PVC 未绑定到 PV", storagePVCName) + } + + fmt.Printf("找到 %s PVC，对应的 PV: %s\n", storagePVCName, pvName) + + // 获取对应的 PV + pv, err := clientset.CoreV1().PersistentVolumes().Get(context.TODO(), pvName, metav1.GetOptions{}) + if err != nil { + fmt.Printf("获取 PV %s 失败: %v\n", pvName, err) + return "", fmt.Errorf("获取 PV %s 失败: %v", pvName, err) + } + + // 检查是否是 cephfs PV + if pv.Spec.CSI == nil || pv.Spec.CSI.Driver != "rook-ceph.cephfs.csi.ceph.com" { + fmt.Printf("%s PV 不是 cephfs 类型\n", storagePVCName) + return "", fmt.Errorf("%s PV 不是 cephfs 类型", storagePVCName) + } + + fmt.Printf("%s PV 是 cephfs 类型\n", storagePVCName) + + if subvolumePath := strings.TrimSpace(pv.Spec.CSI.VolumeAttributes["subvolumePath"]); subvolumePath != "" { + pvcPath := path.Clean("/mnt/mycephfs/" + strings.TrimLeft(subvolumePath, "/")) + fmt.Printf("2. 从 PV volumeAttributes.subvolumePath 直接获取 PVC 路径: %s\n", pvcPath) + return pvcPath, nil + } + + // 检查 PV 的 volumeHandle + if pv.Spec.CSI.VolumeHandle == "" { + fmt.Printf("%s PV 没有 volumeHandle\n", storagePVCName) + return "", fmt.Errorf("%s PV 没有 volumeHandle", storagePVCName) + } + + volumeHandle := pv.Spec.CSI.VolumeHandle + fmt.Printf("crater-storage PV 的 volumeHandle: %s\n", volumeHandle) + + // 提取 UUID 部分 + parts := strings.Split(volumeHandle, "-") + if len(parts) < 5 { + fmt.Println("volumeHandle 格式不正确") + return "", fmt.Errorf("volumeHandle 格式不正确") + } + + // 从后往前查找 UUID 部分 + var uuid string + for i := len(parts) - 5; i >= 0; i-- { + if len(parts[i]) == 8 { + uuidParts := parts[i : i+5] + uuid = strings.Join(uuidParts, "-") + if len(uuid) == 36 { + break + } + } + } + + if uuid == "" { + fmt.Println("无法从 volumeHandle 中提取 UUID") + return "", fmt.Errorf("无法从 volumeHandle 中提取 UUID") + } + + fmt.Printf("提取 UUID: %s\n", uuid) + + // 构建 csi-vol- 名称 + csiVolName := "csi-vol-" + uuid + fmt.Printf("构建 csi-vol- 名称: %s\n", csiVolName) + + // 构建 PVC 路径 + csiPath := fmt.Sprintf("/mnt/mycephfs/volumes/csi/%s", csiVolName) + fmt.Printf("构建 PVC 路径: %s\n", csiPath) + + // 查找该路径下的唯一子文件夹 + fmt.Println("2. 正在查找子文件夹...") + command := []string{"ls", csiPath} + output, err := ExecInPod(clientset, config, pod, command) + if err != nil { + fmt.Printf("列出 PVC 目录失败: %v\n", err) + return "", fmt.Errorf("列出 PVC 目录失败: %v", err) + } + + fmt.Printf("PVC 目录内容: %s\n", output) + + lines := strings.Split(strings.TrimSpace(output), "\n") + var subDir string + for _, line := range lines { + if line != "." && line != ".." && !strings.HasPrefix(line, ".") { + subDir = line + fmt.Printf("找到子文件夹: %s\n", subDir) + break + } + } + + if subDir == "" { + fmt.Println("未找到子文件夹") + return "", fmt.Errorf("未找到子文件夹") + } + + // 构建完整的 PVC 挂载路径 + pvcPath := fmt.Sprintf("%s/%s", csiPath, subDir) + // 确保路径使用正斜杠 + pvcPath = strings.ReplaceAll(pvcPath, "\\", "/") + // 清理路径 + pvcPath = path.Clean(pvcPath) + + fmt.Printf("3. 构建完整 PVC 挂载路径: %s\n", pvcPath) + + return pvcPath, nil +} + +// GetCephDirectorySize 获取 CephFS 目录大小 +func GetCephDirectorySize( + clientset kubernetes.Interface, + config *rest.Config, + namespace, logicalPath string, + prefixConfig StoragePrefixConfig, +) (int64, error) { + toolboxPod, err := FindCephToolboxPod(clientset, namespace) + if err != nil { + return 0, err + } + + fullPath, err := ResolveCephFSPath(clientset, config, namespace, logicalPath, prefixConfig) + if err != nil { + return 0, err + } + + // 确保路径使用正斜杠 + fullPath = strings.ReplaceAll(fullPath, "\\", "/") + + command := []string{"getfattr", "-n", "ceph.dir.rbytes", fullPath} + output, err := ExecInPod(clientset, config, toolboxPod, command) + if err != nil { + return 0, err + } + + lines := strings.Split(output, "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "ceph.dir.rbytes=") { + sizeStr := strings.Trim(strings.TrimPrefix(line, "ceph.dir.rbytes="), "\"") + size, err := strconv.ParseInt(sizeStr, 10, 64) + if err != nil { + return 0, fmt.Errorf("解析大小失败: %v", err) + } + return size, nil + } + } + + return 0, fmt.Errorf("未找到 ceph.dir.rbytes 信息: %s", output) +} + +// SetCephDirectoryQuota 设置 CephFS 目录配额 +func SetCephDirectoryQuota( + clientset kubernetes.Interface, + config *rest.Config, + namespace, logicalPath string, + prefixConfig StoragePrefixConfig, + maxBytes int64, +) error { + toolboxPod, err := FindCephToolboxPod(clientset, namespace) + if err != nil { + return err + } + + fullPath, err := ResolveCephFSPath(clientset, config, namespace, logicalPath, prefixConfig) + if err != nil { + return err + } + + // 确保路径使用正斜杠 + fullPath = strings.ReplaceAll(fullPath, "\\", "/") + + // 检查路径是否存在 + checkCommand := []string{"ls", "-la", fullPath} + checkOutput, err := ExecInPod(clientset, config, toolboxPod, checkCommand) + if err != nil { + return fmt.Errorf("检查路径失败: %v, 输出: %s", err, checkOutput) + } + + // 构建 setfattr 命令 + // 注意：-1 表示无限制，需要设置为 0 + var command []string + if maxBytes == -1 { + // 移除配额限制（设置为 0） + command = []string{"setfattr", "-n", "ceph.quota.max_bytes", "-v", "0", fullPath} + } else { + // 设置配额限制 + command = []string{"setfattr", "-n", "ceph.quota.max_bytes", "-v", fmt.Sprintf("%d", maxBytes), fullPath} + } + + // 执行命令并获取详细输出 + output, err := ExecInPod(clientset, config, toolboxPod, command) + if err != nil { + return fmt.Errorf("设置配额失败: %v, 命令: %v, 输出: %s", err, command, output) + } + + return nil +} + +// GetCraterStorageCapacity 通过 Kubernetes API 读取 crater-storage PVC 的容量， +// 并通过 getfattr 读取已用量。 +// 返回 (totalBytes, usedBytes, error)。 +func GetCraterStorageCapacity(clientset kubernetes.Interface, config *rest.Config, namespace string) (int64, int64, error) { + storagePVCName := sharedStoragePVCName() + + // 1. 从 K8s API 读取 PVC 容量（不需要 exec） + // 使用提供的 namespace 参数，而不是空字符串 + pvcs, err := clientset.CoreV1().PersistentVolumeClaims(namespace).List(context.TODO(), metav1.ListOptions{ + FieldSelector: "metadata.name=" + storagePVCName, + }) + if err != nil { + return UnknownSizeBytes, UnknownSizeBytes, nil + } + + if len(pvcs.Items) == 0 { + // 如果在指定命名空间中找不到，尝试在所有命名空间中查找 + pvcs, err = clientset.CoreV1().PersistentVolumeClaims("").List(context.TODO(), metav1.ListOptions{ + FieldSelector: "metadata.name=" + storagePVCName, + }) + if err != nil || len(pvcs.Items) == 0 { + return UnknownSizeBytes, UnknownSizeBytes, nil + } + } + + pvc := pvcs.Items[0] + var totalBytes int64 + if cap, ok := pvc.Status.Capacity[corev1.ResourceStorage]; ok { + totalBytes = cap.Value() + } else if req, ok := pvc.Spec.Resources.Requests[corev1.ResourceStorage]; ok { + totalBytes = req.Value() + } else { + totalBytes = UnknownSizeBytes + } + + // 2. 通过 getfattr 读取 crater-storage PVC 对应 subvolume 的已用量 + toolboxPod, err := FindCephToolboxPod(clientset, namespace) + if err != nil { + return totalBytes, UnknownSizeBytes, nil + } + + // crater-storage 只是 CephFS 下的一个 subvolume，不应直接统计整个 /mnt/mycephfs 根目录。 + mountPath, err := findCephMountPath(clientset, config, toolboxPod) + if err != nil { + return totalBytes, UnknownSizeBytes, nil + } + + // 尝试获取 crater-storage subvolume 的已使用容量 + out, err := ExecInPod(clientset, config, toolboxPod, []string{"getfattr", "-n", "ceph.dir.rbytes", mountPath}) + if err != nil { + return totalBytes, UnknownSizeBytes, nil + } + + usedBytes := UnknownSizeBytes + for _, line := range strings.Split(out, "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "ceph.dir.rbytes=") { + valStr := strings.Trim(strings.TrimPrefix(line, "ceph.dir.rbytes="), "\"") + if v, parseErr := strconv.ParseInt(valStr, 10, 64); parseErr == nil { + usedBytes = v + } else { + return totalBytes, UnknownSizeBytes, nil + } + break + } + } + + // 确保返回有效的值 + if usedBytes == UnknownSizeBytes { + // 如果无法获取已用量，尝试使用 du 命令作为备选方案 + out, err := ExecInPod(clientset, config, toolboxPod, []string{"du", "-sb", mountPath}) + if err == nil { + parts := strings.Fields(out) + if len(parts) > 0 { + if v, parseErr := strconv.ParseInt(parts[0], 10, 64); parseErr == nil { + usedBytes = v + } + } + } + } + + return totalBytes, usedBytes, nil +} + +// GetCephMountRoot 获取 CephFS 挂载根路径（用于全局统计） +func GetCephMountRoot(clientset kubernetes.Interface, config *rest.Config, namespace string) (string, error) { + toolboxPod, err := FindCephToolboxPod(clientset, namespace) + if err != nil { + return "", err + } + return findCephMountPath(clientset, config, toolboxPod) +} + +// GetAllUserSpaceSizes 获取所有用户空间的大小 +func GetAllUserSpaceSizes( + clientset kubernetes.Interface, + config *rest.Config, + namespace string, + prefixConfig StoragePrefixConfig, + page, pageSize int, +) (map[string]int64, int, error) { + toolboxPod, err := FindCephToolboxPod(clientset, namespace) + if err != nil { + return nil, 0, err + } + + // 直接构建用户空间根目录路径 + cephMountPath, err := findCephMountPath(clientset, config, toolboxPod) + if err != nil { + return nil, 0, err + } + + // 构建用户空间根目录路径 + userRootPath := fmt.Sprintf("%s/%s", cephMountPath, prefixConfig.User) + // 确保路径使用正斜杠 + userRootPath = strings.ReplaceAll(userRootPath, "\\", "/") + + // 列出用户空间根目录下的所有子目录（即用户空间） + command := []string{"ls", userRootPath} + output, err := ExecInPod(clientset, config, toolboxPod, command) + if err != nil { + return nil, 0, fmt.Errorf("列出用户空间目录失败: %v", err) + } + + // 解析输出，获取所有用户空间名称 + lines := strings.Split(strings.TrimSpace(output), "\n") + + // 过滤有效用户目录 + var validUsers []string + for _, user := range lines { + user = strings.TrimSpace(user) + if user != "" && user != "." && user != ".." && !strings.HasPrefix(user, ".") { + validUsers = append(validUsers, user) + } + } + + // 计算总用户数 + totalUsers := len(validUsers) + + // 计算分页范围 + if page < 1 { + page = 1 + } + if pageSize < 1 { + pageSize = 10 + } + start := (page - 1) * pageSize + end := start + pageSize + if start >= totalUsers { + return make(map[string]int64), totalUsers, nil + } + if end > totalUsers { + end = totalUsers + } + + // 获取当前页的用户 + currentPageUsers := validUsers[start:end] + + // 对当前页的每个用户空间获取大小 + userSpaces := make(map[string]int64) + for _, user := range currentPageUsers { + // 构建用户空间的逻辑路径 + userPath := fmt.Sprintf("/user/%s", user) + + // 获取用户空间大小 + size, err := GetCephDirectorySize(clientset, config, namespace, userPath, prefixConfig) + if err != nil { + // 记录错误但继续处理其他用户 + fmt.Printf("获取用户 %s 空间大小失败: %v\n", user, err) + continue + } + + userSpaces[user] = size + } + + return userSpaces, totalUsers, nil +} diff --git a/backend/pkg/cronjob/manger.go b/backend/pkg/cronjob/manger.go index f4d9444a1..043f40433 100644 --- a/backend/pkg/cronjob/manger.go +++ b/backend/pkg/cronjob/manger.go @@ -1,21 +1,28 @@ package cronjob import ( + "context" + "fmt" "sync" "time" "github.com/robfig/cron/v3" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/raids-lab/crater/dao/model" "github.com/raids-lab/crater/pkg/cleaner" "github.com/raids-lab/crater/pkg/monitor" "github.com/raids-lab/crater/pkg/patrol" + "github.com/raids-lab/crater/pkg/storagegovernance" + "github.com/raids-lab/crater/pkg/storageindex" ) type CronJobManager struct { Client client.Client KubeClient kubernetes.Interface + KubeConfig *rest.Config PromClient monitor.PrometheusInterface cleanerClients *cleaner.Clients patrolClients *patrol.Clients @@ -26,13 +33,23 @@ type CronJobManager struct { func NewCronJobManager( cli client.Client, kubeClient kubernetes.Interface, + kubeConfig *rest.Config, promClient monitor.PrometheusInterface, gpuAnalysisService patrol.GpuAnalysisServiceInterface, billingService patrol.BillingServiceInterface, ) *CronJobManager { + decisionEngine := storagegovernance.NewEngine( + kubeClient, + kubeConfig, + promClient, + storagegovernance.DefaultConstraintConfig(), + ) + indexService := storageindex.NewService(kubeClient, kubeConfig) + return &CronJobManager{ Client: cli, KubeClient: kubeClient, + KubeConfig: kubeConfig, PromClient: promClient, cleanerClients: &cleaner.Clients{ Client: cli, @@ -42,10 +59,99 @@ func NewCronJobManager( patrolClients: &patrol.Clients{ Client: cli, KubeClient: kubeClient, + KubeConfig: kubeConfig, PromClient: promClient, GpuAnalysisService: gpuAnalysisService, BillingService: billingService, + RecordDecision: func(ctx context.Context, jobID string, action string, runErr error) { + _ = storagegovernance.MarkDecisionExecution(ctx, jobID, action, runErr) + }, + StorageAgent: func(tenantID string) (*patrol.AgentDecision, error) { + resp, jobID, err := decisionEngine.DecideAndRecord(context.Background(), storagegovernance.DecisionRequest{ + Username: tenantID, + Source: model.StorageDecisionSourcePatrol, + TriggerReason: "cronjob analyze-storage-alerts", + }) + if err != nil { + return nil, err + } + return &patrol.AgentDecision{ + AllowExpand: resp.AllowExpand, + ExpandBytes: resp.ExpandBytes, + FreezeNewJobs: resp.FreezeNewJobs, + Reason: resp.Reason, + DecisionJobID: jobID, + }, nil + }, + StorageAgentStart: func(ctx context.Context, tenantID string) (string, error) { + return decisionEngine.StartAsyncDecision(ctx, storagegovernance.DecisionRequest{ + Username: tenantID, + Source: model.StorageDecisionSourcePatrol, + TriggerReason: "cronjob analyze-storage-alerts", + }) + }, + StorageAgentAwait: awaitStorageDecisionResult, + StorageIndex: indexService, }, cron: cron.New(cron.WithLocation(time.Local)), } } + +func awaitStorageDecisionResult(ctx context.Context, tenantID, jobID string) (*patrol.AgentDecision, error) { + const ( + defaultTimeout = 5 * time.Minute + pollInterval = 2 * time.Second + ) + + if _, hasDeadline := ctx.Deadline(); !hasDeadline { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, defaultTimeout) + defer cancel() + } + + ticker := time.NewTicker(pollInterval) + defer ticker.Stop() + + for { + status, err := storagegovernance.GetDecisionStatus(ctx, jobID) + if err != nil { + return nil, fmt.Errorf("query storage decision status failed: %w", err) + } + + switch status.Status { + case string(model.StorageDecisionStatusDone): + if status.Result == nil { + return nil, fmt.Errorf("storage decision %s finished without result", jobID) + } + return &patrol.AgentDecision{ + AllowExpand: status.Result.AllowExpand, + ExpandBytes: status.Result.ExpandBytes, + FreezeNewJobs: status.Result.FreezeNewJobs, + Reason: status.Result.Reason, + DecisionJobID: jobID, + }, nil + case string(model.StorageDecisionStatusError): + if status.ErrorMsg == "" { + return nil, fmt.Errorf("storage decision %s failed", jobID) + } + return nil, fmt.Errorf("storage decision %s failed: %s", jobID, status.ErrorMsg) + case string(model.StorageDecisionStatusPending), string(model.StorageDecisionStatusRunning): + select { + case <-ctx.Done(): + return nil, fmt.Errorf("wait storage decision timeout for user %s job %s: %w", tenantID, jobID, ctx.Err()) + case <-ticker.C: + } + default: + select { + case <-ctx.Done(): + return nil, fmt.Errorf("wait storage decision timeout for user %s job %s: %w", tenantID, jobID, ctx.Err()) + case <-ticker.C: + } + } + } +} + +// GetPatrolClients returns the patrol clients +func (cm *CronJobManager) GetPatrolClients() *patrol.Clients { + return cm.patrolClients +} diff --git a/backend/pkg/cronjob/manger_test.go b/backend/pkg/cronjob/manger_test.go index 4d1a30dfa..dca910c3d 100644 --- a/backend/pkg/cronjob/manger_test.go +++ b/backend/pkg/cronjob/manger_test.go @@ -14,7 +14,7 @@ import ( func TestCronJob(t *testing.T) { t.Run("newCronJobFunc", func(t *testing.T) { - manager := NewCronJobManager(nil, nil, nil, nil, nil) + manager := NewCronJobManager(nil, nil, nil, nil, nil, nil) PatchConvey("newCronJobFunc", t, func() { jobName := cleaner.CLEAN_LONG_TIME_RUNNING_JOB jobConfig := datatypes.JSON(`{"batchDays": 4, "interactiveDays": 4}`) @@ -50,7 +50,7 @@ func TestCronJob(t *testing.T) { t.Run("prepareUpdateConfig", func(t *testing.T) { PatchConvey("prepareUpdateConfig", t, func() { - manager := NewCronJobManager(nil, nil, nil, nil, nil) + manager := NewCronJobManager(nil, nil, nil, nil, nil, nil) cur := &model.CronJobConfig{ Name: "test", Type: model.CronJobTypeCleanerFunc, diff --git a/backend/pkg/llm/direct_decision.go b/backend/pkg/llm/direct_decision.go new file mode 100644 index 000000000..f519a97d3 --- /dev/null +++ b/backend/pkg/llm/direct_decision.go @@ -0,0 +1,525 @@ +package llm + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" + + "k8s.io/klog/v2" + + "github.com/raids-lab/crater/dao/model" + "github.com/raids-lab/crater/dao/query" + "github.com/raids-lab/crater/pkg/crypto" +) + +const ( + StorageDecisionModeEnv = "CRATER_STORAGE_DECISION_MODE" + StorageDecisionModeAgent = "agent" + StorageDecisionModeDirect = "direct" + StorageDecisionConfigSourceEnv = "CRATER_STORAGE_DECISION_CONFIG_SOURCE" + StorageDecisionConfigSourcePlatform = "platform" + StorageDecisionConfigSourceCustom = "custom" + + DirectModelBaseURLEnv = "CRATER_STORAGE_DIRECT_MODEL_BASE_URL" + DirectModelAPIKeyEnv = "CRATER_STORAGE_DIRECT_MODEL_API_KEY" + DirectModelNameEnv = "CRATER_STORAGE_DIRECT_MODEL_NAME" + DefaultDirectModelBaseURL = "http://192.168.5.68:30186/v1" + + directDecisionInstruction = `你是面向 AI 集群的存储治理决策模型。请根据输入的结构化存储治理快照，输出一个 JSON 决策对象。 +输出要求： +1. 只输出 JSON，不要附加解释文本。 +2. JSON 字段固定为，且顺序必须如下： + - reason: string + - allow_expand: boolean + - expand_bytes: integer + - freeze_new_jobs: boolean +3. reason 需要简洁说明决策依据，优先引用 usage_ratio、growth_rate_bytes_per_hour、平台容量和 Prometheus 相关字段。 +4. reason 必须与 allow_expand、expand_bytes、freeze_new_jobs 严格一致： + - allow_expand=false 时，reason 不得写成“建议扩容”“应扩容”或任何支持扩容的表述 + - allow_expand=true 时，expand_bytes 必须为正数，reason 需要明确支持扩容 + - freeze_new_jobs=true 时，reason 必须明确说明需要冻结新作业 + - freeze_new_jobs=false 时，reason 不得写成“建议冻结新作业”或任何支持冻结的表述 +5. expand_bytes 必须使用字节数。 +6. 不要重复输入，不要输出 markdown，不要输出额外内容。` + + directDecisionRefinementInstruction = `你是存储治理决策的一致性修正器。你会看到： +1. 输入 snapshot +2. 首轮 JSON 决策 +3. 外部一致性验证器指出的冲突点 + +请你重写一个新的最终 JSON 决策对象，并修复所有冲突。 +要求： +1. 只输出 JSON，不要解释修改过程。 +2. reason 必须与 allow_expand、expand_bytes、freeze_new_jobs 严格一致。 +3. allow_expand=false 时 expand_bytes 必须为 0。 +4. allow_expand=true 时 expand_bytes 必须为正数。 +5. freeze_new_jobs=true 时 reason 必须明确写出冻结新作业的原因；freeze_new_jobs=false 时 reason 不得写成建议冻结。 +6. 直接基于 snapshot 的证据重写，不要保留首轮输出里自相矛盾的描述。` +) + +type storageDecisionRuntimeConfig struct { + Mode string + ConfigSource string + BaseURL string + APIKey string + ModelName string +} + +type directDecisionAttempt struct { + Decision *LLMDecisionResponse + RawJSON string + RawText string +} + +type directDecisionValidationIssue struct { + Code string `json:"code"` + Message string `json:"message"` +} + +// GetStorageDecisionMode returns the active storage decision mode. +// Database config is used by default, while environment variables can still +// temporarily override it for local testing. +func GetStorageDecisionMode(ctx context.Context) string { + cfg, err := loadStorageDecisionRuntimeConfig(ctx) + if err != nil { + klog.Warningf("GetStorageDecisionMode: failed to load runtime config, fallback to agent: %v", err) + return StorageDecisionModeAgent + } + return normalizeStorageDecisionMode(cfg.Mode) +} + +// AskDirectDecision sends a precomputed storage snapshot to a specialized model. +// It performs a first-pass generation, validates reason/decision consistency, and +// when conflicts are found, feeds them back to the model for one rewrite. +func AskDirectDecision(ctx context.Context, snapshotJSON string) (*LLMDecisionResponse, error) { + llmConfig, err := loadDirectDecisionConfig(ctx) + if err != nil { + return nil, fmt.Errorf("加载运行时 LLM 配置失败: %w", err) + } + + firstAttempt, err := askDirectDecisionOnce(ctx, *llmConfig, snapshotJSON) + if err != nil { + return nil, err + } + + firstIssues := validateDirectDecisionConsistency(*firstAttempt.Decision) + if len(firstIssues) == 0 { + return firstAttempt.Decision, nil + } + + klog.Warningf( + "AskDirectDecision: first-pass direct decision has %d consistency issue(s): %s", + len(firstIssues), + joinValidationIssueMessages(firstIssues), + ) + + refinedAttempt, err := refineDirectDecision( + ctx, + *llmConfig, + snapshotJSON, + firstAttempt.RawJSON, + firstIssues, + ) + if err != nil { + klog.Warningf("AskDirectDecision: refinement failed, keeping first-pass decision: %v", err) + return firstAttempt.Decision, nil + } + + refinedIssues := validateDirectDecisionConsistency(*refinedAttempt.Decision) + switch { + case len(refinedIssues) == 0: + return refinedAttempt.Decision, nil + case len(refinedIssues) < len(firstIssues): + klog.Warningf( + "AskDirectDecision: refined decision still has %d issue(s), but improved from %d: %s", + len(refinedIssues), + len(firstIssues), + joinValidationIssueMessages(refinedIssues), + ) + return refinedAttempt.Decision, nil + default: + klog.Warningf( + "AskDirectDecision: refinement did not improve consistency, keeping first-pass decision. first=%d refined=%d refinedIssues=%s", + len(firstIssues), + len(refinedIssues), + joinValidationIssueMessages(refinedIssues), + ) + return firstAttempt.Decision, nil + } +} + +func askDirectDecisionOnce(ctx context.Context, cfg ProviderConfig, snapshotJSON string) (*directDecisionAttempt, error) { + temperature := 0.0 + prompt := directDecisionInstruction + "\n\n输入:\n" + snapshotJSON + "\n\n输出:\n" + resp, err := callConfiguredCompletion(ctx, cfg, completionRequest{ + Prompt: prompt, + MaxTokens: 256, + Temperature: &temperature, + }) + if err != nil { + return nil, fmt.Errorf("调用直连决策模型失败: %w", err) + } + if len(resp.Choices) == 0 { + return nil, fmt.Errorf("直连决策模型未返回任何候选结果") + } + + return decodeDirectDecisionAttempt(resp.Choices[0].Text) +} + +func refineDirectDecision( + ctx context.Context, + cfg ProviderConfig, + snapshotJSON string, + firstDecisionJSON string, + issues []directDecisionValidationIssue, +) (*directDecisionAttempt, error) { + feedbackJSON, err := json.MarshalIndent(issues, "", " ") + if err != nil { + return nil, fmt.Errorf("failed to marshal direct decision validation feedback: %w", err) + } + + temperature := 0.0 + prompt := directDecisionRefinementInstruction + + "\n\nsnapshot:\n" + snapshotJSON + + "\n\n首轮决策:\n" + firstDecisionJSON + + "\n\n一致性验证反馈:\n" + string(feedbackJSON) + + "\n\n请输出修复后的 JSON 决策:\n" + + resp, err := callConfiguredCompletion(ctx, cfg, completionRequest{ + Prompt: prompt, + MaxTokens: 256, + Temperature: &temperature, + }) + if err != nil { + return nil, fmt.Errorf("failed to refine direct decision: %w", err) + } + if len(resp.Choices) == 0 { + return nil, fmt.Errorf("direct decision refinement returned no choices") + } + + return decodeDirectDecisionAttempt(resp.Choices[0].Text) +} + +func decodeDirectDecisionAttempt(rawText string) (*directDecisionAttempt, error) { + rawJSON := extractFirstJSONObject(rawText) + + var decision LLMDecisionResponse + if err := json.Unmarshal([]byte(rawJSON), &decision); err != nil { + return nil, fmt.Errorf("解析直连决策 JSON 失败: %w\n原始响应: %s", err, rawText) + } + + return &directDecisionAttempt{ + Decision: &decision, + RawJSON: rawJSON, + RawText: rawText, + }, nil +} + +func loadDirectDecisionConfig(ctx context.Context) (*ProviderConfig, error) { + return GetStorageDecisionProviderConfig(ctx) +} + +func GetStorageDecisionProviderConfig(ctx context.Context) (*ProviderConfig, error) { + storageCfg, err := loadStorageDecisionRuntimeConfig(ctx) + if err != nil { + return nil, err + } + + if storageCfg.ConfigSource == StorageDecisionConfigSourceCustom { + if strings.TrimSpace(storageCfg.BaseURL) == "" { + return nil, fmt.Errorf("storage decision base url is not configured") + } + if strings.TrimSpace(storageCfg.ModelName) == "" { + return nil, fmt.Errorf("storage decision model name is not configured") + } + + return &ProviderConfig{ + BaseURL: strings.TrimSpace(storageCfg.BaseURL), + APIKey: strings.TrimSpace(storageCfg.APIKey), + ModelName: strings.TrimSpace(storageCfg.ModelName), + }, nil + } + + return loadRuntimeLLMConfig(ctx) +} + +func loadStorageDecisionRuntimeConfig(ctx context.Context) (*storageDecisionRuntimeConfig, error) { + cfg := &storageDecisionRuntimeConfig{} + + var rows []model.SystemConfig + err := query.GetDB().WithContext(ctx). + Where("key IN ?", []string{ + model.ConfigKeyStorageDecisionMode, + model.ConfigKeyStorageDecisionConfigSource, + model.ConfigKeyStorageDirectModelBaseURL, + model.ConfigKeyStorageDirectModelAPIKey, + model.ConfigKeyStorageDirectModelName, + }). + Find(&rows).Error + if err != nil { + return nil, fmt.Errorf("failed to load storage decision config from database: %w", err) + } + + configMap := make(map[string]string, len(rows)) + for _, row := range rows { + configMap[row.Key] = strings.TrimSpace(row.Value) + } + + cfg.Mode = normalizeStorageDecisionMode(configMap[model.ConfigKeyStorageDecisionMode]) + cfg.ConfigSource = normalizeStorageDecisionConfigSource( + configMap[model.ConfigKeyStorageDecisionConfigSource], + configMap[model.ConfigKeyStorageDirectModelName], + ) + cfg.BaseURL = configMap[model.ConfigKeyStorageDirectModelBaseURL] + cfg.ModelName = configMap[model.ConfigKeyStorageDirectModelName] + + if encryptedKey := configMap[model.ConfigKeyStorageDirectModelAPIKey]; encryptedKey != "" { + plainKey, decryptErr := crypto.Decrypt(encryptedKey) + if decryptErr != nil { + klog.Warningf("loadStorageDecisionRuntimeConfig: failed to decrypt direct model api key, using raw value: %v", decryptErr) + cfg.APIKey = encryptedKey + } else { + cfg.APIKey = plainKey + } + } + + if envMode := strings.TrimSpace(os.Getenv(StorageDecisionModeEnv)); envMode != "" { + cfg.Mode = normalizeStorageDecisionMode(envMode) + } + if envSource := strings.TrimSpace(os.Getenv(StorageDecisionConfigSourceEnv)); envSource != "" { + cfg.ConfigSource = normalizeStorageDecisionConfigSource(envSource, cfg.ModelName) + } + if envBaseURL := strings.TrimSpace(os.Getenv(DirectModelBaseURLEnv)); envBaseURL != "" { + cfg.BaseURL = envBaseURL + } + if envAPIKey := strings.TrimSpace(os.Getenv(DirectModelAPIKeyEnv)); envAPIKey != "" { + cfg.APIKey = envAPIKey + } + if envModelName := strings.TrimSpace(os.Getenv(DirectModelNameEnv)); envModelName != "" { + cfg.ModelName = envModelName + } + if cfg.ConfigSource == StorageDecisionConfigSourceCustom && strings.TrimSpace(cfg.BaseURL) == "" { + cfg.BaseURL = DefaultDirectModelBaseURL + } + + return cfg, nil +} + +func normalizeStorageDecisionMode(mode string) string { + switch strings.TrimSpace(strings.ToLower(mode)) { + case StorageDecisionModeDirect: + return StorageDecisionModeDirect + default: + return StorageDecisionModeAgent + } +} + +func normalizeStorageDecisionConfigSource(source, modelName string) string { + switch strings.TrimSpace(strings.ToLower(source)) { + case StorageDecisionConfigSourceCustom: + return StorageDecisionConfigSourceCustom + case StorageDecisionConfigSourcePlatform: + return StorageDecisionConfigSourcePlatform + default: + if strings.TrimSpace(modelName) != "" { + return StorageDecisionConfigSourceCustom + } + return StorageDecisionConfigSourcePlatform + } +} + +func extractFirstJSONObject(text string) string { + start := strings.Index(text, "{") + if start == -1 { + return strings.TrimSpace(text) + } + + depth := 0 + inString := false + escaped := false + for i := start; i < len(text); i++ { + ch := text[i] + + if escaped { + escaped = false + continue + } + if ch == '\\' { + escaped = true + continue + } + if ch == '"' { + inString = !inString + continue + } + if inString { + continue + } + + switch ch { + case '{': + depth++ + case '}': + depth-- + if depth == 0 { + return strings.TrimSpace(text[start : i+1]) + } + } + } + + return strings.TrimSpace(text[start:]) +} + +func validateDirectDecisionConsistency(decision LLMDecisionResponse) []directDecisionValidationIssue { + issues := make([]directDecisionValidationIssue, 0) + reason := strings.TrimSpace(strings.ToLower(decision.Reason)) + + if reason == "" { + issues = append(issues, directDecisionValidationIssue{ + Code: "empty_reason", + Message: "reason is empty and does not explain the decision fields", + }) + } + + if decision.AllowExpand && decision.ExpandBytes <= 0 { + issues = append(issues, directDecisionValidationIssue{ + Code: "expand_bytes_missing", + Message: fmt.Sprintf( + "allow_expand=true but expand_bytes=%d; expansion decisions must use a positive expand_bytes", + decision.ExpandBytes, + ), + }) + } + if !decision.AllowExpand && decision.ExpandBytes != 0 { + issues = append(issues, directDecisionValidationIssue{ + Code: "expand_bytes_should_be_zero", + Message: fmt.Sprintf("allow_expand=false but expand_bytes=%d; expand_bytes must be 0 when expansion is disabled", decision.ExpandBytes), + }) + } + + if reason == "" { + return issues + } + + hasExpandNegative := containsReasonPhrase(reason, directExpandNegativePhrases) + hasExpandPositive := containsReasonPhrase( + removeReasonPhrases(reason, directExpandNegativePhrases), + directExpandPositivePhrases, + ) + hasFreezeNegative := containsReasonPhrase(reason, directFreezeNegativePhrases) + hasFreezePositive := containsReasonPhrase( + removeReasonPhrases(reason, directFreezeNegativePhrases), + directFreezePositivePhrases, + ) + + if decision.AllowExpand && hasExpandNegative { + issues = append(issues, directDecisionValidationIssue{ + Code: "reason_blocks_expansion", + Message: "allow_expand=true but reason describes expansion as unnecessary, blocked, or forbidden", + }) + } + if !decision.AllowExpand && hasExpandPositive { + issues = append(issues, directDecisionValidationIssue{ + Code: "reason_supports_expansion", + Message: "allow_expand=false but reason still recommends or supports expansion", + }) + } + if decision.FreezeNewJobs && hasFreezeNegative { + issues = append(issues, directDecisionValidationIssue{ + Code: "reason_blocks_freeze", + Message: "freeze_new_jobs=true but reason says freezing new jobs is unnecessary or should not happen", + }) + } + if !decision.FreezeNewJobs && hasFreezePositive { + issues = append(issues, directDecisionValidationIssue{ + Code: "reason_supports_freeze", + Message: "freeze_new_jobs=false but reason still recommends freezing or pausing new jobs", + }) + } + + return issues +} + +func joinValidationIssueMessages(issues []directDecisionValidationIssue) string { + parts := make([]string, 0, len(issues)) + for _, issue := range issues { + parts = append(parts, issue.Code+": "+issue.Message) + } + return strings.Join(parts, "; ") +} + +func containsReasonPhrase(reason string, phrases []string) bool { + for _, phrase := range phrases { + if strings.Contains(reason, phrase) { + return true + } + } + return false +} + +func removeReasonPhrases(reason string, phrases []string) string { + cleaned := reason + for _, phrase := range phrases { + cleaned = strings.ReplaceAll(cleaned, phrase, " ") + } + return cleaned +} + +var directExpandPositivePhrases = []string{ + "建议扩容", + "需要扩容", + "应当扩容", + "应该扩容", + "可以扩容", + "允许扩容", + "优先扩容", + "保守扩容", + "临时扩容", + "扩容保护", + "expand", + "allow expand", +} + +var directExpandNegativePhrases = []string{ + "无需扩容", + "不需要扩容", + "不应扩容", + "不应该扩容", + "不建议扩容", + "不能扩容", + "不可扩容", + "不允许扩容", + "禁止扩容", + "无需临时扩容", + "no expand", + "do not expand", + "deny expansion", +} + +var directFreezePositivePhrases = []string{ + "冻结新作业", + "冻结新任务", + "暂停新作业", + "暂停新任务", + "禁止新作业", + "禁止新任务", + "停止新作业", + "停止新任务", + "freeze new jobs", + "freeze new job", + "freeze jobs", +} + +var directFreezeNegativePhrases = []string{ + "不冻结新作业", + "不冻结新任务", + "无需冻结", + "不需要冻结", + "不必冻结", + "继续接受新作业", + "继续提交新作业", + "no freeze", + "do not freeze", +} diff --git a/backend/pkg/llm/direct_decision_test.go b/backend/pkg/llm/direct_decision_test.go new file mode 100644 index 000000000..53c9bd41a --- /dev/null +++ b/backend/pkg/llm/direct_decision_test.go @@ -0,0 +1,118 @@ +package llm + +import "testing" + +func TestValidateDirectDecisionConsistencyFieldConflicts(t *testing.T) { + tests := []struct { + name string + decision LLMDecisionResponse + wantCode string + }{ + { + name: "expand enabled but bytes missing", + decision: LLMDecisionResponse{ + Reason: "建议扩容保护作业", + AllowExpand: true, + ExpandBytes: 0, + }, + wantCode: "expand_bytes_missing", + }, + { + name: "expand disabled but bytes kept", + decision: LLMDecisionResponse{ + Reason: "无需扩容，保持观察", + AllowExpand: false, + ExpandBytes: 1024, + }, + wantCode: "expand_bytes_should_be_zero", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + issues := validateDirectDecisionConsistency(tc.decision) + if !hasValidationIssue(issues, tc.wantCode) { + t.Fatalf("expected issue %q, got %+v", tc.wantCode, issues) + } + }) + } +} + +func TestValidateDirectDecisionConsistencyReasonConflicts(t *testing.T) { + tests := []struct { + name string + decision LLMDecisionResponse + wantCode string + }{ + { + name: "allow expand but reason says no expansion", + decision: LLMDecisionResponse{ + Reason: "当前无需扩容，继续观察即可", + AllowExpand: true, + ExpandBytes: 1024, + }, + wantCode: "reason_blocks_expansion", + }, + { + name: "disallow expand but reason still supports expansion", + decision: LLMDecisionResponse{ + Reason: "建议扩容保护落盘阶段", + AllowExpand: false, + ExpandBytes: 0, + }, + wantCode: "reason_supports_expansion", + }, + { + name: "freeze enabled but reason says no freeze", + decision: LLMDecisionResponse{ + Reason: "当前不需要冻结新作业，只需观察", + FreezeNewJobs: true, + AllowExpand: false, + ExpandBytes: 0, + }, + wantCode: "reason_blocks_freeze", + }, + { + name: "freeze disabled but reason still recommends freeze", + decision: LLMDecisionResponse{ + Reason: "建议冻结新作业，避免继续上涨", + FreezeNewJobs: false, + AllowExpand: false, + ExpandBytes: 0, + }, + wantCode: "reason_supports_freeze", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + issues := validateDirectDecisionConsistency(tc.decision) + if !hasValidationIssue(issues, tc.wantCode) { + t.Fatalf("expected issue %q, got %+v", tc.wantCode, issues) + } + }) + } +} + +func TestValidateDirectDecisionConsistencyAlignedDecision(t *testing.T) { + decision := LLMDecisionResponse{ + Reason: "usage_ratio 接近阈值且平台仍有余量，建议扩容 21474836480 字节保护作业，本轮不冻结新作业", + AllowExpand: true, + ExpandBytes: 21474836480, + FreezeNewJobs: false, + } + + issues := validateDirectDecisionConsistency(decision) + if len(issues) != 0 { + t.Fatalf("expected no issues, got %+v", issues) + } +} + +func hasValidationIssue(issues []directDecisionValidationIssue, code string) bool { + for _, issue := range issues { + if issue.Code == code { + return true + } + } + return false +} diff --git a/backend/pkg/llm/llm_tools.go b/backend/pkg/llm/llm_tools.go new file mode 100644 index 000000000..d63b38d45 --- /dev/null +++ b/backend/pkg/llm/llm_tools.go @@ -0,0 +1,1008 @@ +//nolint:gocritic,gocyclo,mnd,lll,staticcheck,unused // Storage governance tool wiring keeps prompts, handlers, and K8s traversal centralized. +package llm + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + + "github.com/raids-lab/crater/dao/model" + "github.com/raids-lab/crater/dao/query" + "github.com/raids-lab/crater/pkg/ceph" + "github.com/raids-lab/crater/pkg/config" + "github.com/raids-lab/crater/pkg/monitor" +) + +const ( + labelKeyTaskUser = "crater.raids.io/task-user" +) + +// ---- 业务响应类型 ---- + +type LLMDecisionResponse struct { + Reason string `json:"reason"` + AllowExpand bool `json:"allow_expand"` + ExpandBytes int64 `json:"expand_bytes"` + FreezeNewJobs bool `json:"freeze_new_jobs"` +} + +type PlatformCapacityResponse struct { + TotalCapacity int64 `json:"total_capacity"` + UsedCapacity int64 `json:"used_capacity"` + AvailableCapacity int64 `json:"available_capacity"` +} + +type TenantPodResponse struct { + PodName string `json:"pod_name"` + Phase string `json:"phase"` + GPURrequests int `json:"gpu_requests"` +} + +type TenantPodsResponse struct { + TenantID string `json:"tenant_id"` + Pods []TenantPodResponse `json:"pods"` +} + +type PodDetailsResponse struct { + PodName string `json:"pod_name"` + StartTime time.Time `json:"start_time"` + RunningTime int `json:"running_time"` + ContainerImages []string `json:"container_images"` + RestartCount int `json:"restart_count"` + GPUModel string `json:"gpu_model"` + GPUCount int `json:"gpu_count"` + CPUCount int `json:"cpu_count"` + MemorySize string `json:"memory_size"` +} + +type ComputeQuotaResponse struct { + TenantID string `json:"tenant_id"` + GPULimit int `json:"gpu_limit"` + GPURequest int `json:"gpu_request"` + CPULimit int `json:"cpu_limit"` + CPURequest int `json:"cpu_request"` + MemoryLimit string `json:"memory_limit"` + MemoryRequest string `json:"memory_request"` +} + +type UsageTrend struct { + Timestamp time.Time `json:"timestamp"` + UsageBytes int64 `json:"usage_bytes"` +} + +type TenantStorageTrendResponse struct { + TenantID string `json:"tenant_id"` + CurrentUsage int64 `json:"current_usage"` + History []UsageTrend `json:"history"` +} + +// ---- DeepSeek / OpenAI 兼容类型 ---- + +type dsToolFunction struct { + Name string `json:"name"` + Description string `json:"description"` + Parameters any `json:"parameters"` +} + +type dsTool struct { + Type string `json:"type"` // "function" + Function dsToolFunction `json:"function"` +} + +type dsToolCallFunction struct { + Name string `json:"name"` + Arguments string `json:"arguments"` +} + +type dsToolCall struct { + ID string `json:"id"` + Type string `json:"type"` + Function dsToolCallFunction `json:"function"` +} + +type dsMessage struct { + Role string `json:"role"` + Content string `json:"content,omitempty"` + ToolCalls []dsToolCall `json:"tool_calls,omitempty"` + ToolCallID string `json:"tool_call_id,omitempty"` +} + +type dsRequest struct { + Model string `json:"model"` + Messages []dsMessage `json:"messages"` + Tools []dsTool `json:"tools,omitempty"` + MaxTokens int `json:"max_tokens,omitempty"` +} + +type dsResponse struct { + Choices []struct { + Message dsMessage `json:"message"` + FinishReason string `json:"finish_reason"` + } `json:"choices"` + Error *struct { + Message string `json:"message"` + } `json:"error,omitempty"` +} + +// ---- 工具定义 ---- + +func getTools() []dsTool { + return []dsTool{ + { + Type: "function", + Function: dsToolFunction{ + Name: "query_platform_capacity", + Description: "获取整个平台（硬限制 8TB）的总量、已用量、可用量", + Parameters: map[string]any{ + "type": "object", + "properties": map[string]any{}, + "required": []string{}, + }, + }, + }, + { + Type: "function", + Function: dsToolFunction{ + Name: "list_tenant_pods", + Description: "列出租户当前活跃的 Pod 列表", + Parameters: map[string]any{ + "type": "object", + "properties": map[string]any{ + "tenant_id": map[string]any{"type": "string", "description": "租户 ID"}, + }, + "required": []string{"tenant_id"}, + }, + }, + }, + { + Type: "function", + Function: dsToolFunction{ + Name: "inspect_pod_details", + Description: "深入查看某个特定 Pod 的启动时间、镜像和重启次数、GPU型号、个数，CPU核数和内存大小", + Parameters: map[string]any{ + "type": "object", + "properties": map[string]any{ + "tenant_id": map[string]any{"type": "string", "description": "租户 ID"}, + "pod_name": map[string]any{"type": "string", "description": "Pod 名称"}, + }, + "required": []string{"tenant_id", "pod_name"}, + }, + }, + }, + { + Type: "function", + Function: dsToolFunction{ + Name: "get_tenant_compute_quota", + Description: "获取租户当前所有活跃 Pod 的 GPU/CPU/内存请求与限制总量", + Parameters: map[string]any{ + "type": "object", + "properties": map[string]any{ + "tenant_id": map[string]any{"type": "string", "description": "租户 ID"}, + }, + "required": []string{"tenant_id"}, + }, + }, + }, + { + Type: "function", + Function: dsToolFunction{ + Name: "query_tenant_storage_trend", + Description: "获取指定租户当前的真实存储占用以及最近的几次历史记录，用于推导增长斜率", + Parameters: map[string]any{ + "type": "object", + "properties": map[string]any{ + "tenant_id": map[string]any{"type": "string", "description": "租户 ID"}, + }, + "required": []string{"tenant_id"}, + }, + }, + }, + { + Type: "function", + Function: dsToolFunction{ + Name: "query_pod_realtime_metrics", + Description: "通过 Prometheus 查询指定 Pod 过去 5 分钟的真实资源利用率（CPU 核数、内存 MB、GPU 利用率 %、GPU 显存 MB）。用于检测僵尸作业：gpu_data_available=true 时以 gpu_util_percent 判断，gpu_data_available=false 时以 cpu_cores 判断（< 0.05 核视为进程挂死）", + Parameters: map[string]any{ + "type": "object", + "properties": map[string]any{ + "pod_name": map[string]any{"type": "string", "description": "Pod 名称"}, + }, + "required": []string{"pod_name"}, + }, + }, + }, + { + Type: "function", + Function: dsToolFunction{ + Name: "query_pod_gpu_history", + Description: "查询指定 Pod 在整个生命周期（或最近 24 小时）内的 GPU 历史利用率，返回平均值和最大值。用于区分两种低 GPU 利用率场景：(1) 作业曾经高强度使用 GPU（max_util_ever > 50%），当前低利用率说明正处于落盘/IO 阶段，是有价值的作业；(2) 从未有过高 GPU 利用率（max_util_ever ≈ 0%），则可能是僵尸作业或纯 CPU 作业。data_available=false 表示 DCGM 未采集到该 Pod 数据", + Parameters: map[string]any{ + "type": "object", + "properties": map[string]any{ + "pod_name": map[string]any{"type": "string", "description": "Pod 名称"}, + "duration_hours": map[string]any{"type": "number", "description": "查询历史时长（小时），默认 24。建议传入作业已运行时长以覆盖完整生命周期"}, + }, + "required": []string{"pod_name"}, + }, + }, + }, + { + Type: "function", + Function: dsToolFunction{ + Name: "diagnose_prometheus", + Description: "诊断 Prometheus 连通性与 DCGM 监控可用性。返回：Prometheus 是否可达、DCGM 指标是否存在、当前 namespace 下被 DCGM 追踪的 Pod 列表，以及（可选）指定 Pod 的原始指标查询结果。当 query_pod_realtime_metrics 返回 gpu_data_available=false 时必须调用此工具排查原因", + Parameters: map[string]any{ + "type": "object", + "properties": map[string]any{ + "pod_name": map[string]any{"type": "string", "description": "（可选）需要额外查询实时指标的 Pod 名称"}, + }, + "required": []string{}, + }, + }, + }, + } +} + +// ---- Agent 主循环 ---- + +func AskAgentForDecision(clientset kubernetes.Interface, restConfig *rest.Config, tenantID string, promClient monitor.PrometheusInterface) (*LLMDecisionResponse, error) { + ctx := context.Background() + llmConfig, err := GetStorageDecisionProviderConfig(ctx) + if err != nil { + return nil, fmt.Errorf("加载存储决策 LLM 配置失败: %w", err) + } + skillText, skillSource, err := loadStorageAgentSkill() + if err != nil { + return nil, fmt.Errorf("加载 storage agent skill 失败: %w", err) + } + + const ( + systemPrompt = `你是一个平台运维 AI 助手，负责分析租户存储告警并给出临时扩容决策。 + +【背景知识】 +GPU 密集型训练完成后，作业通常进入 IO 密集型落盘阶段（写 checkpoint / 模型参数）。 +此阶段特征：GPU 利用率降至 0、CPU 极低（IO bound）、存储快速增长。 +这是有价值的正常行为，必须优先保护。区分落盘作业与僵尸作业的关键依据是 GPU 历史记录： + · 落盘作业：历史上曾有过高 GPU 利用率（max_util_ever > 50%），说明完成了真实的 GPU 计算 + · 僵尸作业：整个生命周期 GPU 利用率始终接近 0，从未进行有效计算 + +【工具调用策略 — 请严格按序执行，禁止冗余调用】 + +第一步（必做）：调用 query_tenant_storage_trend。 + 从 history 记录计算增长速率（growth_rate）： + · 将记录按时间排序，用最新与最早的 usage_bytes 之差除以时间跨度，得到 growth_rate（字节/小时） + · 不足 2 条记录时视为"增长趋势未知" + +▸ usage_ratio >= 100%（已超出配额）： + → freeze_new_jobs=true，allow_expand=false，立即输出 JSON。 + +▸ 90% <= usage_ratio < 100%（接近配额）： + + 第二步：调用 query_platform_capacity 确认平台剩余空间。 + · 平台空间不足（可用量 <= 当前配额 20%）： + → allow_expand=false，freeze_new_jobs=false + → reason 说明平台空间不足，建议管理员释放集群存储，输出 JSON。 + + 第三步（平台空间充足时）：判断是否需要作业层面分析。 + · 增长平缓（growth_rate < 配额 1%/小时）或趋势未知： + → 属于正常数据积累，allow_expand=true，expand_bytes=配额的 20%，freeze_new_jobs=false，输出 JSON。 + · 增长较快（growth_rate >= 配额 1%/小时）： + → 进入第四步，分析活跃作业以辅助决策。 + + 第四步（仅在增长较快时执行）： + → 调用 list_tenant_pods 获取活跃 Pod 列表 + → 对每个 gpu_requests > 0 的 Pod，调用 query_pod_gpu_history（duration_hours 传入该 Pod 的预估运行时长，默认 24） + → 根据 GPU 历史数据判断作业性质： + + · max_util_ever_percent > 50%（曾进行 GPU 密集计算）： + → 判定为【落盘作业】，当前低利用率是正常落盘行为 + → allow_expand=true，expand_bytes=配额的 50%（为落盘预留充足空间） + → reason 中注明："检测到作业 [pod名] 历史 GPU 峰值为 X%，当前处于落盘阶段，扩容保护训练成果" + + · max_util_ever_percent <= 50% 且 data_available=true（DCGM 有数据但 GPU 从未高负载）： + → 判定为【可疑作业】，在 reason 中说明情况，建议人工排查 + → 但仍 allow_expand=true，expand_bytes=配额的 20%（不确定时优先保护用户） + → reason 中注明："作业 [pod名] GPU 历史峰值仅 X%，未见明显 GPU 计算，存储增长原因待排查" + + · data_available=false（DCGM 无数据，无法获取 GPU 历史）： + → 无法区分，保守但偏向保护：allow_expand=true，expand_bytes=配额的 20% + → reason 中注明 GPU 历史数据不可用 + + → freeze_new_jobs=false，输出 JSON。 + +▸ usage_ratio < 90%： + → 无需冻结，无需扩容，立即输出 JSON。 + +【仅在上述流程中明确要求时才调用对应工具；inspect_pod_details / get_tenant_compute_quota / query_pod_realtime_metrics / diagnose_prometheus 不得主动调用】 + +完成分析后，仅输出如下格式的 JSON，不要附加任何其他文字： +{"reason": "<决策理由，必须包含 usage_ratio、growth_rate 及关键作业诊断证据>", "allow_expand": true/false, "expand_bytes": <字节数>, "freeze_new_jobs": true/false}` + maxLoops = 10 + ) + + messages := []dsMessage{ + {Role: "system", Content: systemPrompt}, + } + if skillText != "" { + klog.Infof("AskAgentForDecision[%s] loaded storage agent skill from %s", tenantID, skillSource) + messages = append(messages, dsMessage{ + Role: "system", + Content: "以下为存储治理领域技能补充，请在工具调用和最终决策时遵循：\n" + skillText, + }) + } + messages = append(messages, dsMessage{ + Role: "user", + Content: fmt.Sprintf("租户 %s 触发存储告警（使用率超过 90%%），请调用工具进行全面排查，然后给出是否需要临时扩容的决策。", tenantID), + }) + + for i := 0; i < maxLoops; i++ { + resp, err := callConfiguredLLM(ctx, *llmConfig, dsRequest{ + MaxTokens: 4096, + Tools: getTools(), + Messages: messages, + }) + if err != nil { + return nil, fmt.Errorf("调用配置化 LLM Provider 失败: %w", err) + } + + choice := resp.Choices[0] + messages = append(messages, choice.Message) + + // 无工具调用 → 模型给出了最终决策 + if choice.FinishReason == "stop" || len(choice.Message.ToolCalls) == 0 { + klog.Infof("AskAgentForDecision[%s] 完整分析结果:\n%s", tenantID, choice.Message.Content) + raw := extractJSON(choice.Message.Content) + var decision LLMDecisionResponse + if err := json.Unmarshal([]byte(raw), &decision); err != nil { + return nil, fmt.Errorf("解析决策 JSON 失败: %w\n原始响应: %s", err, choice.Message.Content) + } + return &decision, nil + } + + // 执行所有工具调用，收集结果 + for _, toolCall := range choice.Message.ToolCalls { + result, toolErr := dispatchTool(clientset, restConfig, tenantID, promClient, toolCall) + if toolErr != nil { + result = fmt.Sprintf(`{"error": %q}`, toolErr.Error()) + } + messages = append(messages, dsMessage{ + Role: "tool", + ToolCallID: toolCall.ID, + Content: result, + }) + } + } + + return nil, fmt.Errorf("超过最大对话轮数 (%d)，未能得出决策", maxLoops) +} + +// dispatchTool 根据工具名分发到对应 handler +func dispatchTool(clientset kubernetes.Interface, restConfig *rest.Config, tenantID string, promClient monitor.PrometheusInterface, toolCall dsToolCall) (string, error) { + switch toolCall.Function.Name { + case "query_platform_capacity": + return HandleQueryPlatformCapacity(clientset, restConfig) + + case "list_tenant_pods": + return HandleListTenantPods(clientset, tenantID) + + case "inspect_pod_details": + var args struct { + TenantID string `json:"tenant_id"` + PodName string `json:"pod_name"` + } + if err := json.Unmarshal([]byte(toolCall.Function.Arguments), &args); err != nil { + return "", fmt.Errorf("解析 inspect_pod_details 参数失败: %w", err) + } + return HandleInspectPodDetails(clientset, args.TenantID, args.PodName) + + case "get_tenant_compute_quota": + return HandleGetComputeQuota(clientset, tenantID) + + case "query_tenant_storage_trend": + return HandleQueryTenantStorageTrend(clientset, restConfig, tenantID) + + case "query_pod_realtime_metrics": + var args struct { + PodName string `json:"pod_name"` + } + if err := json.Unmarshal([]byte(toolCall.Function.Arguments), &args); err != nil { + return "", fmt.Errorf("解析 query_pod_realtime_metrics 参数失败: %w", err) + } + return HandleQueryPodRealtimeMetrics(args.PodName, clientset, promClient) + + case "query_pod_gpu_history": + var args struct { + PodName string `json:"pod_name"` + DurationHours float64 `json:"duration_hours"` + } + if err := json.Unmarshal([]byte(toolCall.Function.Arguments), &args); err != nil { + return "", fmt.Errorf("解析 query_pod_gpu_history 参数失败: %w", err) + } + return HandleQueryPodGPUHistory(args.PodName, args.DurationHours, promClient) + + case "diagnose_prometheus": + var args struct { + PodName string `json:"pod_name"` + } + // pod_name 是可选参数，忽略解析错误 + _ = json.Unmarshal([]byte(toolCall.Function.Arguments), &args) + result := DiagnosePrometheus(args.PodName, clientset, promClient) + data, err := json.Marshal(result) + if err != nil { + return "", fmt.Errorf("序列化诊断结果失败: %w", err) + } + return string(data), nil + + default: + return "", fmt.Errorf("未知工具: %s", toolCall.Function.Name) + } +} + +// ---- Prometheus 诊断 ---- + +// PrometheusDiagnosis 是 DiagnosePrometheus 的返回结构。 +type PrometheusDiagnosis struct { + PrometheusURL string `json:"prometheus_url"` + Reachable bool `json:"reachable"` + ConnectError string `json:"connect_error,omitempty"` + DCGMAvailable bool `json:"dcgm_available"` + DCGMSeriesCount int `json:"dcgm_series_count"` + TrackedPods []string `json:"tracked_pods_in_namespace"` + Namespace string `json:"namespace"` + PodMetrics *struct { + PodName string `json:"pod_name"` + CPUCores float64 `json:"cpu_cores"` + MemoryMB float64 `json:"memory_mb"` + GPUUtilPercent float64 `json:"gpu_util_percent"` + GPUDataFound bool `json:"gpu_data_found"` + } `json:"pod_metrics,omitempty"` +} + +// DiagnosePrometheus 验证 Prometheus 连通性、DCGM 指标可用性， +// 并列出 jobNamespace 下被 DCGM 追踪的 Pod。 +// podName 非空时额外查询该 Pod 的实时指标。 +func DiagnosePrometheus(podName string, _ kubernetes.Interface, promClient monitor.PrometheusInterface) *PrometheusDiagnosis { + jobNamespace := config.GetConfig().Namespaces.Job + + diag := &PrometheusDiagnosis{ + PrometheusURL: config.GetConfig().PrometheusAPI, + Namespace: jobNamespace, + TrackedPods: []string{}, + } + + if promClient == nil { + diag.ConnectError = "Prometheus 客户端未初始化" + return diag + } + + // 1. 基础连通性：vector(1) 强制返回向量，任何 Prometheus 均支持 + v, ok, err := promClient.QueryInstant("vector(1)") + if err != nil { + diag.ConnectError = fmt.Sprintf("Prometheus 查询失败: %v", err) + return diag + } + if !ok || v != 1 { + diag.ConnectError = fmt.Sprintf("Prometheus 返回异常值: ok=%v val=%v", ok, v) + return diag + } + diag.Reachable = true + + // 2. 检查 DCGM 是否存在任意时间序列 + if cnt, ok, err := promClient.QueryInstant("count(DCGM_FI_DEV_GPU_UTIL)"); err == nil && ok { + diag.DCGMAvailable = true + diag.DCGMSeriesCount = int(cnt) + } + + // 3. 列出 jobNamespace 下 DCGM 追踪的所有 Pod 名称 + pods := promClient.QueryInstantLabels( + fmt.Sprintf(`count by (pod) (DCGM_FI_DEV_GPU_UTIL{namespace=%q})`, jobNamespace), + "pod") + if pods != nil { + diag.TrackedPods = pods + } + + // 4. 如果指定了 Pod，额外查询其实时指标 + if podName != "" { + pm := &struct { + PodName string `json:"pod_name"` + CPUCores float64 `json:"cpu_cores"` + MemoryMB float64 `json:"memory_mb"` + GPUUtilPercent float64 `json:"gpu_util_percent"` + GPUDataFound bool `json:"gpu_data_found"` + }{PodName: podName} + + if v, ok, _ := promClient.QueryInstant( + fmt.Sprintf(`sum(rate(container_cpu_usage_seconds_total{pod=%q,container!=""}[5m]))`, podName), + ); ok { + pm.CPUCores = v + } + if v, ok, _ := promClient.QueryInstant( + fmt.Sprintf(`sum(container_memory_usage_bytes{pod=%q,container!=""})`, podName), + ); ok { + pm.MemoryMB = v / 1024 / 1024 + } + // GPU：先带 namespace，再退化 + for _, q := range []string{ + fmt.Sprintf(`avg(DCGM_FI_DEV_GPU_UTIL{namespace=%q,pod=%q})`, jobNamespace, podName), + fmt.Sprintf(`avg(DCGM_FI_DEV_GPU_UTIL{pod=%q})`, podName), + } { + if v, ok, _ := promClient.QueryInstant(q); ok { + pm.GPUUtilPercent = v + pm.GPUDataFound = true + break + } + } + diag.PodMetrics = pm + } + + return diag +} + +// HandleQueryPodRealtimeMetrics 通过 Prometheus 查询 Pod 的真实资源利用率， +// 用于识别高申请低利用的僵尸作业。 +func HandleQueryPodRealtimeMetrics(podName string, _ kubernetes.Interface, promClient monitor.PrometheusInterface) (string, error) { + if promClient == nil { + return "", fmt.Errorf("Prometheus 客户端未初始化") + } + + // Job namespace — DCGM 标签与 Kubernetes namespace 绑定，必须传入才能匹配到指标 + jobNamespace := config.GetConfig().Namespaces.Job + + type metricsResult struct { + PodName string `json:"pod_name"` + Namespace string `json:"namespace"` + CPUCores float64 `json:"cpu_cores"` + MemoryMB float64 `json:"memory_mb"` + GPUUtilPercent float64 `json:"gpu_util_percent"` + GPUMemoryMB float64 `json:"gpu_memory_mb"` + GPUDataAvailable bool `json:"gpu_data_available"` + Note string `json:"note"` + } + + res := metricsResult{PodName: podName, Namespace: jobNamespace} + + // CPU 使用量（核数，5 分钟均值） + if v, ok, err := promClient.QueryInstant( + fmt.Sprintf(`sum(rate(container_cpu_usage_seconds_total{pod=%q,container!=""}[5m]))`, podName), + ); err != nil { + klog.Warningf("query_pod_realtime_metrics: CPU 查询失败 pod=%s: %v", podName, err) + } else if ok { + res.CPUCores = v + } + + // 内存使用量（MB） + if v, ok, err := promClient.QueryInstant( + fmt.Sprintf(`sum(container_memory_usage_bytes{pod=%q,container!=""})`, podName), + ); err != nil { + klog.Warningf("query_pod_realtime_metrics: 内存查询失败 pod=%s: %v", podName, err) + } else if ok { + res.MemoryMB = v / 1024 / 1024 + } + + // GPU 利用率（%，来自 DCGM_FI_DEV_GPU_UTIL） + // 优先用 namespace+pod 双标签（与现有 monitor 包一致），若无结果则退化为仅 pod 标签 + gpuUtilQueries := []string{ + fmt.Sprintf(`avg(DCGM_FI_DEV_GPU_UTIL{namespace=%q,pod=%q})`, jobNamespace, podName), + fmt.Sprintf(`avg(DCGM_FI_DEV_GPU_UTIL{pod=%q})`, podName), + } + for _, q := range gpuUtilQueries { + v, ok, err := promClient.QueryInstant(q) + if err != nil { + klog.Warningf("query_pod_realtime_metrics: GPU 利用率查询失败 pod=%s query=%s: %v", podName, q, err) + continue + } + if ok { + res.GPUUtilPercent = v + res.GPUDataAvailable = true + break + } + } + + // GPU 显存使用量（MB，来自 DCGM_FI_DEV_FB_USED） + gpuMemQueries := []string{ + fmt.Sprintf(`avg(DCGM_FI_DEV_FB_USED{namespace=%q,pod=%q})`, jobNamespace, podName), + fmt.Sprintf(`avg(DCGM_FI_DEV_FB_USED{pod=%q})`, podName), + } + for _, q := range gpuMemQueries { + v, ok, err := promClient.QueryInstant(q) + if err != nil { + klog.Warningf("query_pod_realtime_metrics: GPU 显存查询失败 pod=%s query=%s: %v", podName, q, err) + continue + } + if ok { + res.GPUMemoryMB = v + break + } + } + + res.Note = "cpu_cores 为过去 5 分钟均值；memory_mb 为当前值；gpu_util_percent/gpu_memory_mb 来自 DCGM（gpu_data_available=false 表示该 Pod 无 GPU 或 DCGM 未采集到数据）" + + data, err := json.Marshal(res) + if err != nil { + return "", fmt.Errorf("序列化利用率响应失败: %w", err) + } + return string(data), nil +} + +// HandleQueryPodGPUHistory 查询 Pod 在指定时间窗口内的 GPU 历史利用率， +// 用于区分"正在落盘的有价值作业"（历史上曾高强度用过 GPU）与"僵尸作业"（从未有效使用 GPU）。 +func HandleQueryPodGPUHistory(podName string, durationHours float64, promClient monitor.PrometheusInterface) (string, error) { + if promClient == nil { + return "", fmt.Errorf("Prometheus 客户端未初始化") + } + if durationHours <= 0 { + durationHours = 24 + } + + jobNamespace := config.GetConfig().Namespaces.Job + dur := fmt.Sprintf("%.0fh", durationHours) + // 不足 1 小时时用分钟表示，避免 Prometheus 解析错误 + if durationHours < 1 { + dur = fmt.Sprintf("%.0fm", durationHours*60) + } + + type gpuHistoryResult struct { + PodName string `json:"pod_name"` + DurationHours float64 `json:"duration_hours"` + AvgUtil float64 `json:"avg_util_percent"` + MaxUtil float64 `json:"max_util_ever_percent"` + DataAvailable bool `json:"data_available"` + Note string `json:"note"` + } + + res := gpuHistoryResult{ + PodName: podName, + DurationHours: durationHours, + Note: fmt.Sprintf("查询过去 %s 内的 GPU 利用率历史。max_util_ever_percent > 50 表明作业曾进行 GPU 密集型计算（如模型训练），当前低利用率很可能是落盘/IO 阶段", dur), + } + + // 平均利用率：namespace+pod 优先，退化为仅 pod + avgQueries := []string{ + fmt.Sprintf(`avg_over_time(DCGM_FI_DEV_GPU_UTIL{namespace=%q,pod=%q}[%s])`, jobNamespace, podName, dur), + fmt.Sprintf(`avg_over_time(DCGM_FI_DEV_GPU_UTIL{pod=%q}[%s])`, podName, dur), + } + for _, q := range avgQueries { + if v, ok, err := promClient.QueryInstant(q); err == nil && ok { + res.AvgUtil = v + res.DataAvailable = true + break + } + } + + // 峰值利用率：用 max_over_time 捕获历史最高点 + maxQueries := []string{ + fmt.Sprintf(`max_over_time(DCGM_FI_DEV_GPU_UTIL{namespace=%q,pod=%q}[%s])`, jobNamespace, podName, dur), + fmt.Sprintf(`max_over_time(DCGM_FI_DEV_GPU_UTIL{pod=%q}[%s])`, podName, dur), + } + for _, q := range maxQueries { + if v, ok, err := promClient.QueryInstant(q); err == nil && ok { + res.MaxUtil = v + res.DataAvailable = true + break + } + } + + data, err := json.Marshal(res) + if err != nil { + return "", fmt.Errorf("序列化 GPU 历史响应失败: %w", err) + } + return string(data), nil +} + +// ---- Handler 实现 ---- + +func HandleQueryPlatformCapacity(clientset kubernetes.Interface, restConfig *rest.Config) (string, error) { + totalCapacity, usedCapacity, err := ceph.GetCraterStorageCapacity(clientset, restConfig, "rook-ceph") + if err != nil { + return "", fmt.Errorf("获取平台容量失败: %w", err) + } + availableCapacity := ceph.AvailableBytes(totalCapacity, usedCapacity) + + data, err := json.Marshal(map[string]any{ + "total_capacity_bytes": totalCapacity, + "used_capacity_bytes": usedCapacity, + "available_capacity_bytes": availableCapacity, + "total_capacity_formatted": formatBytes(totalCapacity), + "used_capacity_formatted": formatBytes(usedCapacity), + "note": "所有容量单位均为字节(bytes)，formatted 字段为人类可读格式", + }) + if err != nil { + return "", fmt.Errorf("序列化平台容量响应失败: %w", err) + } + return string(data), nil +} + +func HandleListTenantPods(clientset kubernetes.Interface, tenantID string) (string, error) { + jobNamespace := config.GetConfig().Namespaces.Job + pods, err := clientset.CoreV1().Pods(jobNamespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", labelKeyTaskUser, tenantID), + }) + if err != nil { + return "", fmt.Errorf("列出 Pod 失败: %w", err) + } + + podResponses := make([]TenantPodResponse, 0) + for _, pod := range pods.Items { + phase := string(pod.Status.Phase) + if phase != "Running" && phase != "Pending" { + continue + } + gpuRequests := 0 + for _, c := range pod.Spec.Containers { + for k, v := range c.Resources.Requests { + if strings.Contains(string(k), "nvidia.com/") { + gpuRequests += int(v.Value()) + } + } + } + podResponses = append(podResponses, TenantPodResponse{ + PodName: pod.Name, + Phase: phase, + GPURrequests: gpuRequests, + }) + } + + data, err := json.Marshal(TenantPodsResponse{TenantID: tenantID, Pods: podResponses}) + if err != nil { + return "", fmt.Errorf("序列化租户 Pod 列表响应失败: %w", err) + } + return string(data), nil +} + +func HandleInspectPodDetails(clientset kubernetes.Interface, _ string, podName string) (string, error) { + jobNamespace := config.GetConfig().Namespaces.Job + pod, err := clientset.CoreV1().Pods(jobNamespace).Get(context.TODO(), podName, metav1.GetOptions{}) + if err != nil { + return "", fmt.Errorf("获取 Pod 失败: %w", err) + } + + var startTime time.Time + if pod.Status.StartTime != nil { + startTime = pod.Status.StartTime.Time + } + runningMinutes := 0 + if !startTime.IsZero() { + runningMinutes = int(time.Since(startTime).Minutes()) + } + + images := make([]string, 0, len(pod.Spec.Containers)) + for _, c := range pod.Spec.Containers { + images = append(images, c.Image) + } + + restartCount := 0 + for _, cs := range pod.Status.ContainerStatuses { + restartCount += int(cs.RestartCount) + } + + gpuCount := 0 + cpuMillis := int64(0) + memBytes := int64(0) + for _, c := range pod.Spec.Containers { + for k, v := range c.Resources.Requests { + if strings.Contains(string(k), "nvidia.com/") { + gpuCount += int(v.Value()) + } + } + if cpu, ok := c.Resources.Requests[corev1.ResourceCPU]; ok { + cpuMillis += cpu.MilliValue() + } + if mem, ok := c.Resources.Requests[corev1.ResourceMemory]; ok { + memBytes += mem.Value() + } + } + + gpuModel := "unknown" + if pod.Spec.NodeName != "" { + if node, err := clientset.CoreV1().Nodes().Get(context.TODO(), pod.Spec.NodeName, metav1.GetOptions{}); err == nil { + for k, v := range node.Labels { + if strings.Contains(k, "nvidia.com/gpu.product") || k == "gpu-model" { + gpuModel = v + break + } + } + } + } + + data, err := json.Marshal(PodDetailsResponse{ + PodName: podName, + StartTime: startTime, + RunningTime: runningMinutes, + ContainerImages: images, + RestartCount: restartCount, + GPUModel: gpuModel, + GPUCount: gpuCount, + CPUCount: int(cpuMillis / 1000), + MemorySize: formatBytes(memBytes), + }) + if err != nil { + return "", fmt.Errorf("序列化 Pod 详情响应失败: %w", err) + } + return string(data), nil +} + +func HandleGetComputeQuota(clientset kubernetes.Interface, tenantID string) (string, error) { + jobNamespace := config.GetConfig().Namespaces.Job + pods, err := clientset.CoreV1().Pods(jobNamespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", labelKeyTaskUser, tenantID), + }) + if err != nil { + return "", fmt.Errorf("列出 Pod 失败: %w", err) + } + + gpuRequest, gpuLimit := 0, 0 + cpuReqMillis, cpuLimMillis := int64(0), int64(0) + memReqBytes, memLimBytes := int64(0), int64(0) + + for _, pod := range pods.Items { + for _, c := range pod.Spec.Containers { + for k, v := range c.Resources.Requests { + if strings.Contains(string(k), "nvidia.com/") { + gpuRequest += int(v.Value()) + } + } + for k, v := range c.Resources.Limits { + if strings.Contains(string(k), "nvidia.com/") { + gpuLimit += int(v.Value()) + } + } + if v, ok := c.Resources.Requests[corev1.ResourceCPU]; ok { + cpuReqMillis += v.MilliValue() + } + if v, ok := c.Resources.Limits[corev1.ResourceCPU]; ok { + cpuLimMillis += v.MilliValue() + } + if v, ok := c.Resources.Requests[corev1.ResourceMemory]; ok { + memReqBytes += v.Value() + } + if v, ok := c.Resources.Limits[corev1.ResourceMemory]; ok { + memLimBytes += v.Value() + } + } + } + + data, err := json.Marshal(ComputeQuotaResponse{ + TenantID: tenantID, + GPULimit: gpuLimit, + GPURequest: gpuRequest, + CPULimit: int(cpuLimMillis / 1000), + CPURequest: int(cpuReqMillis / 1000), + MemoryLimit: formatBytes(memLimBytes), + MemoryRequest: formatBytes(memReqBytes), + }) + if err != nil { + return "", fmt.Errorf("序列化计算配额响应失败: %w", err) + } + return string(data), nil +} + +func HandleQueryTenantStorageTrend(clientset kubernetes.Interface, restConfig *rest.Config, tenantID string) (string, error) { + db := query.GetDB() + + // 先查基础用户信息和 space_quota + var userRow struct { + model.User + SpaceQuota int64 `gorm:"column:space_quota"` + } + if err := db.Model(&model.User{}). + Select("users.*, users.space_quota"). + Where("name = ?", tenantID). + First(&userRow).Error; err != nil { + return "", fmt.Errorf("用户 %s 不存在: %w", tenantID, err) + } + user := userRow.User + + // 尝试获取 original_space_quota（临时扩容时才有值），用作理论配额 + // 若迁移未执行列不存在，保持使用 space_quota + spaceQuota := userRow.SpaceQuota + var origRow struct { + OriginalSpaceQuota *int64 `gorm:"column:original_space_quota"` + } + if err := db.Raw("SELECT original_space_quota FROM users WHERE id = ?", user.ID).Scan(&origRow).Error; err == nil && origRow.OriginalSpaceQuota != nil { + spaceQuota = *origRow.OriginalSpaceQuota + } + if user.Space == "" { + return "", fmt.Errorf("用户 %s 的空间路径为空", tenantID) + } + + cfg := config.GetConfig() + prefixConfig := ceph.StoragePrefixConfig{ + User: cfg.Storage.Prefix.User, + Account: cfg.Storage.Prefix.Account, + Public: cfg.Storage.Prefix.Public, + } + currentUsage, err := ceph.GetCephDirectorySize(clientset, restConfig, "rook-ceph", "/user/"+user.Space, prefixConfig) + if err != nil { + currentUsage = ceph.UnknownSizeBytes + } + + var historyRecords []model.TenantUsageHistory + db.Where("tenant_id = ?", user.ID).Order("recorded_at DESC").Limit(10).Find(&historyRecords) + + type historyItem struct { + Timestamp time.Time `json:"timestamp"` + UsageBytes int64 `json:"usage_bytes"` + UsageBytesFormatted string `json:"usage_bytes_formatted"` + } + history := make([]historyItem, 0, len(historyRecords)) + for _, h := range historyRecords { + history = append(history, historyItem{ + Timestamp: h.RecordedAt, + UsageBytes: h.UsageBytes, + UsageBytesFormatted: formatBytes(h.UsageBytes), + }) + } + + usageRatio := "" + if spaceQuota > 0 && currentUsage >= 0 { + usageRatio = fmt.Sprintf("%.1f%%", float64(currentUsage)/float64(spaceQuota)*100) + } else if spaceQuota == -1 { + usageRatio = "unlimited" + } + + data, err := json.Marshal(map[string]any{ + "tenant_id": tenantID, + "current_usage_bytes": currentUsage, + "current_usage_formatted": formatBytes(currentUsage), + "quota_bytes": spaceQuota, + "quota_formatted": formatBytes(spaceQuota), + "usage_ratio": usageRatio, + "history": history, + "note": "所有大小单位均为字节(bytes)，formatted 字段为人类可读格式。quota_bytes=-1 表示无限制。usage_ratio 为当前使用量占配额的百分比。", + }) + if err != nil { + return "", fmt.Errorf("序列化租户存储趋势响应失败: %w", err) + } + return string(data), nil +} + +// ---- 工具函数 ---- + +// extractJSON 从可能包含分析文字的响应中提取最后一个 JSON 对象 +func extractJSON(s string) string { + start := strings.LastIndex(s, "{") + end := strings.LastIndex(s, "}") + if start != -1 && end != -1 && end > start { + return strings.TrimSpace(s[start : end+1]) + } + return strings.TrimSpace(s) +} + +func parseGetfattrValue(output, attr string) int64 { + prefix := attr + "=" + for _, line := range strings.Split(output, "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, prefix) { + val := strings.Trim(strings.TrimPrefix(line, prefix), "\"") + if n, err := strconv.ParseInt(val, 10, 64); err == nil { + return n + } + } + } + return 0 +} + +func formatBytes(bytes int64) string { + const unit = 1024 + if bytes <= 0 { + return "0 B" + } + if bytes < unit { + return fmt.Sprintf("%d B", bytes) + } + div, exp := int64(unit), 0 + for n := bytes / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp]) +} diff --git a/backend/pkg/llm/provider.go b/backend/pkg/llm/provider.go new file mode 100644 index 000000000..9d0f0421e --- /dev/null +++ b/backend/pkg/llm/provider.go @@ -0,0 +1,207 @@ +//nolint:dupl // Chat and completion requests intentionally share nearly identical HTTP transport flow. +package llm + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "strings" + "time" + + "k8s.io/klog/v2" + + "github.com/raids-lab/crater/dao/model" + "github.com/raids-lab/crater/dao/query" + "github.com/raids-lab/crater/pkg/crypto" +) + +type ProviderConfig struct { + BaseURL string + APIKey string + ModelName string +} + +func (c ProviderConfig) CompletionURL() string { + baseURL := strings.TrimSuffix(strings.TrimSpace(c.BaseURL), "/") + if baseURL == "" { + return "" + } + if strings.HasSuffix(baseURL, "/completions") { + return baseURL + } + return baseURL + "/completions" +} + +func (c ProviderConfig) ChatCompletionURL() string { + baseURL := strings.TrimSuffix(strings.TrimSpace(c.BaseURL), "/") + if baseURL == "" { + return "" + } + if strings.HasSuffix(baseURL, "/chat/completions") { + return baseURL + } + return baseURL + "/chat/completions" +} + +func loadRuntimeLLMConfig(ctx context.Context) (*ProviderConfig, error) { + cfg := &ProviderConfig{ + BaseURL: strings.TrimSpace(os.Getenv("LLM_API_BASE_URL")), + APIKey: strings.TrimSpace(os.Getenv("LLM_API_KEY")), + ModelName: strings.TrimSpace(os.Getenv("LLM_MODEL_NAME")), + } + + var rows []model.SystemConfig + err := query.GetDB().WithContext(ctx). + Where("key IN ?", []string{ + model.ConfigKeyLLMBaseURL, + model.ConfigKeyLLMAPIKey, + model.ConfigKeyLLMModelName, + }). + Find(&rows).Error + if err != nil { + if cfg.BaseURL == "" || cfg.ModelName == "" { + return nil, fmt.Errorf("failed to load llm config from database: %w", err) + } + return cfg, nil + } + + configMap := make(map[string]string, len(rows)) + for _, row := range rows { + configMap[row.Key] = strings.TrimSpace(row.Value) + } + + if baseURL := configMap[model.ConfigKeyLLMBaseURL]; baseURL != "" { + cfg.BaseURL = baseURL + } + if modelName := configMap[model.ConfigKeyLLMModelName]; modelName != "" { + cfg.ModelName = modelName + } + if encryptedKey := configMap[model.ConfigKeyLLMAPIKey]; encryptedKey != "" { + plainKey, decryptErr := crypto.Decrypt(encryptedKey) + if decryptErr != nil { + klog.Warningf("loadRuntimeLLMConfig: failed to decrypt llm api key, using raw value: %v", decryptErr) + cfg.APIKey = encryptedKey + } else { + cfg.APIKey = plainKey + } + } + + if cfg.BaseURL == "" { + return nil, fmt.Errorf("llm base url is not configured") + } + if cfg.ModelName == "" { + return nil, fmt.Errorf("llm model name is not configured") + } + + return cfg, nil +} + +type completionRequest struct { + Model string `json:"model"` + Prompt string `json:"prompt"` + MaxTokens int `json:"max_tokens,omitempty"` + Temperature *float64 `json:"temperature,omitempty"` +} + +type completionResponse struct { + Choices []struct { + Text string `json:"text"` + FinishReason string `json:"finish_reason"` + } `json:"choices"` + Error *struct { + Message string `json:"message"` + } `json:"error,omitempty"` +} + +func callConfiguredLLM(ctx context.Context, cfg ProviderConfig, req dsRequest) (*dsResponse, error) { + req.Model = cfg.ModelName + + body, err := json.Marshal(req) + if err != nil { + return nil, fmt.Errorf("序列化请求失败: %w", err) + } + + chatCompletionURL := cfg.ChatCompletionURL() + if chatCompletionURL == "" { + return nil, fmt.Errorf("llm chat completion url is empty") + } + + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, chatCompletionURL, bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("创建 HTTP 请求失败: %w", err) + } + if cfg.APIKey != "" { + httpReq.Header.Set("Authorization", "Bearer "+cfg.APIKey) + } + httpReq.Header.Set("Content-Type", "application/json") + + httpClient := &http.Client{Timeout: 120 * time.Second} + httpResp, err := httpClient.Do(httpReq) + if err != nil { + return nil, fmt.Errorf("HTTP 请求失败: %w", err) + } + defer httpResp.Body.Close() + + var apiResp dsResponse + if err := json.NewDecoder(httpResp.Body).Decode(&apiResp); err != nil { + return nil, fmt.Errorf("解析 API 响应失败: %w", err) + } + + if httpResp.StatusCode != http.StatusOK { + errMsg := fmt.Sprintf("HTTP %d", httpResp.StatusCode) + if apiResp.Error != nil { + errMsg = apiResp.Error.Message + } + return nil, fmt.Errorf("API 错误: %s", errMsg) + } + + return &apiResp, nil +} + +func callConfiguredCompletion(ctx context.Context, cfg ProviderConfig, req completionRequest) (*completionResponse, error) { + req.Model = cfg.ModelName + + body, err := json.Marshal(req) + if err != nil { + return nil, fmt.Errorf("failed to marshal completion request: %w", err) + } + + completionURL := cfg.CompletionURL() + if completionURL == "" { + return nil, fmt.Errorf("llm completion url is empty") + } + + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, completionURL, bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("failed to create completion request: %w", err) + } + if cfg.APIKey != "" { + httpReq.Header.Set("Authorization", "Bearer "+cfg.APIKey) + } + httpReq.Header.Set("Content-Type", "application/json") + + httpClient := &http.Client{Timeout: 120 * time.Second} + httpResp, err := httpClient.Do(httpReq) + if err != nil { + return nil, fmt.Errorf("completion request failed: %w", err) + } + defer httpResp.Body.Close() + + var apiResp completionResponse + if err := json.NewDecoder(httpResp.Body).Decode(&apiResp); err != nil { + return nil, fmt.Errorf("failed to decode completion response: %w", err) + } + + if httpResp.StatusCode != http.StatusOK { + errMsg := fmt.Sprintf("HTTP %d", httpResp.StatusCode) + if apiResp.Error != nil { + errMsg = apiResp.Error.Message + } + return nil, fmt.Errorf("completion API error: %s", errMsg) + } + + return &apiResp, nil +} diff --git a/backend/pkg/llm/skill.go b/backend/pkg/llm/skill.go new file mode 100644 index 000000000..e0ffe6665 --- /dev/null +++ b/backend/pkg/llm/skill.go @@ -0,0 +1,46 @@ +package llm + +import ( + "embed" + "fmt" + "os" + "strconv" + "strings" +) + +const ( + StorageAgentSkillEnabledEnv = "CRATER_STORAGE_AGENT_SKILL_ENABLED" + StorageAgentSkillPathEnv = "CRATER_STORAGE_AGENT_SKILL_PATH" + defaultStorageAgentSkill = "skills/storage-governance-agent/SKILL.md" +) + +//go:embed skills/storage-governance-agent/SKILL.md +var embeddedSkills embed.FS + +func loadStorageAgentSkill() (content, source string, err error) { + enabled := true + if raw := strings.TrimSpace(os.Getenv(StorageAgentSkillEnabledEnv)); raw != "" { + parsed, err := strconv.ParseBool(raw) + if err != nil { + return "", "", fmt.Errorf("invalid %s value %q: %w", StorageAgentSkillEnabledEnv, raw, err) + } + enabled = parsed + } + if !enabled { + return "", "", nil + } + + if customPath := strings.TrimSpace(os.Getenv(StorageAgentSkillPathEnv)); customPath != "" { + data, err := os.ReadFile(customPath) + if err != nil { + return "", "", fmt.Errorf("failed to read storage agent skill from %s: %w", customPath, err) + } + return strings.TrimSpace(string(data)), customPath, nil + } + + data, err := embeddedSkills.ReadFile(defaultStorageAgentSkill) + if err != nil { + return "", "", fmt.Errorf("failed to read embedded storage agent skill: %w", err) + } + return strings.TrimSpace(string(data)), defaultStorageAgentSkill, nil +} diff --git a/backend/pkg/llm/skills/README.md b/backend/pkg/llm/skills/README.md new file mode 100644 index 000000000..d3ebf0722 --- /dev/null +++ b/backend/pkg/llm/skills/README.md @@ -0,0 +1,19 @@ +# LLM Skills + +This directory stores domain-specific prompt skills for the Crater backend. + +## Storage Governance Agent Skill + +- Default skill file: + [storage-governance-agent/SKILL.md](D:/crater/backend/pkg/llm/skills/storage-governance-agent/SKILL.md) +- Loaded only by DeepSeek/OpenAI-compatible agent mode: + [llm_tools.go](D:/crater/backend/pkg/llm/llm_tools.go) + +## Runtime Controls + +- `CRATER_STORAGE_AGENT_SKILL_ENABLED` + - Optional, default `true` + - Set to `false` to disable loading the skill +- `CRATER_STORAGE_AGENT_SKILL_PATH` + - Optional + - When set, the backend loads the skill text from this external file path instead of the embedded default skill diff --git a/backend/pkg/llm/skills/storage-governance-agent/SKILL.md b/backend/pkg/llm/skills/storage-governance-agent/SKILL.md new file mode 100644 index 000000000..8f42ec4c1 --- /dev/null +++ b/backend/pkg/llm/skills/storage-governance-agent/SKILL.md @@ -0,0 +1,22 @@ +# Storage Governance Agent Skill + +## Role +你是存储治理领域技能层，负责补充对存储扩容、冻结、Prometheus 指标和 GPU 历史行为的判断经验。 + +## Core Heuristics +1. 当 `usage_ratio >= 1.0` 时，如果无法安全扩容，应优先考虑 `freeze_new_jobs=true`。 +2. 当 `usage_ratio < 0.9` 时，通常不应扩容，也不应冻结。 +3. 当 `gpu_data_available=true` 且 `max_gpu_history_percent > 50` 时，当前低 GPU 利用率可能属于正常落盘/IO 阶段，不应轻易判定为异常。 +4. 当 `gpu_data_available=false` 时，不要过度自信地下结论，必须在 `reason` 中明确指出监控缺失或证据不足。 +5. 当平台剩余空间充足、且增长速率低于配额阈值时，优先采用保守扩容而不是冻结。 + +## Tool Preference +1. 必须先查看存储趋势与使用率。 +2. 当存在活跃 GPU Pod 时，优先分析 GPU 历史指标来区分“正常落盘”与“可疑作业”。 +3. 只有在 GPU 指标异常缺失或结果矛盾时，才进一步诊断 Prometheus。 +4. 不要做冗余工具调用；每次调用都应服务于最终扩容/冻结判断。 + +## Output Preference +1. 输出必须是纯 JSON。 +2. `reason` 必须引用关键证据字段，例如 `usage_ratio`、`growth_rate`、平台剩余容量、GPU 历史峰值或监控缺失状态。 +3. `reason` 要尽量说明当前属于哪一类场景：正常积累、落盘阶段、可疑增长、超配额冻结、平台容量受限。 diff --git a/backend/pkg/monitor/helper.go b/backend/pkg/monitor/helper.go index 4d517a732..42688719d 100644 --- a/backend/pkg/monitor/helper.go +++ b/backend/pkg/monitor/helper.go @@ -194,6 +194,42 @@ func (p *PrometheusClient) checkGPUUsed(expression string) (int, error) { return 0, fmt.Errorf("expected vector type result but got %s", result.Type()) } +// QueryInstant executes an instant PromQL query and returns the first scalar result. +func (p *PrometheusClient) QueryInstant(query string) (value float64, found bool, err error) { + ctx, cancel := context.WithTimeout(context.Background(), queryTimeout) + defer cancel() + result, _, err := p.v1api.Query(ctx, query, time.Now()) + if err != nil { + return 0, false, err + } + switch result.Type() { + case model.ValVector: + vector := result.(model.Vector) + if len(vector) > 0 { + return float64(vector[0].Value), true, nil + } + case model.ValScalar: + scalar := result.(*model.Scalar) + return float64(scalar.Value), true, nil + } + return 0, false, nil +} + +// QueryInstantLabels executes a PromQL query and returns all values of the given label key. +func (p *PrometheusClient) QueryInstantLabels(query, labelKey string) []string { + vector, err := p.queryVector(query) + if err != nil { + return nil + } + var labels []string + for _, sample := range vector { + if v, ok := sample.Metric[model.LabelName(labelKey)]; ok { + labels = append(labels, string(v)) + } + } + return labels +} + // checkIfGPURequested 检查 Pod 是否申请了 GPU func (p *PrometheusClient) checkIfGPURequested(namespacedName types.NamespacedName) bool { // 这里实现检查逻辑，例如通过查询 Pod 的资源请求或 Prometheus 中的相关指标 diff --git a/backend/pkg/monitor/interface.go b/backend/pkg/monitor/interface.go index f0534dd5f..9929d11ec 100644 --- a/backend/pkg/monitor/interface.go +++ b/backend/pkg/monitor/interface.go @@ -70,4 +70,14 @@ type PrometheusInterface interface { // GetLeastUsedGPUJobList returns the least used GPU job list GetLeastUsedGPUJobList(podName, _time, util string) int + + ///////////// Generic PromQL ////////////// + + // QueryInstant executes an instant PromQL query and returns the first scalar result. + // found=false if the result set is empty (not an error). + QueryInstant(query string) (float64, bool, error) + + // QueryInstantLabels executes a PromQL query and returns all values of the specified + // metric label key across every result sample. + QueryInstantLabels(query, labelKey string) []string } diff --git a/backend/pkg/patrol/patrol.go b/backend/pkg/patrol/patrol.go index c1ffc8092..09d4193f1 100644 --- a/backend/pkg/patrol/patrol.go +++ b/backend/pkg/patrol/patrol.go @@ -1,15 +1,27 @@ +//nolint:gocritic,gocyclo,lll,mnd,revive // Patrol orchestration intentionally centralizes policy execution and thresholds. package patrol import ( "context" "encoding/json" "fmt" + "os" + "strconv" + "sync" + "time" "gorm.io/datatypes" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/raids-lab/crater/dao/model" + "github.com/raids-lab/crater/dao/query" + "github.com/raids-lab/crater/pkg/ceph" + "github.com/raids-lab/crater/pkg/config" "github.com/raids-lab/crater/pkg/monitor" + "github.com/raids-lab/crater/pkg/storageindex" "github.com/raids-lab/crater/pkg/util" ) @@ -18,8 +30,23 @@ const ( TRIGGER_GPU_ANALYSIS_JOB = "trigger-gpu-analysis-job" // Billing 基础循环 TRIGGER_BILLING_BASE_LOOP_JOB = "biling-base-loop" - // 未来可以扩展其他巡检任务，例如： - // CHECK_NODE_HEALTH = "check-node-health" + // 更新用户空间大小任务 + UPDATE_USER_SPACE_SIZE = "update-user-space-size" + // 存储告警 AI 分析任务 + ANALYZE_STORAGE_ALERTS = "analyze-storage-alerts" + AUTO_SHRINK_STORAGE_EXPANSIONS = "auto-shrink-storage-expansions" + REFRESH_PUBLIC_STORAGE_INDEX = "refresh-public-storage-index-baseline" + REFRESH_USER_STORAGE_INDEX = "refresh-user-storage-index-daily" + + // AI 分析最大并发数 + defaultMaxConcurrentStorageAnalysis = 3 + autoShrinkToBufferThreshold = 0.90 + autoShrinkRecoverThreshold = 0.80 + autoShrinkObservationWindow = time.Hour + autoShrinkStageExpanded = "expanded" + autoShrinkStageBuffer = "buffer_reduction" + + storageAnalysisConcurrencyEnv = "CRATER_STORAGE_ANALYSIS_MAX_CONCURRENCY" ) type GpuAnalysisServiceInterface interface { @@ -30,18 +57,61 @@ type BillingServiceInterface interface { RunBaseLoopOnce(ctx context.Context) (any, error) } +// AgentDecision 是 LLM 存储扩容决策的结果，定义在 patrol 包以避免循环依赖。 +type AgentDecision struct { + AllowExpand bool + ExpandBytes int64 + FreezeNewJobs bool + Reason string + DecisionJobID string +} + +// StorageAgentFunc 是调用 LLM 进行存储分析的函数签名。 +type StorageAgentFunc func(tenantID string) (*AgentDecision, error) + +type StorageAgentStartFunc func(ctx context.Context, tenantID string) (string, error) + +type StorageAgentAwaitFunc func(ctx context.Context, tenantID string, jobID string) (*AgentDecision, error) + // Clients 包含巡检任务所需的客户端 type Clients struct { Client client.Client KubeClient kubernetes.Interface + KubeConfig *rest.Config PromClient monitor.PrometheusInterface GpuAnalysisService GpuAnalysisServiceInterface BillingService BillingServiceInterface + RecordDecision func(ctx context.Context, jobID string, action string, runErr error) + StorageAgent StorageAgentFunc // 注入的 LLM 分析函数，nil 时跳过 AI 分析 + StorageAgentStart StorageAgentStartFunc + StorageAgentAwait StorageAgentAwaitFunc + StorageIndex *storageindex.Service +} + +func storageAnalysisConcurrency() int { + raw := os.Getenv(storageAnalysisConcurrencyEnv) + if raw == "" { + return defaultMaxConcurrentStorageAnalysis + } + + value, err := strconv.Atoi(raw) + if err != nil || value <= 0 { + klog.Warningf( + "storageAnalysisConcurrency: invalid %s=%q, fallback to %d", + storageAnalysisConcurrencyEnv, + raw, + defaultMaxConcurrentStorageAnalysis, + ) + return defaultMaxConcurrentStorageAnalysis + } + + return value } func NewPatrolClients( cli client.Client, kubeClient kubernetes.Interface, + kubeConfig *rest.Config, promClient monitor.PrometheusInterface, gpuAnalysisService GpuAnalysisServiceInterface, billingService BillingServiceInterface, @@ -49,13 +119,513 @@ func NewPatrolClients( return &Clients{ Client: cli, KubeClient: kubeClient, + KubeConfig: kubeConfig, PromClient: promClient, GpuAnalysisService: gpuAnalysisService, BillingService: billingService, } } +// RunUpdateUserSpaceSize 更新用户空间大小 +func RunUpdateUserSpaceSize(_ context.Context, clients *Clients) (any, error) { + var users []model.User + db := query.GetDB() + if err := db.Find(&users).Error; err != nil { + klog.Errorf("RunUpdateUserSpaceSize: 获取用户列表失败: %v", err) + return nil, fmt.Errorf("获取用户列表失败: %w", err) + } + klog.Infof("RunUpdateUserSpaceSize: 共有 %d 个用户", len(users)) + + cfg := config.GetConfig() + prefixConfig := ceph.StoragePrefixConfig{ + User: cfg.Storage.Prefix.User, + Account: cfg.Storage.Prefix.Account, + Public: cfg.Storage.Prefix.Public, + } + + updatedCount := 0 + for _, user := range users { + if user.Space == "" { + klog.Warningf("RunUpdateUserSpaceSize: 用户 %s 的空间路径为空，跳过", user.Name) + continue + } + klog.Infof("RunUpdateUserSpaceSize: 正在获取用户 %s 的空间大小，路径: /user/%s", user.Name, user.Space) + + size, err := ceph.GetCephDirectorySize(clients.KubeClient, clients.KubeConfig, "rook-ceph", "/user/"+user.Space, prefixConfig) + if err != nil { + klog.Errorf("RunUpdateUserSpaceSize: 获取用户 %s 空间大小失败: %v", user.Name, err) + continue + } + klog.Infof("RunUpdateUserSpaceSize: 用户 %s 空间大小: %d bytes", user.Name, size) + + // 检查是否需要记录历史数据 + var lastHistory model.TenantUsageHistory + result := db.Where("tenant_id = ?", user.ID).Order("recorded_at DESC").First(&lastHistory) + + // 检查是否需要插入新记录 + needInsert := false + if result.Error != nil { + // 没有历史记录，需要插入 + needInsert = true + } else { + // 检查字节数差异是否大于 100MB，或者时间超过 1 小时 + byteDiff := size - lastHistory.UsageBytes + if byteDiff < 0 { + byteDiff = -byteDiff + } + timeDiff := time.Since(lastHistory.RecordedAt) + if byteDiff > 100*1024*1024 || timeDiff.Hours() > 1 { + needInsert = true + } + } + + // 插入历史记录 + if needInsert { + history := model.TenantUsageHistory{ + TenantID: user.ID, + UsageBytes: size, + RecordedAt: time.Now(), + } + if err := db.Create(&history).Error; err != nil { + klog.Errorf("RunUpdateUserSpaceSize: 记录用户 %s 空间大小历史失败: %v", user.Name, err) + } else { + klog.Infof("RunUpdateUserSpaceSize: 记录用户 %s 空间大小历史成功", user.Name) + } + } + + var userSpaceSize model.UserSpaceSize + result = db.Where("user_id = ?", user.ID).First(&userSpaceSize) + if result.Error != nil { + userSpaceSize = model.UserSpaceSize{ + UserID: user.ID, + Username: user.Name, + Size: size, + } + if err := db.Create(&userSpaceSize).Error; err != nil { + klog.Errorf("RunUpdateUserSpaceSize: 创建用户 %s 空间大小记录失败: %v", user.Name, err) + continue + } + klog.Infof("RunUpdateUserSpaceSize: 创建用户 %s 空间大小记录成功", user.Name) + } else { + userSpaceSize.Username = user.Name + userSpaceSize.Size = size + if err := db.Save(&userSpaceSize).Error; err != nil { + klog.Errorf("RunUpdateUserSpaceSize: 更新用户 %s 空间大小记录失败: %v", user.Name, err) + continue + } + klog.Infof("RunUpdateUserSpaceSize: 更新用户 %s 空间大小记录成功", user.Name) + } + + updatedCount++ + } + + klog.Infof("RunUpdateUserSpaceSize: 完成，共更新了 %d 个用户的空间大小", updatedCount) + return fmt.Sprintf("更新了 %d 个用户的空间大小", updatedCount), nil +} + +// RunAnalyzeStorageAlerts 对超过90%理论配额且未临时扩容的用户并发执行 AI 分析， +// 并自动应用决策（冻结作业 / 临时扩容）。 +// +//nolint:funlen // Storage alert analysis keeps candidate selection, LLM decision, and enforcement in one cron action. +func RunAnalyzeStorageAlerts(ctx context.Context, clients *Clients) (any, error) { + db := query.GetDB() + maxConcurrentStorageAnalysis := storageAnalysisConcurrency() + + // 查询有空间大小记录的用户，附带配额信息 + type userWithUsage struct { + ID uint `gorm:"column:id"` + Name string `gorm:"column:name"` + Space string `gorm:"column:space"` + SpaceQuota int64 `gorm:"column:space_quota"` + OriginalSpaceQuota *int64 `gorm:"column:original_space_quota"` + CurrentSize int64 `gorm:"column:current_size"` + } + + var candidates []userWithUsage + if err := db.Raw(` + SELECT u.id, u.name, u.space, u.space_quota, u.original_space_quota, uss.size AS current_size + FROM users u + JOIN user_space_sizes uss ON uss.user_id = u.id + WHERE u.deleted_at IS NULL + AND u.original_space_quota IS NULL + AND u.space_quota > 0 + `).Scan(&candidates).Error; err != nil { + return nil, fmt.Errorf("查询用户列表失败: %w", err) + } + + // 过滤出超过 90% 的用户 + var alertUsers []userWithUsage + for _, u := range candidates { + if float64(u.CurrentSize)/float64(u.SpaceQuota) >= 0.9 { + alertUsers = append(alertUsers, u) + } + } + + klog.Infof("RunAnalyzeStorageAlerts: %d 个用户超过90%%配额，启动并发 AI 分析（最大并发 %d）", + len(alertUsers), maxConcurrentStorageAnalysis) + + if len(alertUsers) == 0 { + return "无超额用户，无需分析", nil + } + + if clients.StorageAgent == nil { + if clients.StorageAgentStart == nil || clients.StorageAgentAwait == nil { + klog.Warningf("RunAnalyzeStorageAlerts: StorageAgent 未注入，跳过 AI 分析") + return "StorageAgent 未配置", nil + } + } + + cfg := config.GetConfig() + prefixConfig := ceph.StoragePrefixConfig{ + User: cfg.Storage.Prefix.User, + Account: cfg.Storage.Prefix.Account, + Public: cfg.Storage.Prefix.Public, + } + + applyDecision := func(u userWithUsage, decision *AgentDecision) { + klog.Infof("RunAnalyzeStorageAlerts: 用户 %s 决策: allow_expand=%v expand_bytes=%d freeze=%v reason=%s", + u.Name, decision.AllowExpand, decision.ExpandBytes, decision.FreezeNewJobs, decision.Reason) + + recordDecision := func(action string, runErr error) { + if clients.RecordDecision != nil && decision.DecisionJobID != "" { + clients.RecordDecision(ctx, decision.DecisionJobID, action, runErr) + } + } + if decision.AllowExpand && decision.ExpandBytes > 0 { + newQuota := u.SpaceQuota + decision.ExpandBytes + if err := db.Exec( + "UPDATE users SET original_space_quota = space_quota, space_quota = ?, jobs_frozen = ? WHERE id = ? AND deleted_at IS NULL", + newQuota, decision.FreezeNewJobs, u.ID, + ).Error; err != nil { + klog.Errorf("RunAnalyzeStorageAlerts: 用户 %s 写入扩容失败: %v", u.Name, err) + recordDecision("expand_failed", err) + return + } + if u.Space != "" { + if cephErr := ceph.SetCephDirectoryQuota( + clients.KubeClient, clients.KubeConfig, "rook-ceph", + "/user/"+u.Space, prefixConfig, newQuota, + ); cephErr != nil { + klog.Errorf("RunAnalyzeStorageAlerts: 用户 %s Ceph 配额同步失败: %v", u.Name, cephErr) + } + } + klog.Infof("RunAnalyzeStorageAlerts: 用户 %s 已临时扩容至 %d bytes", u.Name, newQuota) + recordDecision("expand", nil) + return + } + + if decision.FreezeNewJobs { + if err := db.Exec( + "UPDATE users SET jobs_frozen = true WHERE id = ? AND deleted_at IS NULL", u.ID, + ).Error; err != nil { + klog.Errorf("RunAnalyzeStorageAlerts: 用户 %s 设置 jobs_frozen 失败: %v", u.Name, err) + recordDecision("freeze_failed", err) + return + } + + recordDecision("freeze", nil) + klog.Infof("RunAnalyzeStorageAlerts: 用户 %s 已冻结新作业创建", u.Name) + return + } + + recordDecision("observe", nil) + } + + if clients.StorageAgentStart != nil && clients.StorageAgentAwait != nil { + type pendingDecision struct { + user userWithUsage + jobID string + } + + sem := make(chan struct{}, maxConcurrentStorageAnalysis) + var wg sync.WaitGroup + var mu sync.Mutex + pending := make([]pendingDecision, 0, len(alertUsers)) + + for _, candidate := range alertUsers { + u := candidate + wg.Add(1) + sem <- struct{}{} + go func() { + defer wg.Done() + defer func() { <-sem }() + + klog.Infof("RunAnalyzeStorageAlerts: 开始派发用户 %s 的异步 AI 分析任务", u.Name) + jobID, err := clients.StorageAgentStart(ctx, u.Name) + if err != nil { + klog.Errorf("RunAnalyzeStorageAlerts: 用户 %s AI 分析任务派发失败: %v", u.Name, err) + return + } + + klog.Infof("RunAnalyzeStorageAlerts: 用户 %s AI 分析任务已派发 job_id=%s", u.Name, jobID) + mu.Lock() + pending = append(pending, pendingDecision{user: u, jobID: jobID}) + mu.Unlock() + }() + } + wg.Wait() + + if len(pending) == 0 { + return "没有成功派发任何 AI 分析任务", nil + } + + klog.Infof( + "RunAnalyzeStorageAlerts: %d 个用户的 AI 分析任务已派发完成，开始并发等待结果（最大并发 %d）", + len(pending), + maxConcurrentStorageAnalysis, + ) + + wg = sync.WaitGroup{} + for _, item := range pending { + pendingItem := item + wg.Add(1) + sem <- struct{}{} + go func() { + defer wg.Done() + defer func() { <-sem }() + + decision, err := clients.StorageAgentAwait(ctx, pendingItem.user.Name, pendingItem.jobID) + if err != nil { + klog.Errorf("RunAnalyzeStorageAlerts: 用户 %s 等待 AI 分析结果失败: %v", pendingItem.user.Name, err) + return + } + + applyDecision(pendingItem.user, decision) + }() + } + wg.Wait() + return fmt.Sprintf("分析完成，共处理 %d 个超额用户", len(alertUsers)), nil + } + + // 使用 channel 实现有界并发 + sem := make(chan struct{}, maxConcurrentStorageAnalysis) + var wg sync.WaitGroup + for _, candidate := range alertUsers { + u := candidate + wg.Add(1) + sem <- struct{}{} // 占槽（满时阻塞） + go func() { + defer wg.Done() + defer func() { <-sem }() // 释放槽 + + klog.Infof("RunAnalyzeStorageAlerts: 开始分析用户 %s (size=%d quota=%d %.1f%%)", + u.Name, u.CurrentSize, u.SpaceQuota, float64(u.CurrentSize)/float64(u.SpaceQuota)*100) + + decision, err := clients.StorageAgent(u.Name) + if err != nil { + klog.Errorf("RunAnalyzeStorageAlerts: 用户 %s AI 分析失败: %v", u.Name, err) + return + } + + applyDecision(u, decision) + }() + } + + wg.Wait() + return fmt.Sprintf("分析完成，共处理 %d 个超额用户", len(alertUsers)), nil +} + // GetPatrolFunc 根据作业名称返回对应的巡检函数 +// RunAutoShrinkStorageExpansions automatically recovers temporary storage expansions +// once a user's current usage has fallen below a conservative percentage of the +// original theoretical quota. +func RunAutoShrinkStorageExpansions(ctx context.Context, clients *Clients) (any, error) { + db := query.GetDB() + + type expandedUser struct { + ID uint `gorm:"column:id"` + Name string `gorm:"column:name"` + Space string `gorm:"column:space"` + SpaceQuota int64 `gorm:"column:space_quota"` + OriginalSpaceQuota int64 `gorm:"column:original_space_quota"` + CurrentSize int64 `gorm:"column:current_size"` + ShrinkStage string `gorm:"column:shrink_stage"` + ShrinkStageUpdatedAt *time.Time `gorm:"column:shrink_stage_updated_at"` + } + + var users []expandedUser + if err := db.Raw(` + SELECT u.id, u.name, u.space, u.space_quota, u.original_space_quota, uss.size AS current_size, + u.shrink_stage, u.shrink_stage_updated_at + FROM users u + JOIN user_space_sizes uss ON uss.user_id = u.id + WHERE u.deleted_at IS NULL + AND u.original_space_quota IS NOT NULL + `).Scan(&users).Error; err != nil { + return nil, fmt.Errorf("query expanded users failed: %w", err) + } + + if len(users) == 0 { + return "当前没有处于临时扩容状态的用户，无需执行自动缩容。", nil + } + + cfg := config.GetConfig() + prefixConfig := ceph.StoragePrefixConfig{ + User: cfg.Storage.Prefix.User, + Account: cfg.Storage.Prefix.Account, + Public: cfg.Storage.Prefix.Public, + } + + shrunk := 0 + skipped := 0 + for _, user := range users { + if user.OriginalSpaceQuota <= 0 { + skipped++ + continue + } + + usageRatio := float64(user.CurrentSize) / float64(user.OriginalSpaceQuota) + stage := user.ShrinkStage + if stage == "" { + stage = autoShrinkStageExpanded + } + + switch stage { + case autoShrinkStageExpanded: + if usageRatio >= autoShrinkToBufferThreshold { + skipped++ + continue + } + + bufferQuota := calculateShrinkBufferQuota(user.OriginalSpaceQuota, user.SpaceQuota) + if bufferQuota <= user.OriginalSpaceQuota { + bufferQuota = user.OriginalSpaceQuota + } + + if err := db.Exec( + "UPDATE users SET space_quota = ?, shrink_stage = ?, shrink_stage_updated_at = NOW() WHERE id = ? AND deleted_at IS NULL", + bufferQuota, autoShrinkStageBuffer, user.ID, + ).Error; err != nil { + klog.Errorf("RunAutoShrinkStorageExpansions: user=%s buffer shrink failed: %v", user.Name, err) + skipped++ + continue + } + + if user.Space != "" { + if cephErr := ceph.SetCephDirectoryQuota( + clients.KubeClient, + clients.KubeConfig, + "rook-ceph", + "/user/"+user.Space, + prefixConfig, + bufferQuota, + ); cephErr != nil { + klog.Errorf("RunAutoShrinkStorageExpansions: user=%s ceph buffer shrink failed: %v", user.Name, cephErr) + skipped++ + continue + } + } + + shrunk++ + klog.Infof( + "RunAutoShrinkStorageExpansions: user=%s moved to buffer stage quota=%d current_size=%d ratio=%.2f", + user.Name, + bufferQuota, + user.CurrentSize, + usageRatio, + ) + case autoShrinkStageBuffer: + if user.ShrinkStageUpdatedAt == nil || time.Since(*user.ShrinkStageUpdatedAt) < autoShrinkObservationWindow { + skipped++ + continue + } + if usageRatio >= autoShrinkRecoverThreshold { + skipped++ + continue + } + + if err := db.Exec( + "UPDATE users SET space_quota = ?, original_space_quota = NULL, jobs_frozen = false, shrink_stage = NULL, shrink_stage_updated_at = NULL WHERE id = ? AND deleted_at IS NULL", + user.OriginalSpaceQuota, user.ID, + ).Error; err != nil { + klog.Errorf("RunAutoShrinkStorageExpansions: user=%s final shrink failed: %v", user.Name, err) + skipped++ + continue + } + + if user.Space != "" { + if cephErr := ceph.SetCephDirectoryQuota( + clients.KubeClient, + clients.KubeConfig, + "rook-ceph", + "/user/"+user.Space, + prefixConfig, + user.OriginalSpaceQuota, + ); cephErr != nil { + klog.Errorf("RunAutoShrinkStorageExpansions: user=%s ceph final shrink failed: %v", user.Name, cephErr) + skipped++ + continue + } + } + + shrunk++ + klog.Infof( + "RunAutoShrinkStorageExpansions: user=%s fully restored quota=%d current_size=%d ratio=%.2f", + user.Name, + user.OriginalSpaceQuota, + user.CurrentSize, + usageRatio, + ) + default: + skipped++ + } + } + + return fmt.Sprintf("自动缩容扫描完成：已处理 %d 个用户，跳过 %d 个用户。", shrunk, skipped), nil +} + +func calculateShrinkBufferQuota(originalQuota, currentQuota int64) int64 { + if currentQuota <= originalQuota { + return originalQuota + } + + delta := currentQuota - originalQuota + bufferQuota := originalQuota + delta/2 + if bufferQuota <= originalQuota { + return originalQuota + } + return bufferQuota +} + +func RunRefreshPublicStorageIndexBaseline(ctx context.Context, clients *Clients) (any, error) { + if clients.StorageIndex == nil { + return nil, fmt.Errorf("storage index service is not initialized in patrol clients") + } + + job, err := clients.StorageIndex.RefreshPublicBaseline(ctx) + if err != nil { + return nil, fmt.Errorf("refresh public storage index baseline failed: %w", err) + } + + klog.Infof("RunRefreshPublicStorageIndexBaseline: scan_id=%s status=%s redundancy=%d", + job.ScanID, job.Status, job.RedundancyCount) + + return map[string]any{ + "scan_id": job.ScanID, + "workspace_type": job.WorkspaceType, + "workspace_name": job.WorkspaceName, + "status": job.Status, + "entry_count": job.EntryCount, + "redundancy_count": job.RedundancyCount, + }, nil +} + +func RunRefreshUserStorageIndexDaily(ctx context.Context, clients *Clients) (any, error) { + if clients.StorageIndex == nil { + return nil, fmt.Errorf("storage index service is not initialized in patrol clients") + } + + result, err := clients.StorageIndex.RefreshAllUserWorkspaces(ctx) + if err != nil { + return nil, fmt.Errorf("refresh user storage index daily failed: %w", err) + } + + klog.Infof("RunRefreshUserStorageIndexDaily: total=%v success=%v failed=%v", + result["total"], result["success"], result["failed"]) + + return result, nil +} + func GetPatrolFunc(jobName string, clients *Clients, jobConfig datatypes.JSON) (util.AnyFunc, error) { var f util.AnyFunc switch jobName { @@ -74,7 +644,26 @@ func GetPatrolFunc(jobName string, clients *Clients, jobConfig datatypes.JSON) ( f = func(ctx context.Context) (any, error) { return RunTriggerBillingBaseLoop(ctx, clients) } - + case UPDATE_USER_SPACE_SIZE: + f = func(ctx context.Context) (any, error) { + return RunUpdateUserSpaceSize(ctx, clients) + } + case ANALYZE_STORAGE_ALERTS: + f = func(ctx context.Context) (any, error) { + return RunAnalyzeStorageAlerts(ctx, clients) + } + case AUTO_SHRINK_STORAGE_EXPANSIONS: + f = func(ctx context.Context) (any, error) { + return RunAutoShrinkStorageExpansions(ctx, clients) + } + case REFRESH_PUBLIC_STORAGE_INDEX: + f = func(ctx context.Context) (any, error) { + return RunRefreshPublicStorageIndexBaseline(ctx, clients) + } + case REFRESH_USER_STORAGE_INDEX: + f = func(ctx context.Context) (any, error) { + return RunRefreshUserStorageIndexDaily(ctx, clients) + } default: return nil, fmt.Errorf("unsupported patrol job name: %s", jobName) } diff --git a/backend/pkg/storagegovernance/policy.go b/backend/pkg/storagegovernance/policy.go new file mode 100644 index 000000000..38c146ecf --- /dev/null +++ b/backend/pkg/storagegovernance/policy.go @@ -0,0 +1,282 @@ +//nolint:gocritic,gocyclo // Policy evaluation intentionally operates on full snapshot values in a single rules engine. +package storagegovernance + +import ( + "fmt" + "math" + "strings" + "time" + + "github.com/raids-lab/crater/pkg/llm" +) + +const ( + decisionEvidenceCapacity = 5 + usageRatioNearLimit = 0.95 + usageRatioAlertLimit = 0.90 + highGPUHistoryPercent float64 = 80 + lowGPUHistoryPercent float64 = 10 +) + +func ApplySafetyConstraints( + snapshot DecisionSnapshot, + decision llm.LLMDecisionResponse, + cfg ConstraintConfig, + now time.Time, +) (llm.LLMDecisionResponse, ConstraintEvaluation) { + originalDecision := decision + finalDecision := decision + evaluation := ConstraintEvaluation{ + PolicyVersion: cfg.PolicyVersion, + Violations: []string{}, + Adjustments: []string{}, + } + + if snapshot.UsageRatio < cfg.AlertThreshold { + if finalDecision.AllowExpand || finalDecision.FreezeNewJobs { + evaluation.Violations = append(evaluation.Violations, "存储使用率低于告警阈值") + finalDecision.AllowExpand = false + finalDecision.ExpandBytes = 0 + finalDecision.FreezeNewJobs = false + evaluation.Adjustments = append(evaluation.Adjustments, "已禁用扩容，因为当前存储使用率低于告警阈值") + } + } + + if snapshot.TheoreticalQuotaBytes <= 0 && finalDecision.AllowExpand { + evaluation.Violations = append(evaluation.Violations, "理论配额为无限制或未设置") + finalDecision.AllowExpand = false + finalDecision.ExpandBytes = 0 + evaluation.Adjustments = append(evaluation.Adjustments, "已禁用扩容，因为用户不存在有限的理论配额") + } + + if snapshot.IsCurrentlyExpanded && finalDecision.AllowExpand { + evaluation.Violations = append(evaluation.Violations, "用户当前已处于临时扩容状态") + finalDecision.AllowExpand = false + finalDecision.ExpandBytes = 0 + evaluation.Adjustments = append(evaluation.Adjustments, "已禁用扩容，因为上一轮临时扩容仍在生效") + } + + if snapshot.LastExpandAt != nil && now.Sub(*snapshot.LastExpandAt) < cfg.ExpansionCooldown && finalDecision.AllowExpand { + evaluation.Violations = append(evaluation.Violations, "最近一次扩容仍处于冷却时间窗口内") + finalDecision.AllowExpand = false + finalDecision.ExpandBytes = 0 + evaluation.Adjustments = append(evaluation.Adjustments, "已禁用扩容，因为距上次扩容执行尚未超过冷却时间") + } + + if snapshot.UsageRatio >= 1.0 && finalDecision.AllowExpand { + evaluation.Violations = append(evaluation.Violations, "用户当前已超过理论配额") + finalDecision.AllowExpand = false + finalDecision.ExpandBytes = 0 + evaluation.Adjustments = append(evaluation.Adjustments, "已禁用扩容，因为当前使用量已经超过理论配额") + } + + if finalDecision.AllowExpand && finalDecision.ExpandBytes <= 0 { + evaluation.Violations = append(evaluation.Violations, "启用扩容时，扩容量必须为正数") + finalDecision.AllowExpand = false + finalDecision.ExpandBytes = 0 + evaluation.Adjustments = append(evaluation.Adjustments, "已禁用扩容，因为建议扩容量不是正数") + } + + if finalDecision.AllowExpand { + maxAllowed := int64(math.MaxInt64) + + if snapshot.TheoreticalQuotaBytes > 0 && cfg.MaxExpandRatio > 0 { + maxByRatio := int64(float64(snapshot.TheoreticalQuotaBytes) * cfg.MaxExpandRatio) + if maxByRatio > 0 && maxByRatio < maxAllowed { + maxAllowed = maxByRatio + } + } + + if cfg.MaxExpandBytes > 0 && cfg.MaxExpandBytes < maxAllowed { + maxAllowed = cfg.MaxExpandBytes + } + + if snapshot.PlatformTotalBytes > 0 { + reservedByRatio := int64(float64(snapshot.PlatformTotalBytes) * cfg.MinPlatformReservedRatio) + reservedBytes := maxInt64(cfg.MinPlatformReservedBytes, reservedByRatio) + maxByPlatform := snapshot.PlatformAvailableBytes - reservedBytes + if maxByPlatform < maxAllowed { + maxAllowed = maxByPlatform + } + } + + if maxAllowed <= 0 { + evaluation.Violations = append(evaluation.Violations, "执行扩容将突破平台预留容量下限") + finalDecision.AllowExpand = false + finalDecision.ExpandBytes = 0 + evaluation.Adjustments = append(evaluation.Adjustments, "已禁用扩容，因为平台预留容量会低于安全阈值") + } else if finalDecision.ExpandBytes > maxAllowed { + evaluation.Violations = append(evaluation.Violations, "建议扩容量超过安全上限") + evaluation.Adjustments = append( + evaluation.Adjustments, + fmt.Sprintf("已将扩容量从 %d 字节收敛到 %d 字节", finalDecision.ExpandBytes, maxAllowed), + ) + finalDecision.ExpandBytes = maxAllowed + } + } + + if cfg.ForceFreezeWhenOverQuota && snapshot.UsageRatio >= 1.0 && !finalDecision.AllowExpand && !finalDecision.FreezeNewJobs { + evaluation.Violations = append(evaluation.Violations, "用户当前已超过理论配额") + finalDecision.FreezeNewJobs = true + evaluation.Adjustments = append(evaluation.Adjustments, "已强制冻结新作业，因为当前使用量已经超过理论配额") + } + + if !finalDecision.AllowExpand { + finalDecision.ExpandBytes = 0 + } + + evaluation.Violations = uniqueStrings(evaluation.Violations) + evaluation.Adjustments = uniqueStrings(evaluation.Adjustments) + evaluation.Adjusted = len(evaluation.Adjustments) > 0 + evaluation.Blocked = originalDecision.AllowExpand && !finalDecision.AllowExpand + finalDecision.Reason = rewriteDecisionReason(snapshot, originalDecision, finalDecision, evaluation) + + return finalDecision, evaluation +} + +func rewriteDecisionReason( + snapshot DecisionSnapshot, + originalDecision llm.LLMDecisionResponse, + finalDecision llm.LLMDecisionResponse, + evaluation ConstraintEvaluation, +) string { + evidenceParts := make([]string, 0, decisionEvidenceCapacity) + + switch { + case snapshot.UsageRatio >= 1.0: + evidenceParts = append(evidenceParts, "当前使用量已经超过理论配额") + case snapshot.UsageRatio >= usageRatioNearLimit: + evidenceParts = append(evidenceParts, "存储使用率已逼近阈值") + case snapshot.UsageRatio >= usageRatioAlertLimit: + evidenceParts = append(evidenceParts, "存储使用率接近阈值") + default: + evidenceParts = append(evidenceParts, "当前使用量已回落到安全区间") + } + + if growthPhrase := describeGrowth(snapshot.GrowthRateBytesPerHour); growthPhrase != "" { + evidenceParts = append(evidenceParts, growthPhrase) + } + + if snapshot.GPUDataAvailable { + switch { + case snapshot.MaxGPUHistoryPercent >= highGPUHistoryPercent: + evidenceParts = append(evidenceParts, "GPU 历史峰值较高") + case snapshot.MaxGPUHistoryPercent <= lowGPUHistoryPercent: + evidenceParts = append(evidenceParts, "GPU 历史峰值不足以支撑高价值训练判断") + } + } + + if snapshot.IsCurrentlyExpanded { + evidenceParts = append(evidenceParts, "用户当前已处于临时扩容状态") + } + + switch snapshot.ShrinkStage { + case "expanded": + evidenceParts = append(evidenceParts, "当前处于临时扩容后的观察阶段") + case "buffer_reduction": + evidenceParts = append(evidenceParts, "当前处于缩容缓冲阶段") + } + + baseReason := strings.Join(uniqueStrings(evidenceParts), "，") + actionClause := describeFinalAction(snapshot, originalDecision, finalDecision) + if baseReason == "" { + baseReason = actionClause + } else { + baseReason = baseReason + "，" + actionClause + } + + if evaluation.Adjusted { + return appendConstraintReason(baseReason, evaluation) + } + return baseReason +} + +func describeGrowth(growthRate *float64) string { + if growthRate == nil { + return "" + } + + const gib = 1024 * 1024 * 1024 + switch { + case *growthRate >= 4*gib: + return "增长速度较快" + case *growthRate > 0: + return "增长速率平稳" + case *growthRate <= -0.5*gib: + return "使用量持续下降" + default: + return "使用量整体稳定" + } +} + +func describeFinalAction( + snapshot DecisionSnapshot, + originalDecision llm.LLMDecisionResponse, + finalDecision llm.LLMDecisionResponse, +) string { + if finalDecision.AllowExpand { + if originalDecision.AllowExpand && originalDecision.ExpandBytes != finalDecision.ExpandBytes { + return fmt.Sprintf("最终建议在安全约束下扩容 %d 字节，以保护当前作业写入", finalDecision.ExpandBytes) + } + return fmt.Sprintf("建议扩容 %d 字节，以保护当前作业写入", finalDecision.ExpandBytes) + } + + if finalDecision.FreezeNewJobs { + return "不应继续扩容，并需要冻结新作业" + } + + switch { + case snapshot.ShrinkStage == "buffer_reduction": + return "无需继续扩容，可以进一步恢复到原始配额" + case snapshot.IsCurrentlyExpanded && snapshot.UsageRatio < 0.9: + return "无需继续扩容，更适合进入分阶段缩容观察" + default: + return "不应继续扩容，建议继续观察" + } +} + +func appendConstraintReason(reason string, evaluation ConstraintEvaluation) string { + suffix := "安全约束：" + if len(evaluation.Adjustments) > 0 { + suffix += " " + joinWithSemicolon(evaluation.Adjustments) + } + if len(evaluation.Violations) > 0 { + suffix += " | 违规项：" + joinWithSemicolon(evaluation.Violations) + } + if reason == "" { + return suffix + } + return reason + " | " + suffix +} + +func joinWithSemicolon(items []string) string { + return strings.Join(uniqueStrings(items), "; ") +} + +func uniqueStrings(items []string) []string { + if len(items) == 0 { + return items + } + + seen := make(map[string]struct{}, len(items)) + out := make([]string, 0, len(items)) + for _, item := range items { + trimmed := strings.TrimSpace(item) + if trimmed == "" { + continue + } + if _, ok := seen[trimmed]; ok { + continue + } + seen[trimmed] = struct{}{} + out = append(out, trimmed) + } + return out +} + +func maxInt64(a, b int64) int64 { + if a > b { + return a + } + return b +} diff --git a/backend/pkg/storagegovernance/policy_test.go b/backend/pkg/storagegovernance/policy_test.go new file mode 100644 index 000000000..0a2d6df2e --- /dev/null +++ b/backend/pkg/storagegovernance/policy_test.go @@ -0,0 +1,112 @@ +package storagegovernance + +import ( + "strings" + "testing" + "time" + + "github.com/raids-lab/crater/pkg/llm" +) + +func TestApplySafetyConstraintsClampsExpansion(t *testing.T) { + cfg := DefaultConstraintConfig() + cfg.MinPlatformReservedBytes = 10 + cfg.MinPlatformReservedRatio = 0 + snapshot := DecisionSnapshot{ + Username: "alice", + CurrentUsageBytes: 95, + TheoreticalQuotaBytes: 100, + UsageRatio: 0.95, + PlatformTotalBytes: 1000, + PlatformAvailableBytes: 400, + } + decision := llm.LLMDecisionResponse{ + AllowExpand: true, + ExpandBytes: 80, + Reason: "raw decision", + } + + finalDecision, evaluation := ApplySafetyConstraints(snapshot, decision, cfg, time.Now()) + + if !finalDecision.AllowExpand { + t.Fatalf("expected expansion to remain enabled after clamping") + } + if finalDecision.ExpandBytes != 30 { + t.Fatalf("expected expansion to be clamped to 30, got %d", finalDecision.ExpandBytes) + } + if !evaluation.Adjusted { + t.Fatalf("expected evaluation to mark the decision as adjusted") + } +} + +func TestApplySafetyConstraintsForcesFreezeWhenOverQuota(t *testing.T) { + cfg := DefaultConstraintConfig() + snapshot := DecisionSnapshot{ + Username: "alice", + CurrentUsageBytes: 120, + TheoreticalQuotaBytes: 100, + UsageRatio: 1.20, + PlatformTotalBytes: 1000, + PlatformAvailableBytes: 500, + } + decision := llm.LLMDecisionResponse{ + AllowExpand: false, + ExpandBytes: 0, + FreezeNewJobs: false, + Reason: "raw decision", + } + + finalDecision, evaluation := ApplySafetyConstraints(snapshot, decision, cfg, time.Now()) + + if !finalDecision.FreezeNewJobs { + t.Fatalf("expected freeze_new_jobs to be forced on when user is over quota") + } + if finalDecision.AllowExpand { + t.Fatalf("expected allow_expand to remain disabled when user is over quota") + } + if !evaluation.Adjusted { + t.Fatalf("expected evaluation to mark the decision as adjusted") + } + if strings.Contains(finalDecision.Reason, "建议扩容") || strings.Contains(finalDecision.Reason, "优先扩容") { + t.Fatalf("expected rewritten final reason to stop supporting expansion, got %q", finalDecision.Reason) + } + if !strings.Contains(finalDecision.Reason, "冻结新作业") { + t.Fatalf("expected rewritten final reason to mention freezing new jobs, got %q", finalDecision.Reason) + } +} + +func TestApplySafetyConstraintsBlocksExpansionWhenAlreadyOverQuota(t *testing.T) { + cfg := DefaultConstraintConfig() + snapshot := DecisionSnapshot{ + Username: "alice", + CurrentUsageBytes: 120, + TheoreticalQuotaBytes: 100, + UsageRatio: 1.20, + PlatformTotalBytes: 1000, + PlatformAvailableBytes: 500, + } + decision := llm.LLMDecisionResponse{ + AllowExpand: true, + ExpandBytes: 20, + FreezeNewJobs: false, + Reason: "建议扩容保护作业", + } + + finalDecision, evaluation := ApplySafetyConstraints(snapshot, decision, cfg, time.Now()) + + if finalDecision.AllowExpand { + t.Fatalf("expected expansion to be disabled when user is already over quota") + } + if finalDecision.ExpandBytes != 0 { + t.Fatalf("expected expand_bytes to be reset to 0, got %d", finalDecision.ExpandBytes) + } + if !finalDecision.FreezeNewJobs { + t.Fatalf("expected freeze_new_jobs to be forced on when user is already over quota") + } + if !evaluation.Adjusted { + t.Fatalf("expected evaluation to mark the decision as adjusted") + } + if strings.Contains(finalDecision.Reason, "建议扩容") { + t.Fatalf("expected final reason to be rewritten without expansion wording, got %q", finalDecision.Reason) + } +} diff --git a/backend/pkg/storagegovernance/query.go b/backend/pkg/storagegovernance/query.go new file mode 100644 index 000000000..32af3ebc6 --- /dev/null +++ b/backend/pkg/storagegovernance/query.go @@ -0,0 +1,175 @@ +//nolint:gocritic // Query projection helpers favor value semantics for readability on bounded records. +package storagegovernance + +import ( + "context" + "encoding/json" + + "github.com/raids-lab/crater/dao/model" + "github.com/raids-lab/crater/dao/query" + "github.com/raids-lab/crater/pkg/llm" +) + +type DecisionRecordSummary struct { + JobID string `json:"job_id"` + Username string `json:"username"` + Source model.StorageDecisionSource `json:"source"` + Status model.StorageDecisionStatus `json:"status"` + TriggerReason string `json:"trigger_reason"` + RawAllowExpand bool `json:"raw_allow_expand"` + RawExpandBytes int64 `json:"raw_expand_bytes"` + RawFreezeNewJobs bool `json:"raw_freeze_new_jobs"` + FinalAllowExpand bool `json:"final_allow_expand"` + FinalExpandBytes int64 `json:"final_expand_bytes"` + FinalFreezeNewJobs bool `json:"final_freeze_new_jobs"` + ConstraintAdjusted bool `json:"constraint_adjusted"` + ConstraintBlocked bool `json:"constraint_blocked"` + AppliedAction string `json:"applied_action"` + ErrorMessage string `json:"error_message"` + ConstraintVersion string `json:"constraint_version"` + LatencyMs int64 `json:"latency_ms"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + +type DecisionRecordDetail struct { + DecisionRecordSummary + CurrentShrinkStage string `json:"current_shrink_stage,omitempty"` + Snapshot *DecisionSnapshot `json:"snapshot,omitempty"` + RawDecision *llm.LLMDecisionResponse `json:"raw_decision,omitempty"` + FinalDecision *llm.LLMDecisionResponse `json:"final_decision,omitempty"` + ConstraintResult *ConstraintEvaluation `json:"constraint_result,omitempty"` +} + +type DecisionRecordPage struct { + Items []DecisionRecordSummary `json:"items"` + Total int64 `json:"total"` + Page int `json:"page"` + PageSize int `json:"page_size"` + TotalPages int `json:"total_pages"` +} + +func ListDecisionRecords( + ctx context.Context, + page int, + pageSize int, + username string, + status string, + source string, +) (*DecisionRecordPage, error) { + if page <= 0 { + page = 1 + } + if pageSize <= 0 { + pageSize = 20 + } + + tx := query.GetDB().WithContext(ctx).Model(&model.StorageDecisionRecord{}) + if username != "" { + tx = tx.Where("username = ?", username) + } + if status != "" { + tx = tx.Where("status = ?", status) + } + if source != "" { + tx = tx.Where("source = ?", source) + } + + var total int64 + if err := tx.Count(&total).Error; err != nil { + return nil, err + } + + var records []model.StorageDecisionRecord + if err := tx.Order("created_at desc"). + Offset((page - 1) * pageSize). + Limit(pageSize). + Find(&records).Error; err != nil { + return nil, err + } + + items := make([]DecisionRecordSummary, 0, len(records)) + for _, record := range records { + items = append(items, summarizeRecord(record)) + } + + return &DecisionRecordPage{ + Items: items, + Total: total, + Page: page, + PageSize: pageSize, + TotalPages: int((total + int64(pageSize) - 1) / int64(pageSize)), + }, nil +} + +func GetDecisionRecord(ctx context.Context, jobID string) (*DecisionRecordDetail, error) { + var record model.StorageDecisionRecord + if err := query.GetDB().WithContext(ctx).Where("job_id = ?", jobID).First(&record).Error; err != nil { + return nil, err + } + + detail := &DecisionRecordDetail{ + DecisionRecordSummary: summarizeRecord(record), + } + if len(record.Snapshot) > 0 { + var snapshot DecisionSnapshot + if err := json.Unmarshal(record.Snapshot, &snapshot); err == nil { + detail.Snapshot = &snapshot + } + } + if len(record.RawDecision) > 0 { + var raw llm.LLMDecisionResponse + if err := json.Unmarshal(record.RawDecision, &raw); err == nil { + detail.RawDecision = &raw + } + } + if len(record.FinalDecision) > 0 { + var final llm.LLMDecisionResponse + if err := json.Unmarshal(record.FinalDecision, &final); err == nil { + detail.FinalDecision = &final + } + } + if len(record.ConstraintResult) > 0 { + var evaluation ConstraintEvaluation + if err := json.Unmarshal(record.ConstraintResult, &evaluation); err == nil { + detail.ConstraintResult = &evaluation + } + } + + var userState struct { + ShrinkStage string `gorm:"column:shrink_stage"` + } + if err := query.GetDB().WithContext(ctx). + Raw("SELECT shrink_stage FROM users WHERE name = ? AND deleted_at IS NULL", record.Username). + Scan(&userState).Error; err == nil { + detail.CurrentShrinkStage = userState.ShrinkStage + } + + return detail, nil +} + +func summarizeRecord(record model.StorageDecisionRecord) DecisionRecordSummary { + return DecisionRecordSummary{ + JobID: record.JobID, + Username: record.Username, + Source: record.Source, + Status: record.Status, + TriggerReason: record.TriggerReason, + RawAllowExpand: record.RawAllowExpand, + RawExpandBytes: record.RawExpandBytes, + RawFreezeNewJobs: record.RawFreezeNewJobs, + FinalAllowExpand: record.FinalAllowExpand, + FinalExpandBytes: record.FinalExpandBytes, + FinalFreezeNewJobs: record.FinalFreezeNewJobs, + ConstraintAdjusted: record.ConstraintAdjusted, + ConstraintBlocked: record.ConstraintBlocked, + AppliedAction: record.AppliedAction, + ErrorMessage: record.ErrorMessage, + ConstraintVersion: record.ConstraintVersion, + LatencyMs: record.LatencyMs, + CreatedAt: record.CreatedAt.Format(timeLayout), + UpdatedAt: record.UpdatedAt.Format(timeLayout), + } +} + +const timeLayout = "2006-01-02 15:04:05" diff --git a/backend/pkg/storagegovernance/replay.go b/backend/pkg/storagegovernance/replay.go new file mode 100644 index 000000000..4051e0235 --- /dev/null +++ b/backend/pkg/storagegovernance/replay.go @@ -0,0 +1,88 @@ +package storagegovernance + +import ( + "context" + "encoding/json" + + "github.com/raids-lab/crater/dao/model" + "github.com/raids-lab/crater/dao/query" + "github.com/raids-lab/crater/pkg/llm" +) + +func ReplayStoredDecisions( + ctx context.Context, + cfg ConstraintConfig, + limit int, +) (*ReplaySummary, error) { + if limit <= 0 { + limit = 100 + } + + var records []model.StorageDecisionRecord + if err := query.GetDB().WithContext(ctx). + Where("status = ?", model.StorageDecisionStatusDone). + Order("created_at desc"). + Limit(limit). + Find(&records).Error; err != nil { + return nil, err + } + + summary := &ReplaySummary{ + PolicyVersion: cfg.PolicyVersion, + Records: make([]ReplayRecord, 0, len(records)), + } + + for i := range records { + record := &records[i] + if len(record.Snapshot) == 0 || len(record.RawDecision) == 0 { + continue + } + + var snapshot DecisionSnapshot + if err := json.Unmarshal(record.Snapshot, &snapshot); err != nil { + return nil, err + } + + var rawDecision llm.LLMDecisionResponse + if err := json.Unmarshal(record.RawDecision, &rawDecision); err != nil { + return nil, err + } + + replayedDecision, evaluation := ApplySafetyConstraints(snapshot, rawDecision, cfg, record.CreatedAt) + replayRecord := ReplayRecord{ + JobID: record.JobID, + Username: record.Username, + StoredAdjusted: record.ConstraintAdjusted, + StoredBlocked: record.ConstraintBlocked, + ReplayAdjusted: evaluation.Adjusted, + ReplayBlocked: evaluation.Blocked, + StoredAllowExpand: record.FinalAllowExpand, + ReplayAllowExpand: replayedDecision.AllowExpand, + StoredExpandBytes: record.FinalExpandBytes, + ReplayExpandBytes: replayedDecision.ExpandBytes, + StoredFreeze: record.FinalFreezeNewJobs, + ReplayFreeze: replayedDecision.FreezeNewJobs, + Evaluation: evaluation, + } + + summary.TotalCases++ + if evaluation.Blocked { + summary.BlockedCases++ + } + if evaluation.Adjusted { + summary.ClampedCases++ + } + if !record.FinalFreezeNewJobs && replayedDecision.FreezeNewJobs { + summary.FreezeEscalations++ + } + if record.FinalAllowExpand != replayedDecision.AllowExpand || + record.FinalExpandBytes != replayedDecision.ExpandBytes || + record.FinalFreezeNewJobs != replayedDecision.FreezeNewJobs { + summary.ChangedCases++ + } + + summary.Records = append(summary.Records, replayRecord) + } + + return summary, nil +} diff --git a/backend/pkg/storagegovernance/service.go b/backend/pkg/storagegovernance/service.go new file mode 100644 index 000000000..c389ed234 --- /dev/null +++ b/backend/pkg/storagegovernance/service.go @@ -0,0 +1,589 @@ +//nolint:gocritic,gocyclo,mnd,unparam // Engine orchestration intentionally centralizes snapshot collection and decision execution. +package storagegovernance + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "strings" + "time" + + "gorm.io/datatypes" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + + "github.com/raids-lab/crater/dao/model" + "github.com/raids-lab/crater/dao/query" + "github.com/raids-lab/crater/pkg/ceph" + "github.com/raids-lab/crater/pkg/config" + "github.com/raids-lab/crater/pkg/llm" + "github.com/raids-lab/crater/pkg/monitor" +) + +type DecisionRequest struct { + Username string + Source model.StorageDecisionSource + TriggerReason string +} + +type StoredDecisionStatus struct { + Status string `json:"status"` + Result *llm.LLMDecisionResponse `json:"result,omitempty"` + ErrorMsg string `json:"error,omitempty"` + ConstraintAdjusted bool `json:"constraint_adjusted,omitempty"` + ConstraintBlocked bool `json:"constraint_blocked,omitempty"` +} + +type Engine struct { + kubeClient kubernetes.Interface + kubeConfig *rest.Config + promClient monitor.PrometheusInterface + config ConstraintConfig +} + +func NewEngine( + kubeClient kubernetes.Interface, + kubeConfig *rest.Config, + promClient monitor.PrometheusInterface, + cfg ConstraintConfig, +) *Engine { + if cfg.PolicyVersion == "" { + cfg = DefaultConstraintConfig() + } + return &Engine{ + kubeClient: kubeClient, + kubeConfig: kubeConfig, + promClient: promClient, + config: cfg, + } +} + +func (e *Engine) StartAsyncDecision(ctx context.Context, req DecisionRequest) (string, error) { + jobID, err := e.createPendingRecord(ctx, req) + if err != nil { + return "", err + } + + go func() { + runCtx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + _, _ = e.RunDecision(runCtx, jobID, req) + }() + + return jobID, nil +} + +func (e *Engine) DecideAndRecord(ctx context.Context, req DecisionRequest) (*llm.LLMDecisionResponse, string, error) { + jobID, err := e.createPendingRecord(ctx, req) + if err != nil { + return nil, "", err + } + + decision, err := e.RunDecision(ctx, jobID, req) + if err != nil { + return nil, jobID, err + } + + return decision, jobID, nil +} + +func (e *Engine) RunDecision(ctx context.Context, jobID string, req DecisionRequest) (*llm.LLMDecisionResponse, error) { + startedAt := time.Now() + _ = query.GetDB().WithContext(ctx). + Model(&model.StorageDecisionRecord{}). + Where("job_id = ?", jobID). + Updates(map[string]any{ + "status": model.StorageDecisionStatusRunning, + "started_at": startedAt, + }).Error + + snapshot, err := e.BuildSnapshot(ctx, req.Username) + if err != nil { + e.markError(ctx, jobID, startedAt, err) + return nil, err + } + + var rawDecision *llm.LLMDecisionResponse + switch llm.GetStorageDecisionMode(ctx) { + case llm.StorageDecisionModeDirect: + snapshotJSON, marshalErr := json.Marshal(snapshot) + if marshalErr != nil { + e.persistFailure(ctx, jobID, snapshot, startedAt, marshalErr) + return nil, marshalErr + } + rawDecision, err = llm.AskDirectDecision(ctx, string(snapshotJSON)) + default: + rawDecision, err = llm.AskAgentForDecision(e.kubeClient, e.kubeConfig, req.Username, e.promClient) + } + if err != nil { + e.persistFailure(ctx, jobID, snapshot, startedAt, err) + return nil, err + } + + finalDecision, evaluation := ApplySafetyConstraints(snapshot, *rawDecision, e.config, time.Now()) + if err := e.persistSuccess(ctx, jobID, req, snapshot, *rawDecision, finalDecision, evaluation, startedAt); err != nil { + return nil, err + } + + return &finalDecision, nil +} + +func (e *Engine) BuildSnapshot(ctx context.Context, username string) (DecisionSnapshot, error) { + var userRow struct { + ID uint `gorm:"column:id"` + Name string `gorm:"column:name"` + Space string `gorm:"column:space"` + SpaceQuota int64 `gorm:"column:space_quota"` + OriginalSpaceQuota *int64 `gorm:"column:original_space_quota"` + JobsFrozen bool `gorm:"column:jobs_frozen"` + ShrinkStage string `gorm:"column:shrink_stage"` + } + + if err := query.GetDB().WithContext(ctx).Raw( + "SELECT id, name, space, space_quota, original_space_quota, jobs_frozen, shrink_stage FROM users WHERE name = ? AND deleted_at IS NULL", + username, + ).Scan(&userRow).Error; err != nil { + return DecisionSnapshot{}, fmt.Errorf("query user snapshot failed: %w", err) + } + if userRow.ID == 0 { + return DecisionSnapshot{}, fmt.Errorf("user %s not found", username) + } + + cfg := config.GetConfig() + prefixConfig := ceph.StoragePrefixConfig{ + User: cfg.Storage.Prefix.User, + Account: cfg.Storage.Prefix.Account, + Public: cfg.Storage.Prefix.Public, + } + + currentUsage, err := ceph.GetCephDirectorySize( + e.kubeClient, + e.kubeConfig, + "rook-ceph", + "/user/"+userRow.Space, + prefixConfig, + ) + if err != nil { + currentUsage = ceph.UnknownSizeBytes + } + + theoreticalQuota := userRow.SpaceQuota + if userRow.OriginalSpaceQuota != nil { + theoreticalQuota = *userRow.OriginalSpaceQuota + } + + totalCapacity, usedCapacity, err := ceph.GetCraterStorageCapacity(e.kubeClient, e.kubeConfig, "rook-ceph") + if err != nil { + return DecisionSnapshot{}, fmt.Errorf("get platform capacity failed: %w", err) + } + + runtimeFeatures, err := e.collectTenantRuntimeFeatures(ctx, username) + if err != nil { + return DecisionSnapshot{}, fmt.Errorf("collect tenant runtime features failed: %w", err) + } + + var historyRows []model.TenantUsageHistory + if err := query.GetDB().WithContext(ctx). + Where("tenant_id = ?", userRow.ID). + Order("recorded_at desc"). + Limit(10). + Find(&historyRows).Error; err != nil { + return DecisionSnapshot{}, fmt.Errorf("query usage history failed: %w", err) + } + + recentHistory := make([]UsageHistoryPoint, 0, len(historyRows)) + for _, row := range historyRows { + recentHistory = append(recentHistory, UsageHistoryPoint{ + RecordedAt: row.RecordedAt, + UsageBytes: row.UsageBytes, + }) + } + slices.Reverse(recentHistory) + + var growthRate *float64 + if len(recentHistory) >= 2 { + first := recentHistory[0] + last := recentHistory[len(recentHistory)-1] + hours := last.RecordedAt.Sub(first.RecordedAt).Hours() + if hours > 0 { + value := float64(last.UsageBytes-first.UsageBytes) / hours + growthRate = &value + } + } + + var usageRatio float64 + if theoreticalQuota > 0 && currentUsage >= 0 { + usageRatio = float64(currentUsage) / float64(theoreticalQuota) + } + availableCapacity := ceph.AvailableBytes(totalCapacity, usedCapacity) + + var lastExpandRecord model.StorageDecisionRecord + var lastExpandAt *time.Time + appliedExpandActions := []string{ + "expand", + "manual_expand", + "manual_expand_and_freeze", + } + if err := query.GetDB().WithContext(ctx). + Where( + "username = ? AND status = ? AND applied_action IN ?", + username, + model.StorageDecisionStatusDone, + appliedExpandActions, + ). + Order("updated_at desc"). + First(&lastExpandRecord).Error; err == nil { + lastExpandAt = &lastExpandRecord.UpdatedAt + } + + return DecisionSnapshot{ + Username: username, + UserID: userRow.ID, + CurrentUsageBytes: currentUsage, + CurrentQuotaBytes: userRow.SpaceQuota, + TheoreticalQuotaBytes: theoreticalQuota, + UsageRatio: usageRatio, + GrowthRateBytesPerHour: growthRate, + PlatformTotalBytes: totalCapacity, + PlatformUsedBytes: usedCapacity, + PlatformAvailableBytes: availableCapacity, + IsCurrentlyExpanded: userRow.OriginalSpaceQuota != nil, + JobsFrozen: userRow.JobsFrozen, + ShrinkStage: userRow.ShrinkStage, + ActivePodCount: runtimeFeatures.ActivePodCount, + ActiveGPUPodCount: runtimeFeatures.ActiveGPUPodCount, + ActiveGPURequestTotal: runtimeFeatures.ActiveGPURequestTotal, + ActiveCPURequestCores: runtimeFeatures.ActiveCPURequestCores, + ActiveMemoryRequestMB: runtimeFeatures.ActiveMemoryRequestMB, + RealtimeCPUCores: runtimeFeatures.RealtimeCPUCores, + RealtimeMemoryMB: runtimeFeatures.RealtimeMemoryMB, + RealtimeGPUUtilPercent: runtimeFeatures.RealtimeGPUUtilPercent, + RealtimeGPUMemoryMB: runtimeFeatures.RealtimeGPUMemoryMB, + GPUDataAvailable: runtimeFeatures.GPUDataAvailable, + MaxGPUHistoryPercent: runtimeFeatures.MaxGPUHistoryPercent, + LastExpandAt: lastExpandAt, + RecentHistory: recentHistory, + }, nil +} + +type tenantRuntimeFeatures struct { + ActivePodCount int + ActiveGPUPodCount int + ActiveGPURequestTotal int + ActiveCPURequestCores float64 + ActiveMemoryRequestMB float64 + RealtimeCPUCores float64 + RealtimeMemoryMB float64 + RealtimeGPUUtilPercent float64 + RealtimeGPUMemoryMB float64 + GPUDataAvailable bool + MaxGPUHistoryPercent float64 +} + +func (e *Engine) collectTenantRuntimeFeatures(ctx context.Context, username string) (*tenantRuntimeFeatures, error) { + features := &tenantRuntimeFeatures{} + jobNamespace := config.GetConfig().Namespaces.Job + + pods, err := e.kubeClient.CoreV1().Pods(jobNamespace).List(ctx, metav1.ListOptions{ + LabelSelector: "crater.raids.io/task-user=" + username, + }) + if err != nil { + return nil, err + } + + gpuUtilSum := 0.0 + gpuUtilCount := 0 + gpuMemSum := 0.0 + + for _, pod := range pods.Items { + phase := string(pod.Status.Phase) + if phase != "Running" && phase != "Pending" { + continue + } + + features.ActivePodCount++ + podGPURequests := 0 + for _, container := range pod.Spec.Containers { + for resourceName, quantity := range container.Resources.Requests { + if resourceName == corev1.ResourceCPU { + features.ActiveCPURequestCores += float64(quantity.MilliValue()) / 1000 + } + if resourceName == corev1.ResourceMemory { + features.ActiveMemoryRequestMB += float64(quantity.Value()) / 1024 / 1024 + } + if strings.Contains(string(resourceName), "nvidia.com/") { + podGPURequests += int(quantity.Value()) + features.ActiveGPURequestTotal += int(quantity.Value()) + } + } + } + if podGPURequests > 0 { + features.ActiveGPUPodCount++ + } + + if e.promClient != nil { + runtimeMetrics, err := e.queryPodRealtimeMetrics(pod.Name) + if err == nil { + features.RealtimeCPUCores += runtimeMetrics.CPUCores + features.RealtimeMemoryMB += runtimeMetrics.MemoryMB + if runtimeMetrics.GPUDataAvailable { + features.GPUDataAvailable = true + gpuUtilSum += runtimeMetrics.GPUUtilPercent + gpuUtilCount++ + gpuMemSum += runtimeMetrics.GPUMemoryMB + } + } + + if podGPURequests > 0 { + historyMetrics, err := e.queryPodGPUHistory(pod.Name, 24) + if err == nil && historyMetrics.MaxUtil > features.MaxGPUHistoryPercent { + features.MaxGPUHistoryPercent = historyMetrics.MaxUtil + } + } + } + } + + if gpuUtilCount > 0 { + features.RealtimeGPUUtilPercent = gpuUtilSum / float64(gpuUtilCount) + features.RealtimeGPUMemoryMB = gpuMemSum / float64(gpuUtilCount) + } + + return features, nil +} + +type podRealtimeMetrics struct { + CPUCores float64 + MemoryMB float64 + GPUUtilPercent float64 + GPUMemoryMB float64 + GPUDataAvailable bool +} + +func (e *Engine) queryPodRealtimeMetrics(podName string) (*podRealtimeMetrics, error) { + jobNamespace := config.GetConfig().Namespaces.Job + result := &podRealtimeMetrics{} + + if v, ok, err := e.promClient.QueryInstant( + fmt.Sprintf(`sum(rate(container_cpu_usage_seconds_total{pod=%q,container!=""}[5m]))`, podName), + ); err == nil && ok { + result.CPUCores = v + } + if v, ok, err := e.promClient.QueryInstant( + fmt.Sprintf(`sum(container_memory_usage_bytes{pod=%q,container!=""})`, podName), + ); err == nil && ok { + result.MemoryMB = v / 1024 / 1024 + } + + for _, query := range []string{ + fmt.Sprintf(`avg(DCGM_FI_DEV_GPU_UTIL{namespace=%q,pod=%q})`, jobNamespace, podName), + fmt.Sprintf(`avg(DCGM_FI_DEV_GPU_UTIL{pod=%q})`, podName), + } { + if v, ok, err := e.promClient.QueryInstant(query); err == nil && ok { + result.GPUUtilPercent = v + result.GPUDataAvailable = true + break + } + } + + for _, query := range []string{ + fmt.Sprintf(`avg(DCGM_FI_DEV_FB_USED{namespace=%q,pod=%q})`, jobNamespace, podName), + fmt.Sprintf(`avg(DCGM_FI_DEV_FB_USED{pod=%q})`, podName), + } { + if v, ok, err := e.promClient.QueryInstant(query); err == nil && ok { + result.GPUMemoryMB = v + break + } + } + + return result, nil +} + +type podGPUHistory struct { + MaxUtil float64 +} + +func (e *Engine) queryPodGPUHistory(podName string, durationHours float64) (*podGPUHistory, error) { + jobNamespace := config.GetConfig().Namespaces.Job + duration := fmt.Sprintf("%.0fh", durationHours) + if durationHours < 1 { + duration = fmt.Sprintf("%.0fm", durationHours*60) + } + + result := &podGPUHistory{} + for _, query := range []string{ + fmt.Sprintf(`max_over_time(DCGM_FI_DEV_GPU_UTIL{namespace=%q,pod=%q}[%s])`, jobNamespace, podName, duration), + fmt.Sprintf(`max_over_time(DCGM_FI_DEV_GPU_UTIL{pod=%q}[%s])`, podName, duration), + } { + if v, ok, err := e.promClient.QueryInstant(query); err == nil && ok { + result.MaxUtil = v + return result, nil + } + } + + return result, nil +} + +func GetDecisionStatus(ctx context.Context, jobID string) (*StoredDecisionStatus, error) { + var record model.StorageDecisionRecord + if err := query.GetDB().WithContext(ctx).Where("job_id = ?", jobID).First(&record).Error; err != nil { + return nil, err + } + + status := &StoredDecisionStatus{ + Status: string(record.Status), + ErrorMsg: record.ErrorMessage, + ConstraintAdjusted: record.ConstraintAdjusted, + ConstraintBlocked: record.ConstraintBlocked, + } + + if len(record.FinalDecision) > 0 { + var decision llm.LLMDecisionResponse + if err := json.Unmarshal(record.FinalDecision, &decision); err == nil { + status.Result = &decision + } + } + + return status, nil +} + +func MarkDecisionExecution(ctx context.Context, jobID, action string, runErr error) error { + updates := map[string]any{ + "applied_action": action, + } + if runErr != nil { + updates["error_message"] = runErr.Error() + } else { + updates["error_message"] = "" + } + + return query.GetDB().WithContext(ctx). + Model(&model.StorageDecisionRecord{}). + Where("job_id = ?", jobID). + Updates(updates).Error +} + +func (e *Engine) createPendingRecord(ctx context.Context, req DecisionRequest) (string, error) { + var userRow struct { + ID uint `gorm:"column:id"` + } + if err := query.GetDB().WithContext(ctx).Raw( + "SELECT id FROM users WHERE name = ? AND deleted_at IS NULL", + req.Username, + ).Scan(&userRow).Error; err != nil { + return "", err + } + if userRow.ID == 0 { + return "", fmt.Errorf("user %s not found", req.Username) + } + + jobID := NewJobID() + record := model.StorageDecisionRecord{ + JobID: jobID, + UserID: userRow.ID, + Username: req.Username, + Source: req.Source, + Status: model.StorageDecisionStatusPending, + TriggerReason: req.TriggerReason, + StartedAt: nil, + } + if err := query.GetDB().WithContext(ctx).Create(&record).Error; err != nil { + return "", err + } + return jobID, nil +} + +func (e *Engine) persistFailure( + ctx context.Context, + jobID string, + snapshot DecisionSnapshot, + startedAt time.Time, + runErr error, +) { + updates := map[string]any{ + "status": model.StorageDecisionStatusError, + "error_message": runErr.Error(), + "finished_at": time.Now(), + "latency_ms": time.Since(startedAt).Milliseconds(), + } + if data, err := json.Marshal(snapshot); err == nil { + updates["snapshot"] = datatypes.JSON(data) + } + _ = query.GetDB().WithContext(ctx). + Model(&model.StorageDecisionRecord{}). + Where("job_id = ?", jobID). + Updates(updates).Error +} + +func (e *Engine) markError(ctx context.Context, jobID string, startedAt time.Time, runErr error) { + _ = query.GetDB().WithContext(ctx). + Model(&model.StorageDecisionRecord{}). + Where("job_id = ?", jobID). + Updates(map[string]any{ + "status": model.StorageDecisionStatusError, + "error_message": runErr.Error(), + "finished_at": time.Now(), + "latency_ms": time.Since(startedAt).Milliseconds(), + }).Error +} + +func (e *Engine) persistSuccess( + ctx context.Context, + jobID string, + req DecisionRequest, + snapshot DecisionSnapshot, + rawDecision llm.LLMDecisionResponse, + finalDecision llm.LLMDecisionResponse, + evaluation ConstraintEvaluation, + startedAt time.Time, +) error { + snapshotJSON, err := json.Marshal(snapshot) + if err != nil { + return err + } + rawDecisionJSON, err := json.Marshal(rawDecision) + if err != nil { + return err + } + finalDecisionJSON, err := json.Marshal(finalDecision) + if err != nil { + return err + } + evaluationJSON, err := json.Marshal(evaluation) + if err != nil { + return err + } + + return query.GetDB().WithContext(ctx). + Model(&model.StorageDecisionRecord{}). + Where("job_id = ?", jobID). + Updates(map[string]any{ + "user_id": snapshot.UserID, + "username": req.Username, + "source": req.Source, + "status": model.StorageDecisionStatusDone, + "snapshot": datatypes.JSON(snapshotJSON), + "raw_decision": datatypes.JSON(rawDecisionJSON), + "final_decision": datatypes.JSON(finalDecisionJSON), + "constraint_result": datatypes.JSON(evaluationJSON), + "raw_allow_expand": rawDecision.AllowExpand, + "raw_expand_bytes": rawDecision.ExpandBytes, + "raw_freeze_new_jobs": rawDecision.FreezeNewJobs, + "final_allow_expand": finalDecision.AllowExpand, + "final_expand_bytes": finalDecision.ExpandBytes, + "final_freeze_new_jobs": finalDecision.FreezeNewJobs, + "constraint_adjusted": evaluation.Adjusted, + "constraint_blocked": evaluation.Blocked, + "constraint_version": evaluation.PolicyVersion, + "applied_action": "", + "error_message": "", + "finished_at": time.Now(), + "latency_ms": time.Since(startedAt).Milliseconds(), + }).Error +} + +func NewJobID() string { + return fmt.Sprintf("sd-%d", time.Now().UnixNano()) +} diff --git a/backend/pkg/storagegovernance/types.go b/backend/pkg/storagegovernance/types.go new file mode 100644 index 000000000..bc5f8e184 --- /dev/null +++ b/backend/pkg/storagegovernance/types.go @@ -0,0 +1,95 @@ +package storagegovernance + +import "time" + +type UsageHistoryPoint struct { + RecordedAt time.Time `json:"recorded_at"` + UsageBytes int64 `json:"usage_bytes"` +} + +type DecisionSnapshot struct { + Username string `json:"username"` + UserID uint `json:"user_id"` + CurrentUsageBytes int64 `json:"current_usage_bytes"` + CurrentQuotaBytes int64 `json:"current_quota_bytes"` + TheoreticalQuotaBytes int64 `json:"theoretical_quota_bytes"` + UsageRatio float64 `json:"usage_ratio"` + GrowthRateBytesPerHour *float64 `json:"growth_rate_bytes_per_hour,omitempty"` + PlatformTotalBytes int64 `json:"platform_total_bytes"` + PlatformUsedBytes int64 `json:"platform_used_bytes"` + PlatformAvailableBytes int64 `json:"platform_available_bytes"` + IsCurrentlyExpanded bool `json:"is_currently_expanded"` + JobsFrozen bool `json:"jobs_frozen"` + ShrinkStage string `json:"shrink_stage"` + ActivePodCount int `json:"active_pod_count"` + ActiveGPUPodCount int `json:"active_gpu_pod_count"` + ActiveGPURequestTotal int `json:"active_gpu_request_total"` + ActiveCPURequestCores float64 `json:"active_cpu_request_cores"` + ActiveMemoryRequestMB float64 `json:"active_memory_request_mb"` + RealtimeCPUCores float64 `json:"realtime_cpu_cores"` + RealtimeMemoryMB float64 `json:"realtime_memory_mb"` + RealtimeGPUUtilPercent float64 `json:"realtime_gpu_util_percent"` + RealtimeGPUMemoryMB float64 `json:"realtime_gpu_memory_mb"` + GPUDataAvailable bool `json:"gpu_data_available"` + MaxGPUHistoryPercent float64 `json:"max_gpu_history_percent"` + LastExpandAt *time.Time `json:"last_expand_at,omitempty"` + RecentHistory []UsageHistoryPoint `json:"recent_history,omitempty"` +} + +type ConstraintConfig struct { + PolicyVersion string + AlertThreshold float64 + MaxExpandRatio float64 + MaxExpandBytes int64 + MinPlatformReservedRatio float64 + MinPlatformReservedBytes int64 + ExpansionCooldown time.Duration + ForceFreezeWhenOverQuota bool +} + +func DefaultConstraintConfig() ConstraintConfig { + return ConstraintConfig{ + PolicyVersion: "storage-safety-v1", + AlertThreshold: 0.90, + MaxExpandRatio: 0.30, + MaxExpandBytes: 500 * 1024 * 1024 * 1024, + MinPlatformReservedRatio: 0.10, + MinPlatformReservedBytes: 200 * 1024 * 1024 * 1024, + ExpansionCooldown: 6 * time.Hour, + ForceFreezeWhenOverQuota: true, + } +} + +type ConstraintEvaluation struct { + PolicyVersion string `json:"policy_version"` + Adjusted bool `json:"adjusted"` + Blocked bool `json:"blocked"` + Violations []string `json:"violations"` + Adjustments []string `json:"adjustments"` +} + +type ReplayRecord struct { + JobID string `json:"job_id"` + Username string `json:"username"` + StoredAdjusted bool `json:"stored_adjusted"` + StoredBlocked bool `json:"stored_blocked"` + ReplayAdjusted bool `json:"replay_adjusted"` + ReplayBlocked bool `json:"replay_blocked"` + StoredAllowExpand bool `json:"stored_allow_expand"` + ReplayAllowExpand bool `json:"replay_allow_expand"` + StoredExpandBytes int64 `json:"stored_expand_bytes"` + ReplayExpandBytes int64 `json:"replay_expand_bytes"` + StoredFreeze bool `json:"stored_freeze"` + ReplayFreeze bool `json:"replay_freeze"` + Evaluation ConstraintEvaluation `json:"evaluation"` +} + +type ReplaySummary struct { + TotalCases int `json:"total_cases"` + ChangedCases int `json:"changed_cases"` + BlockedCases int `json:"blocked_cases"` + ClampedCases int `json:"clamped_cases"` + FreezeEscalations int `json:"freeze_escalations"` + PolicyVersion string `json:"policy_version"` + Records []ReplayRecord `json:"records,omitempty"` +} diff --git a/backend/pkg/storageindex/experiment.go b/backend/pkg/storageindex/experiment.go new file mode 100644 index 000000000..795de57f5 --- /dev/null +++ b/backend/pkg/storageindex/experiment.go @@ -0,0 +1,187 @@ +//nolint:gocritic,lll // Experiment helpers assemble shell probes and aggregate directory signatures. +package storageindex + +import ( + "context" + "fmt" + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" + + "github.com/raids-lab/crater/dao/model" + "github.com/raids-lab/crater/pkg/ceph" + "github.com/raids-lab/crater/pkg/config" +) + +type WorkspaceExperimentStats struct { + WorkspaceType model.StorageIndexWorkspaceType `json:"workspace_type"` + WorkspaceName string `json:"workspace_name"` + LogicalPath string `json:"logical_path"` + ActualPath string `json:"actual_path"` + TotalFileCount int64 `json:"total_file_count"` + TotalDirectoryCount int64 `json:"total_directory_count"` + FilteredFileCount int64 `json:"filtered_file_count"` + FilteredDirectoryCount int64 `json:"filtered_directory_count"` + TopLevelCandidateDirCount int `json:"top_level_candidate_dir_count"` + SelectedTopLevelDirCount int `json:"selected_top_level_dir_count"` + SkippedTopLevelDirCount int `json:"skipped_top_level_dir_count"` + SelectedTopLevelDirNames []string `json:"selected_top_level_dir_names,omitempty"` + SkippedTopLevelDirNames []string `json:"skipped_top_level_dir_names,omitempty"` +} + +func (s *Service) CollectWorkspaceExperimentStats( + ctx context.Context, + workspaceType model.StorageIndexWorkspaceType, + workspaceName string, +) (*WorkspaceExperimentStats, error) { + workspace, err := s.resolveWorkspace(ctx, workspaceType, workspaceName) + if err != nil { + return nil, err + } + + cfg := config.GetConfig() + prefixConfig := ceph.StoragePrefixConfig{ + User: cfg.Storage.Prefix.User, + Account: cfg.Storage.Prefix.Account, + Public: cfg.Storage.Prefix.Public, + } + + toolboxPod, err := ceph.FindCephToolboxPod(s.kubeClient, toolboxNamespace) + if err != nil { + return nil, fmt.Errorf("find ceph toolbox pod failed: %w", err) + } + + actualPath, err := ceph.ResolveCephFSPath( + s.kubeClient, + s.kubeConfig, + toolboxNamespace, + workspace.LogicalPath, + prefixConfig, + ) + if err != nil { + return nil, fmt.Errorf("resolve workspace path failed: %w", err) + } + + signatures, err := s.listImmediateSignatures( + toolboxPod, + "experiment-stats", + workspace.WorkspaceType, + workspace.WorkspaceName, + workspace.LogicalPath, + actualPath, + ) + if err != nil { + return nil, err + } + + selected, skipped := filterTopLevelSignaturesForModelCopyScan(signatures) + selectedDirNames, selectedDirPaths := collectTopLevelDirectories(selected) + skippedDirNames, _ := collectTopLevelDirectories(skipped) + + totalFiles, totalDirs, err := s.countDirectoryTree(toolboxPod, actualPath, false) + if err != nil { + return nil, err + } + + filteredFiles, filteredDirs, err := s.countDirectoryForest(toolboxPod, selectedDirPaths, true) + if err != nil { + return nil, err + } + + return &WorkspaceExperimentStats{ + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + LogicalPath: workspace.LogicalPath, + ActualPath: actualPath, + TotalFileCount: totalFiles, + TotalDirectoryCount: totalDirs, + FilteredFileCount: filteredFiles, + FilteredDirectoryCount: filteredDirs, + TopLevelCandidateDirCount: len(selectedDirNames) + len(skippedDirNames), + SelectedTopLevelDirCount: len(selectedDirNames), + SkippedTopLevelDirCount: len(skippedDirNames), + SelectedTopLevelDirNames: selectedDirNames, + SkippedTopLevelDirNames: skippedDirNames, + }, nil +} + +func collectTopLevelDirectories(items []topLevelSignature) ([]string, []string) { + names := make([]string, 0) + paths := make([]string, 0) + for _, item := range items { + if item.EntryType != model.StorageIndexEntryTypeDir { + continue + } + names = append(names, item.Name) + paths = append(paths, item.ActualPath) + } + return names, paths +} + +func (s *Service) countDirectoryTree( + toolboxPod *corev1.Pod, + actualPath string, + includeRootDir bool, +) (int64, int64, error) { + if strings.TrimSpace(actualPath) == "" { + return 0, 0, nil + } + return s.countDirectoryForest(toolboxPod, []string{actualPath}, includeRootDir) +} + +func (s *Service) countDirectoryForest( + toolboxPod *corev1.Pod, + actualPaths []string, + includeRootDir bool, +) (int64, int64, error) { + quoted := make([]string, 0, len(actualPaths)) + for _, actualPath := range actualPaths { + normalized := normalizeUnixPath(actualPath) + if normalized == "" { + continue + } + quoted = append(quoted, shellQuote(normalized)) + } + if len(quoted) == 0 { + return 0, 0, nil + } + + dirClause := "-mindepth 1 -type d -print" + if includeRootDir { + dirClause = "-type d -print" + } + + script := fmt.Sprintf( + `files=0; dirs=0; for root in %s; do if [ ! -d "$root" ]; then continue; fi; f=$(find "$root" -path '*/.snap' -prune -o -type f -print | wc -l | tr -d ' '); d=$(find "$root" -path '*/.snap' -prune -o %s | wc -l | tr -d ' '); files=$((files + f)); dirs=$((dirs + d)); done; printf '%%s%s%%s' "$files" "$dirs"`, + strings.Join(quoted, " "), + dirClause, + findFieldSeparator, + ) + + output, err := ceph.ExecInPod( + s.kubeClient, + s.kubeConfig, + toolboxPod, + []string{"sh", "-c", script}, + ) + if err != nil { + return 0, 0, fmt.Errorf("count directory forest failed: %w", err) + } + + fields := strings.Split(strings.TrimSpace(output), findFieldSeparator) + if len(fields) != 2 { + return 0, 0, fmt.Errorf("unexpected directory count output: %q", output) + } + + fileCount, err := strconv.ParseInt(strings.TrimSpace(fields[0]), 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("parse file count failed: %w", err) + } + dirCount, err := strconv.ParseInt(strings.TrimSpace(fields[1]), 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("parse directory count failed: %w", err) + } + + return fileCount, dirCount, nil +} diff --git a/backend/pkg/storageindex/service.go b/backend/pkg/storageindex/service.go new file mode 100644 index 000000000..7709033c4 --- /dev/null +++ b/backend/pkg/storageindex/service.go @@ -0,0 +1,6922 @@ +//nolint:dupl,funlen,gocritic,gocyclo,goconst,gosec,lll,mnd,unparam,unused // Metadata indexing orchestration intentionally keeps scanning, aggregation and redundancy detection in one service. +package storageindex + +import ( + "bytes" + "context" + "crypto/sha1" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "path" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/google/uuid" + "gorm.io/gorm" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + + "github.com/raids-lab/crater/dao/model" + "github.com/raids-lab/crater/dao/query" + "github.com/raids-lab/crater/pkg/ceph" + "github.com/raids-lab/crater/pkg/config" +) + +const ( + toolboxNamespace = "rook-ceph" + cephFSVolumeName = "cephfs" + insertBatchSize = 500 + defaultOverviewLimit = 10 + defaultRedundancyPageSize = 50 + minimumRedundantFileBytes = 10 * 1024 * 1024 + manualTriggerSource = "manual" + redundancyConfidenceHigh = "high" + redundancyConfidenceMedium = "medium" + hashAlgorithmSHA256 = "sha256" + hashAlgorithmSampledSHA256 = "sampled_sha256" + compareModeOptimized = "optimized" + compareModeFullHash = "full_hash" + verificationModeMetadata = "metadata" + verificationModeFileName = "file_name_size" + // Keep this within storage_index_redundancy_hits.verification_mode varchar(32). + verificationModeSafeTensorsHdrAndSampledSHA = "safetensors_hdr+sampled_sha256" + findFieldSeparator = "\x1f" + findRecordSeparator = "\x00" + scanProgressLogEveryRecord = 1000 + sampledHashSegmentBytes = 64 * 1024 + safetensorsHeaderMaxBytes = 16 * 1024 * 1024 +) + +var publicBaselineTopLevelAllowList = map[string]struct{}{ + "models": {}, + "dataset": {}, + "datasets": {}, +} + +// Only exclude top-level directories that are very unlikely to contain +// user-managed copies of public models. Keep this list conservative. +var definitelyNotPublicModelCopyTopLevelDirSet = map[string]struct{}{ + "conda": {}, + ".conda": {}, + ".bun": {}, + ".claude": {}, + ".codex": {}, + ".config": {}, + ".copilot": {}, + ".craft-versions": {}, + ".cursor-server": {}, + ".dotnet": {}, + ".gongfeng-copilot": {}, + ".ipython": {}, + ".java": {}, + ".jupyter": {}, + ".lingma": {}, + ".local": {}, + ".marscode": {}, + ".mcp-auth": {}, + ".nvm": {}, + ".opencode": {}, + ".pip": {}, + ".pki": {}, + ".ray": {}, + ".redhat": {}, + ".rest-client": {}, + ".ssh": {}, + ".subversion": {}, + ".trae": {}, + ".trae-aicc": {}, + ".trae-server": {}, + ".nv": {}, + ".zed_server": {}, + "miniconda3": {}, + "anaconda3": {}, + "mambaforge": {}, + ".mamba": {}, + "micromamba": {}, + "venv": {}, + ".venv": {}, + ".git": {}, + ".svn": {}, + ".hg": {}, + ".idea": {}, + ".codeverse": {}, + ".vscode": {}, + ".vscode-server": {}, + ".ipynb_checkpoints": {}, + ".npm": {}, + ".yarn": {}, + ".pnpm-store": {}, + ".cargo": {}, + ".m2": {}, + ".gradle": {}, + ".pytest_cache": {}, + ".mypy_cache": {}, + "__pycache__": {}, +} + +// Keep `.cache` itself as a top-level candidate root, but only retain a +// conservative allowlist of second-level cache subtrees that may store +// user-managed copies of public models. +var selectiveTopLevelSubtreeAllowLists = map[string]map[string]struct{}{ + ".cache": { + "huggingface": {}, + "modelscope": {}, + "torch": {}, + "transformers": {}, + }, +} + +// Recursively prune environment/runtime subtrees that are very unlikely to be +// useful for public model copy detection, even when they appear deep inside a +// user project directory. +var definitelyNotPublicModelCopyNestedDirSet = map[string]struct{}{ + "conda": {}, + ".conda": {}, + "miniconda3": {}, + "anaconda3": {}, + "mambaforge": {}, + ".mamba": {}, + "micromamba": {}, + "venv": {}, + ".venv": {}, + ".ipynb_checkpoints": {}, + ".pytest_cache": {}, + ".mypy_cache": {}, + "__pycache__": {}, +} + +type Service struct { + kubeClient kubernetes.Interface + kubeConfig *rest.Config +} + +type StartScanRequest struct { + WorkspaceType model.StorageIndexWorkspaceType + WorkspaceName string + TriggerSource string + ScanMode model.StorageIndexScanMode +} + +type WorkspaceOverview struct { + WorkspaceType model.StorageIndexWorkspaceType `json:"workspace_type"` + WorkspaceName string `json:"workspace_name"` + LogicalPath string `json:"logical_path"` + LastScanID string `json:"last_scan_id"` + LastScanStatus model.StorageIndexScanStatus `json:"last_scan_status"` + LastScanAt *time.Time `json:"last_scan_at,omitempty"` + EntryCount int64 `json:"entry_count"` + FileCount int64 `json:"file_count"` + DirectoryCount int64 `json:"directory_count"` + RedundancyCount int64 `json:"redundancy_count"` + RedundancyBytes int64 `json:"redundancy_bytes"` + TopDirectories []DirectorySummary `json:"top_directories"` + LargestFiles []FileSummary `json:"largest_files"` +} + +type DirectorySummary struct { + Path string `json:"path"` + Name string `json:"name"` + Depth int `json:"depth"` + FileCount int64 `json:"file_count"` + DirectoryCount int64 `json:"directory_count"` + TotalSizeBytes int64 `json:"total_size_bytes"` + IsTopLevel bool `json:"is_top_level"` +} + +type FileSummary struct { + Path string `json:"path"` + Name string `json:"name"` + SizeBytes int64 `json:"size_bytes"` + ModifiedAt *time.Time `json:"modified_at,omitempty"` +} + +type DirectoryCompareTiming struct { + ScanMs int64 `json:"scan_ms"` + PairingMs int64 `json:"pairing_ms"` + HeaderMs int64 `json:"header_ms"` + SampledHashMs int64 `json:"sampled_hash_ms"` + FullHashMs int64 `json:"full_hash_ms"` + TotalMs int64 `json:"total_ms"` +} + +type DirectoryCompareFileResult struct { + LeftRelativePath string `json:"left_relative_path"` + RightRelativePath string `json:"right_relative_path"` + FileName string `json:"file_name"` + SizeBytes int64 `json:"size_bytes"` + VerificationMode string `json:"verification_mode"` + Same bool `json:"same"` + Reason string `json:"reason,omitempty"` + HeaderMatched *bool `json:"header_matched,omitempty"` + SampledHashMatch *bool `json:"sampled_hash_match,omitempty"` +} + +type DirectoryCompareResult struct { + CompareType string `json:"compare_type"` + CompareMode string `json:"compare_mode"` + LeftPath string `json:"left_path"` + RightPath string `json:"right_path"` + Same bool `json:"same"` + LeftKeyFileCount int `json:"left_key_file_count"` + RightKeyFileCount int `json:"right_key_file_count"` + ExactMatchCount int `json:"exact_match_count"` + FallbackMatchCount int `json:"fallback_match_count"` + ComparedFileCount int `json:"compared_file_count"` + VerifiedFileCount int `json:"verified_file_count"` + MissingLeft []string `json:"missing_left"` + MissingRight []string `json:"missing_right"` + Files []DirectoryCompareFileResult `json:"files"` + Timing DirectoryCompareTiming `json:"timing"` +} + +type DirectoryCompareJobStatus string + +const ( + DirectoryCompareJobStatusPending DirectoryCompareJobStatus = "pending" + DirectoryCompareJobStatusRunning DirectoryCompareJobStatus = "running" + DirectoryCompareJobStatusDone DirectoryCompareJobStatus = "done" + DirectoryCompareJobStatusError DirectoryCompareJobStatus = "error" +) + +type DirectoryCompareJob struct { + mu sync.RWMutex + JobID string `json:"job_id"` + Status DirectoryCompareJobStatus `json:"status"` + LeftPath string `json:"left_path"` + RightPath string `json:"right_path"` + CompareType string `json:"compare_type"` + CompareMode string `json:"compare_mode"` + Result *DirectoryCompareResult `json:"result,omitempty"` + Error string `json:"error,omitempty"` + StartedAt *time.Time `json:"started_at,omitempty"` + FinishedAt *time.Time `json:"finished_at,omitempty"` +} + +type DirectoryCompareJobView struct { + JobID string `json:"job_id"` + Status DirectoryCompareJobStatus `json:"status"` + LeftPath string `json:"left_path"` + RightPath string `json:"right_path"` + CompareType string `json:"compare_type"` + CompareMode string `json:"compare_mode"` + Result *DirectoryCompareResult `json:"result,omitempty"` + Error string `json:"error,omitempty"` + StartedAt *time.Time `json:"started_at,omitempty"` + FinishedAt *time.Time `json:"finished_at,omitempty"` +} + +var directoryCompareJobs sync.Map + +type resolvedWorkspace struct { + WorkspaceType model.StorageIndexWorkspaceType + WorkspaceName string + LogicalPath string +} + +type topLevelSignature struct { + Name string + LogicalPath string + ParentLogicalPath string + ActualPath string + EntryType model.StorageIndexEntryType + SizeBytes int64 + ModifiedAt *time.Time + ChangedAt *time.Time + OwnerUID int64 + OwnerGID int64 + Mode string + LinkCount int64 +} + +type incrementalCollectResult struct { + SnapshotName string + MaterializedSnapshotName string + ScanRoot string + DiffMethod string + ChangedPathCount int64 + ChangedPrefixes []string + RemovedPrefixes []string + NewEntries []model.StorageIndexEntry + NewDirMetrics []model.StorageIndexDirectoryMetric + NewCandidates []model.StorageIndexCandidate + NewCandidateFiles []model.StorageIndexCandidateFile + NewHits []model.StorageIndexRedundancyHit +} + +type incrementalPlan struct { + RescanTargets []topLevelSignature + UpsertEntries []model.StorageIndexEntry + RemovedPrefixes []string + ComparedNodes int64 + PrunedDirs int64 + NewNodes int64 + UpdatedNodes int64 + RemovedNodes int64 + ReusedNodes int64 +} + +type publicResourceRoot struct { + Name string + LogicalPath string + Category string +} + +type publicBaselineBuildResult struct { + Roots []model.StorageIndexPublicRootBaseline + Files []model.StorageIndexPublicFileBaseline +} + +func (j *DirectoryCompareJob) setRunning(startedAt time.Time) { + j.mu.Lock() + defer j.mu.Unlock() + j.Status = DirectoryCompareJobStatusRunning + j.StartedAt = &startedAt +} + +func (j *DirectoryCompareJob) setDone(result *DirectoryCompareResult, finishedAt time.Time) { + j.mu.Lock() + defer j.mu.Unlock() + j.Status = DirectoryCompareJobStatusDone + j.Result = result + j.Error = "" + j.FinishedAt = &finishedAt +} + +func (j *DirectoryCompareJob) setError(message string, finishedAt time.Time) { + j.mu.Lock() + defer j.mu.Unlock() + j.Status = DirectoryCompareJobStatusError + j.Result = nil + j.Error = message + j.FinishedAt = &finishedAt +} + +func (j *DirectoryCompareJob) snapshot() *DirectoryCompareJobView { + j.mu.RLock() + defer j.mu.RUnlock() + + return &DirectoryCompareJobView{ + JobID: j.JobID, + Status: j.Status, + LeftPath: j.LeftPath, + RightPath: j.RightPath, + CompareType: j.CompareType, + CompareMode: j.CompareMode, + Result: j.Result, + Error: j.Error, + StartedAt: j.StartedAt, + FinishedAt: j.FinishedAt, + } +} + +func NewService(kubeClient kubernetes.Interface, kubeConfig *rest.Config) *Service { + return &Service{ + kubeClient: kubeClient, + kubeConfig: kubeConfig, + } +} + +func (s *Service) StartCompareDirectories( + _ context.Context, + leftPath string, + rightPath string, + compareType string, + compareMode string, +) (string, error) { + jobID := "cmp-" + uuid.NewString() + job := &DirectoryCompareJob{ + JobID: jobID, + Status: DirectoryCompareJobStatusPending, + LeftPath: normalizeCompareLogicalPath(leftPath), + RightPath: normalizeCompareLogicalPath(rightPath), + CompareType: normalizeDirectoryCompareType(compareType), + CompareMode: normalizeDirectoryCompareMode(compareMode), + } + directoryCompareJobs.Store(jobID, job) + + go func() { + startedAt := time.Now() + job.setRunning(startedAt) + + runCtx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + + result, err := s.CompareDirectories( + runCtx, + leftPath, + rightPath, + compareType, + compareMode, + ) + finishedAt := time.Now() + if err != nil { + job.setError(err.Error(), finishedAt) + return + } + job.setDone(result, finishedAt) + }() + + return jobID, nil +} + +func (s *Service) GetCompareDirectoryJob(jobID string) (*DirectoryCompareJobView, error) { + value, ok := directoryCompareJobs.Load(strings.TrimSpace(jobID)) + if !ok { + return nil, fmt.Errorf("compare job %s not found", jobID) + } + + job, ok := value.(*DirectoryCompareJob) + if !ok { + return nil, fmt.Errorf("compare job %s has invalid type", jobID) + } + + return job.snapshot(), nil +} + +func (s *Service) StartFullScan(ctx context.Context, req StartScanRequest) (string, error) { + workspace, err := s.resolveWorkspace(ctx, req.WorkspaceType, req.WorkspaceName) + if err != nil { + return "", err + } + + triggerSource := strings.TrimSpace(req.TriggerSource) + if triggerSource == "" { + triggerSource = manualTriggerSource + } + scanMode := req.ScanMode + if scanMode == "" { + scanMode = model.StorageIndexScanModeFull + } + + scanID := uuid.NewString() + baseScanID, _ := s.findLatestCompletedScanID(ctx, workspace) + job := &model.StorageIndexScanJob{ + ScanID: scanID, + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + LogicalPath: workspace.LogicalPath, + TriggerSource: triggerSource, + ScanMode: scanMode, + BaseScanID: baseScanID, + DiffMethod: "db_diff", + Status: model.StorageIndexScanStatusPending, + } + + if err := query.GetDB().WithContext(ctx).Create(job).Error; err != nil { + return "", fmt.Errorf("create metadata scan job failed: %w", err) + } + + klog.Infof("storageindex: 已创建扫描任务 scan_id=%s workspace_type=%s workspace_name=%s logical_path=%s", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, workspace.LogicalPath) + + go s.runFullScan(context.Background(), scanID, workspace) + + return scanID, nil +} + +func (s *Service) RunFullScanNow(ctx context.Context, req StartScanRequest) (*model.StorageIndexScanJob, error) { + workspace, err := s.resolveWorkspace(ctx, req.WorkspaceType, req.WorkspaceName) + if err != nil { + return nil, err + } + + triggerSource := strings.TrimSpace(req.TriggerSource) + if triggerSource == "" { + triggerSource = manualTriggerSource + } + scanMode := req.ScanMode + if scanMode == "" { + scanMode = model.StorageIndexScanModeFull + } + + scanID := uuid.NewString() + baseScanID, _ := s.findLatestCompletedScanID(ctx, workspace) + job := &model.StorageIndexScanJob{ + ScanID: scanID, + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + LogicalPath: workspace.LogicalPath, + TriggerSource: triggerSource, + ScanMode: scanMode, + BaseScanID: baseScanID, + DiffMethod: "db_diff", + Status: model.StorageIndexScanStatusPending, + } + if err := query.GetDB().WithContext(ctx).Create(job).Error; err != nil { + return nil, fmt.Errorf("create metadata scan job failed: %w", err) + } + + s.runFullScan(ctx, scanID, workspace) + return s.GetScanJob(ctx, scanID) +} + +func (s *Service) RefreshPublicBaseline(ctx context.Context) (*model.StorageIndexScanJob, error) { + workspace := resolvedWorkspace{ + WorkspaceType: model.StorageIndexWorkspaceTypePublic, + WorkspaceName: "public", + LogicalPath: "/public", + } + scanID := uuid.NewString() + startedAt := time.Now() + + job := &model.StorageIndexScanJob{ + ScanID: scanID, + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + LogicalPath: workspace.LogicalPath, + TriggerSource: "patrol_daily_public_baseline", + ScanMode: model.StorageIndexScanModeDailyRefresh, + DiffMethod: "business_registry_baseline", + Status: model.StorageIndexScanStatusRunning, + StartedAt: &startedAt, + } + if err := query.GetDB().WithContext(ctx).Create(job).Error; err != nil { + return nil, fmt.Errorf("create public baseline job failed: %w", err) + } + + count, err := s.rebuildPublicFileBaseline(ctx, scanID, workspace) + finishedAt := time.Now() + updates := map[string]any{ + "finished_at": finishedAt, + "latency_ms": finishedAt.Sub(startedAt).Milliseconds(), + } + if err != nil { + updates["status"] = model.StorageIndexScanStatusError + updates["error_message"] = err.Error() + _ = query.GetDB().WithContext(ctx). + Model(&model.StorageIndexScanJob{}). + Where("scan_id = ?", scanID). + Updates(updates).Error + return nil, err + } + + updates["status"] = model.StorageIndexScanStatusDone + updates["entry_count"] = count + updates["file_count"] = count + updates["directory_count"] = 0 + updates["total_size_bytes"] = 0 + updates["redundancy_count"] = 0 + updates["redundancy_bytes"] = 0 + updates["error_message"] = "" + if err := query.GetDB().WithContext(ctx). + Model(&model.StorageIndexScanJob{}). + Where("scan_id = ?", scanID). + Updates(updates).Error; err != nil { + return nil, fmt.Errorf("update public baseline job failed: %w", err) + } + + return s.GetScanJob(ctx, scanID) +} + +func (s *Service) RefreshAllUserWorkspaces(ctx context.Context) (map[string]any, error) { + type userRow struct { + Name string `gorm:"column:name"` + Space string `gorm:"column:space"` + } + var users []userRow + if err := query.GetDB().WithContext(ctx). + Raw("SELECT name, space FROM users WHERE deleted_at IS NULL AND space <> '' ORDER BY id ASC"). + Scan(&users).Error; err != nil { + return nil, fmt.Errorf("query user workspaces failed: %w", err) + } + + success := 0 + failed := 0 + results := make([]map[string]any, 0, len(users)) + for _, user := range users { + job, err := s.RunFullScanNow(ctx, StartScanRequest{ + WorkspaceType: model.StorageIndexWorkspaceTypeUser, + WorkspaceName: user.Name, + TriggerSource: "patrol_daily_user_refresh", + ScanMode: model.StorageIndexScanModeDailyRefresh, + }) + if err != nil { + failed++ + results = append(results, map[string]any{ + "user": user.Name, + "status": "error", + "error": err.Error(), + }) + klog.Warningf("storageindex: 用户空间每日刷新失败 user=%s err=%v", user.Name, err) + continue + } + success++ + results = append(results, map[string]any{ + "user": user.Name, + "status": job.Status, + "scan_id": job.ScanID, + }) + } + + return map[string]any{ + "total": len(users), + "success": success, + "failed": failed, + "items": results, + }, nil +} + +func (s *Service) GetScanJob(ctx context.Context, scanID string) (*model.StorageIndexScanJob, error) { + var job model.StorageIndexScanJob + if err := query.GetDB().WithContext(ctx). + Where("scan_id = ?", scanID). + First(&job).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, fmt.Errorf("metadata scan job %s not found", scanID) + } + return nil, fmt.Errorf("query metadata scan job failed: %w", err) + } + return &job, nil +} + +func (s *Service) GetWorkspaceOverview( + ctx context.Context, + workspaceType model.StorageIndexWorkspaceType, + workspaceName string, +) (*WorkspaceOverview, error) { + workspace, err := s.resolveWorkspace(ctx, workspaceType, workspaceName) + if err != nil { + return nil, err + } + + var latestJob model.StorageIndexScanJob + if err := query.GetDB().WithContext(ctx). + Where("workspace_type = ? AND workspace_name = ?", workspace.WorkspaceType, workspace.WorkspaceName). + // Prefer the newest created scan job so an in-flight run remains visible + // while directory metrics and candidate verification are still being built. + Order("created_at DESC, updated_at DESC, id DESC"). + First(&latestJob).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return &WorkspaceOverview{ + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + LogicalPath: workspace.LogicalPath, + LastScanID: "", + LastScanStatus: "", + LastScanAt: nil, + EntryCount: 0, + FileCount: 0, + DirectoryCount: 0, + RedundancyCount: 0, + RedundancyBytes: 0, + TopDirectories: []DirectorySummary{}, + LargestFiles: []FileSummary{}, + }, nil + } + return nil, fmt.Errorf("query latest metadata scan job failed: %w", err) + } + lastScanAt := latestJob.FinishedAt + if lastScanAt == nil { + lastScanAt = latestJob.StartedAt + } + + var topDirectoryRows []model.StorageIndexDirectoryMetric + if err := query.GetDB().WithContext(ctx). + Where("workspace_type = ? AND workspace_name = ? AND path <> ?", workspace.WorkspaceType, workspace.WorkspaceName, workspace.LogicalPath). + Order("total_size_bytes DESC, file_count DESC"). + Limit(defaultOverviewLimit). + Find(&topDirectoryRows).Error; err != nil { + return nil, fmt.Errorf("query directory overview failed: %w", err) + } + + var largestFiles []model.StorageIndexCandidateFile + if err := query.GetDB().WithContext(ctx). + Where("workspace_type = ? AND workspace_name = ?", workspace.WorkspaceType, workspace.WorkspaceName). + Order("size_bytes DESC"). + Limit(defaultOverviewLimit). + Find(&largestFiles).Error; err != nil { + return nil, fmt.Errorf("query largest candidate files failed: %w", err) + } + + var redundancySummary struct { + Count int64 `gorm:"column:count"` + Bytes int64 `gorm:"column:bytes"` + } + if err := query.GetDB().WithContext(ctx). + Table(model.StorageIndexRedundancyHit{}.TableName()). + Select("COUNT(*) AS count, COALESCE(SUM(estimated_bytes), 0) AS bytes"). + Where("workspace_type = ? AND workspace_name = ?", workspace.WorkspaceType, workspace.WorkspaceName). + Scan(&redundancySummary).Error; err != nil { + return nil, fmt.Errorf("query redundancy summary failed: %w", err) + } + + overview := &WorkspaceOverview{ + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + LogicalPath: workspace.LogicalPath, + LastScanID: latestJob.ScanID, + LastScanStatus: latestJob.Status, + LastScanAt: lastScanAt, + EntryCount: latestJob.EntryCount, + FileCount: latestJob.FileCount, + DirectoryCount: latestJob.DirectoryCount, + RedundancyCount: redundancySummary.Count, + RedundancyBytes: redundancySummary.Bytes, + TopDirectories: make([]DirectorySummary, 0, len(topDirectoryRows)), + LargestFiles: make([]FileSummary, 0, len(largestFiles)), + } + + for _, item := range topDirectoryRows { + overview.TopDirectories = append(overview.TopDirectories, DirectorySummary{ + Path: item.Path, + Name: item.Name, + Depth: item.Depth, + FileCount: item.FileCount, + DirectoryCount: item.DirectoryCount, + TotalSizeBytes: item.TotalSizeBytes, + IsTopLevel: item.IsTopLevel, + }) + } + + for _, item := range largestFiles { + overview.LargestFiles = append(overview.LargestFiles, FileSummary{ + Path: item.FilePath, + Name: item.FileName, + SizeBytes: item.SizeBytes, + ModifiedAt: nil, + }) + } + + return overview, nil +} + +func (s *Service) ListRedundancyHits( + ctx context.Context, + workspaceType model.StorageIndexWorkspaceType, + workspaceName string, + page, + pageSize int, +) ([]model.StorageIndexRedundancyHit, int64, error) { + workspace, err := s.resolveWorkspace(ctx, workspaceType, workspaceName) + if err != nil { + return nil, 0, err + } + + if page < 1 { + page = 1 + } + if pageSize < 1 { + pageSize = defaultRedundancyPageSize + } + + db := query.GetDB().WithContext(ctx) + var total int64 + if err := db.Model(&model.StorageIndexRedundancyHit{}). + Where("workspace_type = ? AND workspace_name = ?", workspace.WorkspaceType, workspace.WorkspaceName). + Count(&total).Error; err != nil { + return nil, 0, fmt.Errorf("count redundancy hits failed: %w", err) + } + + var hits []model.StorageIndexRedundancyHit + if err := db. + Where("workspace_type = ? AND workspace_name = ?", workspace.WorkspaceType, workspace.WorkspaceName). + Order("estimated_bytes DESC, id DESC"). + Offset((page - 1) * pageSize). + Limit(pageSize). + Find(&hits).Error; err != nil { + return nil, 0, fmt.Errorf("list redundancy hits failed: %w", err) + } + + return hits, total, nil +} + +func (s *Service) ListCandidates( + ctx context.Context, + workspaceType model.StorageIndexWorkspaceType, + workspaceName string, + page int, + pageSize int, +) ([]model.StorageIndexCandidate, int64, error) { + workspace, err := s.resolveWorkspace(ctx, workspaceType, workspaceName) + if err != nil { + return nil, 0, err + } + + if page < 1 { + page = 1 + } + if pageSize < 1 { + pageSize = defaultRedundancyPageSize + } + + db := query.GetDB().WithContext(ctx). + Model(&model.StorageIndexCandidate{}). + Where("workspace_type = ? AND workspace_name = ?", workspace.WorkspaceType, workspace.WorkspaceName) + + var total int64 + if err := db.Count(&total).Error; err != nil { + return nil, 0, fmt.Errorf("count candidates failed: %w", err) + } + + var items []model.StorageIndexCandidate + if err := db. + Order("CASE status WHEN 'verified' THEN 0 WHEN 'suspected' THEN 1 ELSE 2 END ASC"). + Order("candidate_score DESC, target_path ASC"). + Offset((page - 1) * pageSize). + Limit(pageSize). + Find(&items).Error; err != nil { + return nil, 0, fmt.Errorf("list candidates failed: %w", err) + } + + return items, total, nil +} + +func (s *Service) ListCandidateFiles( + ctx context.Context, + workspaceType model.StorageIndexWorkspaceType, + workspaceName string, + candidatePath string, + page int, + pageSize int, +) ([]model.StorageIndexCandidateFile, int64, error) { + workspace, err := s.resolveWorkspace(ctx, workspaceType, workspaceName) + if err != nil { + return nil, 0, err + } + candidatePath = normalizeUnixPath(candidatePath) + if candidatePath == "" { + return nil, 0, fmt.Errorf("candidate path cannot be empty") + } + + if page < 1 { + page = 1 + } + if pageSize < 1 { + pageSize = 200 + } + + db := query.GetDB().WithContext(ctx). + Model(&model.StorageIndexCandidateFile{}). + Where( + "workspace_type = ? AND workspace_name = ? AND candidate_path = ?", + workspace.WorkspaceType, + workspace.WorkspaceName, + candidatePath, + ) + + var total int64 + if err := db.Count(&total).Error; err != nil { + return nil, 0, fmt.Errorf("count candidate files failed: %w", err) + } + + var items []model.StorageIndexCandidateFile + if err := db. + Order("relative_path ASC, file_name ASC"). + Offset((page - 1) * pageSize). + Limit(pageSize). + Find(&items).Error; err != nil { + return nil, 0, fmt.Errorf("list candidate files failed: %w", err) + } + + return items, total, nil +} + +func (s *Service) CompareDirectories( + _ context.Context, + leftPath string, + rightPath string, + compareType string, + compareMode string, +) (*DirectoryCompareResult, error) { + startedAt := time.Now() + leftPath = normalizeCompareLogicalPath(leftPath) + rightPath = normalizeCompareLogicalPath(rightPath) + if leftPath == "" || rightPath == "" { + return nil, fmt.Errorf("compare paths cannot be empty") + } + + cfg := config.GetConfig() + prefixConfig := ceph.StoragePrefixConfig{ + User: cfg.Storage.Prefix.User, + Account: cfg.Storage.Prefix.Account, + Public: cfg.Storage.Prefix.Public, + } + + toolboxPod, err := ceph.FindCephToolboxPod(s.kubeClient, toolboxNamespace) + if err != nil { + return nil, fmt.Errorf("find ceph toolbox pod failed: %w", err) + } + + leftActual, err := ceph.ResolveCephFSPath(s.kubeClient, s.kubeConfig, toolboxNamespace, leftPath, prefixConfig) + if err != nil { + return nil, fmt.Errorf("resolve left path failed: %w", err) + } + rightActual, err := ceph.ResolveCephFSPath(s.kubeClient, s.kubeConfig, toolboxNamespace, rightPath, prefixConfig) + if err != nil { + return nil, fmt.Errorf("resolve right path failed: %w", err) + } + + scanStartedAt := time.Now() + resolvedCompareType, leftFiles, rightFiles, err := s.scanFilesForDirectoryCompare(toolboxPod, leftActual, rightActual, compareType) + if err != nil { + return nil, err + } + resolvedCompareMode := normalizeDirectoryCompareMode(compareMode) + + result := &DirectoryCompareResult{ + CompareType: resolvedCompareType, + CompareMode: resolvedCompareMode, + LeftPath: leftPath, + RightPath: rightPath, + LeftKeyFileCount: len(leftFiles), + RightKeyFileCount: len(rightFiles), + MissingLeft: make([]string, 0), + MissingRight: make([]string, 0), + Files: make([]DirectoryCompareFileResult, 0), + } + result.Timing.ScanMs = time.Since(scanStartedAt).Milliseconds() + + pairingStartedAt := time.Now() + var ( + pairs []struct{ left, right candidateFileProbe } + exactMatchCount int + fallbackMatchCount int + missingLeft []string + missingRight []string + ) + if resolvedCompareType == "dataset" { + pairs, exactMatchCount, fallbackMatchCount, missingLeft, missingRight = pairDirectoryFilesByNameAndSize(leftFiles, rightFiles) + } else { + pairs, exactMatchCount, fallbackMatchCount, missingLeft, missingRight = pairCandidateFilesForComparison(leftFiles, rightFiles) + } + result.ExactMatchCount = exactMatchCount + result.FallbackMatchCount = fallbackMatchCount + result.MissingLeft = missingLeft + result.MissingRight = missingRight + result.Timing.PairingMs = time.Since(pairingStartedAt).Milliseconds() + + headerStartedAt := time.Now() + type hashPair struct { + left candidateFileProbe + right candidateFileProbe + file *DirectoryCompareFileResult + } + hashPairs := make([]hashPair, 0, len(pairs)) + for _, pair := range pairs { + fileResult := DirectoryCompareFileResult{ + LeftRelativePath: pair.left.RelativePath, + RightRelativePath: pair.right.RelativePath, + FileName: pair.left.FileName, + SizeBytes: pair.left.SizeBytes, + VerificationMode: verificationModeFileName, + Same: false, + } + if resolvedCompareType == "dataset" { + fileResult.Same = true + result.VerifiedFileCount++ + result.Files = append(result.Files, fileResult) + continue + } + if resolvedCompareMode == compareModeOptimized && + isSafeTensorsFile(pair.left.FileName) && + isSafeTensorsFile(pair.right.FileName) { + headerMatched, headerErr := s.compareSafetensorsHeaders(toolboxPod, pair.left.ActualPath, pair.right.ActualPath) + fileResult.HeaderMatched = &headerMatched + if headerErr != nil { + fileResult.Reason = "safetensors_header_error" + result.Files = append(result.Files, fileResult) + continue + } + if !headerMatched { + fileResult.Reason = "safetensors_header_mismatch" + result.Files = append(result.Files, fileResult) + continue + } + fileResult.VerificationMode = verificationModeSafeTensorsHdrAndSampledSHA + } + if fileResult.VerificationMode == verificationModeFileName && resolvedCompareMode == compareModeOptimized { + fileResult.VerificationMode = hashAlgorithmSampledSHA256 + } + if fileResult.VerificationMode == verificationModeFileName && resolvedCompareMode == compareModeFullHash { + fileResult.VerificationMode = hashAlgorithmSHA256 + } + result.Files = append(result.Files, fileResult) + hashPairs = append(hashPairs, hashPair{ + left: pair.left, + right: pair.right, + file: &result.Files[len(result.Files)-1], + }) + } + result.Timing.HeaderMs = time.Since(headerStartedAt).Milliseconds() + + if resolvedCompareType == "dataset" { + result.ComparedFileCount = len(result.Files) + result.Same = len(result.MissingLeft) == 0 && + len(result.MissingRight) == 0 && + result.ComparedFileCount > 0 && + result.VerifiedFileCount == result.ComparedFileCount + result.Timing.TotalMs = time.Since(startedAt).Milliseconds() + return result, nil + } + + hashStartedAt := time.Now() + leftActualPaths := make([]string, 0, len(hashPairs)) + rightActualPaths := make([]string, 0, len(hashPairs)) + leftSizes := make(map[string]int64, len(hashPairs)) + rightSizes := make(map[string]int64, len(hashPairs)) + for _, pair := range hashPairs { + leftActualPaths = append(leftActualPaths, pair.left.ActualPath) + rightActualPaths = append(rightActualPaths, pair.right.ActualPath) + leftSizes[pair.left.ActualPath] = pair.left.SizeBytes + rightSizes[pair.right.ActualPath] = pair.right.SizeBytes + } + + leftHashes := map[string]string{} + rightHashes := map[string]string{} + if len(hashPairs) > 0 { + switch resolvedCompareMode { + case compareModeFullHash: + leftHashes, err = s.computeActualFileFullHashesBatch(toolboxPod, leftActualPaths) + if err != nil { + return nil, fmt.Errorf("compute left full hashes failed: %w", err) + } + rightHashes, err = s.computeActualFileFullHashesBatch(toolboxPod, rightActualPaths) + if err != nil { + return nil, fmt.Errorf("compute right full hashes failed: %w", err) + } + default: + leftHashes, err = s.computeActualFileHashesBatchWithSizes(toolboxPod, leftActualPaths, leftSizes) + if err != nil { + return nil, fmt.Errorf("compute left sampled hashes failed: %w", err) + } + rightHashes, err = s.computeActualFileHashesBatchWithSizes(toolboxPod, rightActualPaths, rightSizes) + if err != nil { + return nil, fmt.Errorf("compute right sampled hashes failed: %w", err) + } + } + } + for _, pair := range hashPairs { + leftHash := leftHashes[pair.left.ActualPath] + rightHash := rightHashes[pair.right.ActualPath] + matched := leftHash != "" && rightHash != "" && leftHash == rightHash + pair.file.SampledHashMatch = &matched + pair.file.Same = matched + if !matched { + pair.file.Reason = "sampled_hash_mismatch" + continue + } + result.VerifiedFileCount++ + } + if resolvedCompareMode == compareModeFullHash { + result.Timing.FullHashMs = time.Since(hashStartedAt).Milliseconds() + } else { + result.Timing.SampledHashMs = time.Since(hashStartedAt).Milliseconds() + } + + result.ComparedFileCount = len(result.Files) + result.Same = len(result.MissingLeft) == 0 && + len(result.MissingRight) == 0 && + result.ComparedFileCount > 0 && + result.VerifiedFileCount == result.ComparedFileCount + result.Timing.TotalMs = time.Since(startedAt).Milliseconds() + return result, nil +} + +func (s *Service) runFullScan(ctx context.Context, scanID string, workspace resolvedWorkspace) { + startedAt := time.Now() + db := query.GetDB().WithContext(ctx) + + if err := db.Model(&model.StorageIndexScanJob{}). + Where("scan_id = ?", scanID). + Updates(map[string]any{ + "status": model.StorageIndexScanStatusRunning, + "started_at": startedAt, + }).Error; err != nil { + klog.Errorf("storageindex: 标记扫描任务运行中失败 scan_id=%s err=%v", scanID, err) + } + + currentJob, _ := s.getScanJobByID(ctx, scanID) + baseScanID := "" + scanMode := model.StorageIndexScanModeFull + if currentJob != nil { + baseScanID = currentJob.BaseScanID + if currentJob.ScanMode != "" { + scanMode = currentJob.ScanMode + } + } + + klog.Infof( + "storageindex: 开始扫描 scan_id=%s workspace_type=%s workspace_name=%s scan_mode=%s base_scan_id=%s", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, scanMode, baseScanID, + ) + + if scanMode == model.StorageIndexScanModeDailyRefresh && baseScanID != "" { + klog.Infof( + "storageindex: 尝试执行增量扫描 scan_id=%s workspace_type=%s workspace_name=%s base_scan_id=%s", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, baseScanID, + ) + ok, err := s.runIncrementalScan(ctx, scanID, workspace, baseScanID, startedAt) + if err != nil { + klog.Warningf( + "storageindex: 增量扫描失败，回退为全量扫描 scan_id=%s workspace_type=%s workspace_name=%s base_scan_id=%s err=%v", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, baseScanID, err, + ) + } + if ok { + return + } + } + + if scanMode == model.StorageIndexScanModeDailyRefresh && baseScanID == "" { + klog.Infof( + "storageindex: 没有可用基线，改为全量扫描 scan_id=%s workspace_type=%s workspace_name=%s", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, + ) + } + + if baseScanID == "" { + clearedRows, err := cleanupWorkspaceStateBeforeInitialScan(db, workspace) + if err != nil { + s.markScanError(ctx, scanID, startedAt, fmt.Errorf("cleanup workspace state before initial scan failed: %w", err)) + return + } + klog.Infof( + "storageindex: 首次扫描前已清理残留数据 scan_id=%s workspace_type=%s workspace_name=%s cleared_rows=%d", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, clearedRows, + ) + } + + if err := s.runFullSnapshotScan(ctx, scanID, workspace, baseScanID, startedAt); err != nil { + s.markScanError(ctx, scanID, startedAt, err) + } +} + +func (s *Service) markScanError(ctx context.Context, scanID string, startedAt time.Time, runErr error) { + finishedAt := time.Now() + if err := query.GetDB().WithContext(ctx). + Model(&model.StorageIndexScanJob{}). + Where("scan_id = ?", scanID). + Updates(map[string]any{ + "status": model.StorageIndexScanStatusError, + "error_message": runErr.Error(), + "finished_at": finishedAt, + "latency_ms": finishedAt.Sub(startedAt).Milliseconds(), + }).Error; err != nil { + klog.Errorf("storageindex: 标记扫描任务失败失败 scan_id=%s err=%v original=%v", scanID, err, runErr) + } +} + +func (s *Service) runFullSnapshotScan( + ctx context.Context, + scanID string, + workspace resolvedWorkspace, + baseScanID string, + startedAt time.Time, +) error { + snapshotName, materializedSnapshotName, scanRoot, entries, dirMetrics, changedPathCount, hits, totalSize, err := s.collectWorkspaceSnapshot(ctx, scanID, workspace, baseScanID) + if err != nil { + return err + } + + fileCount := int64(0) + dirCount := int64(0) + for _, entry := range entries { + switch entry.EntryType { + case model.StorageIndexEntryTypeFile: + fileCount++ + case model.StorageIndexEntryTypeDir: + dirCount++ + } + } + + redundancyBytes := int64(0) + for _, hit := range hits { + redundancyBytes += hit.EstimatedBytes + } + klog.Infof( + "storageindex: 开始保存全量扫描结果 scan_id=%s workspace_type=%s workspace_name=%s entries=%d metrics=%d hits=%d", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, len(entries), len(dirMetrics), len(hits), + ) + + db := query.GetDB().WithContext(ctx) + err = db.Transaction(func(tx *gorm.DB) error { + if err := tx.Where("workspace_type = ? AND workspace_name = ?", workspace.WorkspaceType, workspace.WorkspaceName). + Delete(&model.StorageIndexEntry{}).Error; err != nil { + return err + } + if err := tx.Where("workspace_type = ? AND workspace_name = ?", workspace.WorkspaceType, workspace.WorkspaceName). + Delete(&model.StorageIndexDirectoryMetric{}).Error; err != nil { + return err + } + if err := tx.Where("workspace_type = ? AND workspace_name = ?", workspace.WorkspaceType, workspace.WorkspaceName). + Delete(&model.StorageIndexRedundancyHit{}).Error; err != nil { + return err + } + + if len(entries) > 0 { + if err := insertEntriesInChunks(tx, scanID, workspace, entries, insertBatchSize); err != nil { + return err + } + } + if len(dirMetrics) > 0 { + if err := insertDirectoryMetricsInChunks(tx, scanID, workspace, dirMetrics, insertBatchSize); err != nil { + return err + } + } + if len(hits) > 0 { + if err := insertRedundancyHitsInChunks(tx, scanID, workspace, hits, insertBatchSize); err != nil { + return err + } + } + + return tx.Model(&model.StorageIndexScanJob{}). + Where("scan_id = ?", scanID). + Updates(map[string]any{ + "snapshot_name": snapshotName, + "materialized_snapshot_name": materializedSnapshotName, + "scan_root": scanRoot, + "status": model.StorageIndexScanStatusRunning, + "entry_count": len(entries), + "file_count": fileCount, + "directory_count": dirCount, + "total_size_bytes": totalSize, + "base_scan_id": baseScanID, + "diff_method": "db_diff", + "changed_path_count": changedPathCount, + "redundancy_count": len(hits), + "redundancy_bytes": redundancyBytes, + "finished_at": nil, + "latency_ms": 0, + "error_message": "", + }).Error + }) + if err != nil { + return fmt.Errorf("persist full metadata scan result failed: %w", err) + } + + if err := s.rebuildCandidates(ctx, scanID, workspace); err != nil { + return fmt.Errorf("rebuild candidates after full scan failed: %w", err) + } + if workspace.WorkspaceType == model.StorageIndexWorkspaceTypePublic { + if _, err := s.rebuildPublicFileBaseline(ctx, scanID, workspace); err != nil { + return fmt.Errorf("rebuild public file baseline failed: %w", err) + } + } + + finalCounts, err := s.queryWorkspaceCounts(ctx, workspace) + if err != nil { + return fmt.Errorf("query final workspace counts failed: %w", err) + } + finishedAt := time.Now() + if err := query.GetDB().WithContext(ctx). + Model(&model.StorageIndexScanJob{}). + Where("scan_id = ?", scanID). + Updates(map[string]any{ + "status": model.StorageIndexScanStatusDone, + "entry_count": finalCounts.EntryCount, + "file_count": finalCounts.FileCount, + "directory_count": finalCounts.DirectoryCount, + "total_size_bytes": finalCounts.TotalSizeBytes, + "redundancy_count": finalCounts.RedundancyCount, + "redundancy_bytes": finalCounts.RedundancyBytes, + "finished_at": finishedAt, + "latency_ms": finishedAt.Sub(startedAt).Milliseconds(), + }).Error; err != nil { + return fmt.Errorf("finalize full metadata scan job failed: %w", err) + } + + klog.Infof( + "storageindex: 全量扫描完成 scan_id=%s workspace_type=%s workspace_name=%s mode=full entries=%d dirs=%d files=%d changed_paths=%d redundancy_hits=%d redundancy_bytes=%d", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, + finalCounts.EntryCount, finalCounts.DirectoryCount, finalCounts.FileCount, changedPathCount, finalCounts.RedundancyCount, finalCounts.RedundancyBytes, + ) + return nil +} + +func (s *Service) runIncrementalScan( + ctx context.Context, + scanID string, + workspace resolvedWorkspace, + baseScanID string, + startedAt time.Time, +) (bool, error) { + result, err := s.collectWorkspaceIncremental(ctx, scanID, workspace, baseScanID) + if err != nil { + return false, err + } + + db := query.GetDB().WithContext(ctx) + finalCounts := workspaceCounts{} + err = db.Transaction(func(tx *gorm.DB) error { + prefixesToDelete := append([]string{}, result.ChangedPrefixes...) + prefixesToDelete = append(prefixesToDelete, result.RemovedPrefixes...) + + for _, prefix := range prefixesToDelete { + if err := deleteWorkspacePathPrefix(tx, workspace, prefix); err != nil { + return err + } + } + for _, entry := range result.NewEntries { + if isCoveredByPrefixes(entry.LogicalPath, result.ChangedPrefixes) { + continue + } + if err := deleteWorkspaceExactPath(tx, workspace, entry.LogicalPath); err != nil { + return err + } + } + + if len(result.NewEntries) > 0 { + if err := insertEntriesInChunks(tx, scanID, workspace, result.NewEntries, insertBatchSize); err != nil { + return err + } + } + if len(result.NewDirMetrics) > 0 { + if err := insertDirectoryMetricsInChunks(tx, scanID, workspace, result.NewDirMetrics, insertBatchSize); err != nil { + return err + } + } + if err := rebuildWorkspaceRootMetric(tx, workspace, scanID); err != nil { + return err + } + + affectedMetricPaths := collectAffectedMetricPaths(workspace.LogicalPath, result.ChangedPrefixes, result.RemovedPrefixes, result.NewEntries) + for _, metricPath := range affectedMetricPaths { + if isCoveredByPrefixes(metricPath, result.ChangedPrefixes) { + continue + } + if err := deleteWorkspaceMetricExactPath(tx, workspace, metricPath); err != nil { + return err + } + if err := rebuildDirectoryMetric(tx, workspace, scanID, metricPath); err != nil { + return err + } + } + + counts, err := s.queryWorkspaceCountsWithDB(tx, workspace) + if err != nil { + return err + } + finalCounts = counts + return tx.Model(&model.StorageIndexScanJob{}). + Where("scan_id = ?", scanID). + Updates(map[string]any{ + "snapshot_name": result.SnapshotName, + "materialized_snapshot_name": result.MaterializedSnapshotName, + "scan_root": result.ScanRoot, + "status": model.StorageIndexScanStatusRunning, + "entry_count": counts.EntryCount, + "file_count": counts.FileCount, + "directory_count": counts.DirectoryCount, + "total_size_bytes": counts.TotalSizeBytes, + "base_scan_id": baseScanID, + "diff_method": result.DiffMethod, + "changed_path_count": result.ChangedPathCount, + "redundancy_count": counts.RedundancyCount, + "redundancy_bytes": counts.RedundancyBytes, + "finished_at": nil, + "latency_ms": 0, + "error_message": "", + }).Error + }) + if err != nil { + return false, fmt.Errorf("persist incremental metadata scan result failed: %w", err) + } + + if err := s.refreshIncrementalDerivedState(ctx, scanID, workspace, result); err != nil { + return false, fmt.Errorf("refresh incremental derived state failed: %w", err) + } + if workspace.WorkspaceType == model.StorageIndexWorkspaceTypePublic { + if _, err := s.rebuildPublicFileBaseline(ctx, scanID, workspace); err != nil { + return false, fmt.Errorf("rebuild public file baseline failed: %w", err) + } + } + + finalCounts, err = s.queryWorkspaceCounts(ctx, workspace) + if err != nil { + return false, fmt.Errorf("query final workspace counts failed: %w", err) + } + finishedAt := time.Now() + if err := query.GetDB().WithContext(ctx). + Model(&model.StorageIndexScanJob{}). + Where("scan_id = ?", scanID). + Updates(map[string]any{ + "status": model.StorageIndexScanStatusDone, + "entry_count": finalCounts.EntryCount, + "file_count": finalCounts.FileCount, + "directory_count": finalCounts.DirectoryCount, + "total_size_bytes": finalCounts.TotalSizeBytes, + "redundancy_count": finalCounts.RedundancyCount, + "redundancy_bytes": finalCounts.RedundancyBytes, + "finished_at": finishedAt, + "latency_ms": finishedAt.Sub(startedAt).Milliseconds(), + }).Error; err != nil { + return false, fmt.Errorf("finalize incremental metadata scan job failed: %w", err) + } + + klog.Infof( + "storageindex: 增量扫描完成 scan_id=%s workspace_type=%s workspace_name=%s mode=incremental diff_method=%s changed_paths=%d rescan_targets=%d upsert_entries=%d removed_prefixes=%d entry_count=%d file_count=%d dir_count=%d redundancy_hits=%d redundancy_bytes=%d", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, result.DiffMethod, result.ChangedPathCount, + len(result.ChangedPrefixes), len(result.NewEntries), len(result.RemovedPrefixes), + finalCounts.EntryCount, finalCounts.FileCount, finalCounts.DirectoryCount, finalCounts.RedundancyCount, finalCounts.RedundancyBytes, + ) + return true, nil +} + +func (s *Service) collectWorkspaceSnapshot( + ctx context.Context, + scanID string, + workspace resolvedWorkspace, + baseScanID string, +) (string, string, string, []model.StorageIndexEntry, []model.StorageIndexDirectoryMetric, int64, []model.StorageIndexRedundancyHit, int64, error) { + cfg := config.GetConfig() + prefixConfig := ceph.StoragePrefixConfig{ + User: cfg.Storage.Prefix.User, + Account: cfg.Storage.Prefix.Account, + Public: cfg.Storage.Prefix.Public, + } + + toolboxPod, err := ceph.FindCephToolboxPod(s.kubeClient, toolboxNamespace) + if err != nil { + return "", "", "", nil, nil, 0, nil, 0, fmt.Errorf("find ceph toolbox pod failed: %w", err) + } + + rootPath, err := ceph.ResolveCephFSPath( + s.kubeClient, + s.kubeConfig, + toolboxNamespace, + workspace.LogicalPath, + prefixConfig, + ) + if err != nil { + return "", "", "", nil, nil, 0, nil, 0, fmt.Errorf("resolve workspace path failed: %w", err) + } + + subvolumeRoot, err := ceph.GetCephMountRoot(s.kubeClient, s.kubeConfig, toolboxNamespace) + if err != nil { + return "", "", "", nil, nil, 0, nil, 0, fmt.Errorf("resolve ceph subvolume root failed: %w", err) + } + + klog.Infof( + "storageindex: 正在准备扫描路径 scan_id=%s workspace_type=%s workspace_name=%s root_path=%s subvolume_root=%s", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, rootPath, subvolumeRoot, + ) + + scanPath, snapshotName, materializedSnapshotName, cleanup, err := s.prepareScanPath(toolboxPod, subvolumeRoot, rootPath, scanID) + if err != nil { + return "", "", "", nil, nil, 0, nil, 0, err + } + + klog.Infof( + "storageindex: 开始扫描工作空间条目 scan_id=%s workspace_type=%s workspace_name=%s scan_path=%s snapshot_name=%s", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, scanPath, snapshotName, + ) + + if workspace.WorkspaceType == model.StorageIndexWorkspaceTypePublic { + return s.collectPublicWorkspaceSnapshot(ctx, toolboxPod, prefixConfig, scanID, workspace, scanPath, snapshotName, materializedSnapshotName, baseScanID) + } + if shouldApplyTopLevelModelCopyPrefilter(workspace) { + return s.collectFilteredUserWorkspaceSnapshot(ctx, toolboxPod, prefixConfig, scanID, workspace, scanPath, snapshotName, materializedSnapshotName, baseScanID) + } + + entries, err := s.scanWorkspaceEntries(toolboxPod, scanID, workspace, scanPath) + if err != nil { + if cleanup != nil { + cleanup() + } + return snapshotName, materializedSnapshotName, scanPath, nil, nil, 0, nil, 0, err + } + klog.Infof( + "storageindex: 条目解析完成 scan_id=%s workspace_type=%s workspace_name=%s entry_count=%d", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, len(entries), + ) + + dirMetrics, totalSize := buildDirectoryMetrics(scanID, workspace, entries) + klog.Infof( + "storageindex: 目录聚合完成 scan_id=%s workspace_type=%s workspace_name=%s metric_count=%d total_size_bytes=%d", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, len(dirMetrics), totalSize, + ) + changedPathCount, err := s.applyGrowthFromPreviousScan(ctx, workspace, baseScanID, dirMetrics) + if err != nil { + if cleanup != nil { + cleanup() + } + return snapshotName, materializedSnapshotName, scanPath, nil, nil, 0, nil, 0, err + } + klog.Infof( + "storageindex: 目录增长差异计算完成 scan_id=%s workspace_type=%s workspace_name=%s base_scan_id=%s changed_path_count=%d", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, baseScanID, changedPathCount, + ) + + klog.Infof( + "storageindex: 开始执行冗余检测 scan_id=%s workspace_type=%s workspace_name=%s", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, + ) + hits, err := s.detectRedundancy(ctx, toolboxPod, prefixConfig, scanID, workspace, entries, dirMetrics) + if err != nil { + if cleanup != nil { + cleanup() + } + return snapshotName, materializedSnapshotName, scanPath, nil, nil, 0, nil, 0, err + } + klog.Infof( + "storageindex: 冗余检测完成 scan_id=%s workspace_type=%s workspace_name=%s hit_count=%d", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, len(hits), + ) + + return snapshotName, materializedSnapshotName, scanPath, entries, dirMetrics, changedPathCount, hits, totalSize, nil +} + +func (s *Service) collectFilteredUserWorkspaceSnapshot( + ctx context.Context, + toolboxPod *corev1.Pod, + prefixConfig ceph.StoragePrefixConfig, + scanID string, + workspace resolvedWorkspace, + scanPath string, + snapshotName string, + materializedSnapshotName string, + baseScanID string, +) (string, string, string, []model.StorageIndexEntry, []model.StorageIndexDirectoryMetric, int64, []model.StorageIndexRedundancyHit, int64, error) { + signatures, err := s.listImmediateSignatures(toolboxPod, scanID, workspace.WorkspaceType, workspace.WorkspaceName, workspace.LogicalPath, scanPath) + if err != nil { + return snapshotName, materializedSnapshotName, scanPath, nil, nil, 0, nil, 0, err + } + + selected, skipped := filterTopLevelSignaturesForModelCopyScan(signatures) + selectedNames := make([]string, 0, len(selected)) + for _, item := range selected { + selectedNames = append(selectedNames, item.Name) + } + skippedNames := make([]string, 0, len(skipped)) + for _, item := range skipped { + skippedNames = append(skippedNames, item.Name) + } + klog.Infof( + "storageindex: 用户空间顶层目录过滤完成 scan_id=%s workspace_name=%s selected_top_level=%v skipped_top_level=%v total_candidates=%d", + scanID, workspace.WorkspaceName, selectedNames, skippedNames, len(signatures), + ) + + entries := make([]model.StorageIndexEntry, 0) + dirMetrics := make([]model.StorageIndexDirectoryMetric, 0) + for _, sig := range selected { + if sig.EntryType != model.StorageIndexEntryTypeDir { + continue + } + + klog.Infof( + "storageindex: 开始扫描用户空间候选顶层子树 scan_id=%s subtree=%s actual_path=%s", + scanID, sig.LogicalPath, sig.ActualPath, + ) + + subtreeEntries, scanErr := s.scanUserWorkspaceSubtree(toolboxPod, scanID, workspace, sig) + if scanErr != nil { + return snapshotName, materializedSnapshotName, scanPath, nil, nil, 0, nil, 0, scanErr + } + entries = append(entries, subtreeEntries...) + + subtreeMetrics, _ := buildDirectoryMetrics( + scanID, + resolvedWorkspace{ + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + LogicalPath: sig.LogicalPath, + }, + subtreeEntries, + ) + dirMetrics = append(dirMetrics, subtreeMetrics...) + } + + rootMetric := model.StorageIndexDirectoryMetric{ + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + ScanID: scanID, + Path: workspace.LogicalPath, + Name: path.Base(workspace.LogicalPath), + Depth: 0, + IsTopLevel: false, + } + for _, entry := range entries { + switch entry.EntryType { + case model.StorageIndexEntryTypeFile: + rootMetric.FileCount++ + rootMetric.TotalSizeBytes += entry.SizeBytes + case model.StorageIndexEntryTypeDir: + if entry.LogicalPath != workspace.LogicalPath { + rootMetric.DirectoryCount++ + } + } + } + dirMetrics = append(dirMetrics, rootMetric) + + changedPathCount, err := s.applyGrowthFromPreviousScan(ctx, workspace, baseScanID, dirMetrics) + if err != nil { + return snapshotName, materializedSnapshotName, scanPath, nil, nil, 0, nil, 0, err + } + + hits, err := s.detectRedundancy(ctx, toolboxPod, prefixConfig, scanID, workspace, entries, dirMetrics) + if err != nil { + return snapshotName, materializedSnapshotName, scanPath, nil, nil, 0, nil, 0, err + } + + klog.Infof( + "storageindex: 用户空间候选顶层子树扫描完成 scan_id=%s workspace_name=%s selected_subtrees=%d skipped_subtrees=%d entry_count=%d metric_count=%d hit_count=%d", + scanID, workspace.WorkspaceName, len(selected), len(skipped), len(entries), len(dirMetrics), len(hits), + ) + + return snapshotName, materializedSnapshotName, scanPath, entries, dirMetrics, changedPathCount, hits, rootMetric.TotalSizeBytes, nil +} + +func (s *Service) scanUserWorkspaceSubtree( + toolboxPod *corev1.Pod, + scanID string, + workspace resolvedWorkspace, + sig topLevelSignature, +) ([]model.StorageIndexEntry, error) { + if path.Clean(sig.ParentLogicalPath) != path.Clean(workspace.LogicalPath) { + return s.scanPathDirectories( + toolboxPod, + scanID, + workspace.WorkspaceType, + workspace.WorkspaceName, + sig.LogicalPath, + sig.ActualPath, + ) + } + + allowList, ok := immediateSubtreeAllowListForTopLevel(sig.Name) + if !ok { + return s.scanPathDirectories( + toolboxPod, + scanID, + workspace.WorkspaceType, + workspace.WorkspaceName, + sig.LogicalPath, + sig.ActualPath, + ) + } + + children, err := s.listImmediateSignatures( + toolboxPod, + scanID, + workspace.WorkspaceType, + workspace.WorkspaceName, + sig.LogicalPath, + sig.ActualPath, + ) + if err != nil { + return nil, err + } + + selectedChildren, skippedChildren := filterImmediateSubtreesByAllowList(children, allowList) + selectedNames := make([]string, 0, len(selectedChildren)) + skippedNames := make([]string, 0, len(skippedChildren)) + pruneActualPaths := make([]string, 0, len(skippedChildren)) + for _, child := range selectedChildren { + selectedNames = append(selectedNames, child.Name) + } + for _, child := range skippedChildren { + skippedNames = append(skippedNames, child.Name) + pruneActualPaths = append(pruneActualPaths, child.ActualPath) + } + + klog.Infof( + "storageindex: 顶层子树二级白名单过滤完成 scan_id=%s workspace_name=%s subtree=%s selected_children=%v skipped_children=%v total_children=%d", + scanID, workspace.WorkspaceName, sig.LogicalPath, selectedNames, skippedNames, len(children), + ) + + return s.scanPathDirectoriesWithPrunedChildren( + toolboxPod, + scanID, + workspace.WorkspaceType, + workspace.WorkspaceName, + sig.LogicalPath, + sig.ActualPath, + pruneActualPaths, + ) +} + +func (s *Service) collectPublicWorkspaceSnapshot( + ctx context.Context, + toolboxPod *corev1.Pod, + _ ceph.StoragePrefixConfig, + scanID string, + workspace resolvedWorkspace, + scanPath string, + snapshotName string, + materializedSnapshotName string, + baseScanID string, +) (string, string, string, []model.StorageIndexEntry, []model.StorageIndexDirectoryMetric, int64, []model.StorageIndexRedundancyHit, int64, error) { + signatures, err := s.listImmediateSignatures(toolboxPod, scanID, workspace.WorkspaceType, workspace.WorkspaceName, workspace.LogicalPath, scanPath) + if err != nil { + return snapshotName, materializedSnapshotName, scanPath, nil, nil, 0, nil, 0, err + } + + selected := make([]topLevelSignature, 0) + for _, sig := range signatures { + if _, ok := publicBaselineTopLevelAllowList[strings.ToLower(sig.Name)]; ok { + selected = append(selected, sig) + } + } + + sort.Slice(selected, func(i, j int) bool { + return selected[i].LogicalPath < selected[j].LogicalPath + }) + + selectedNames := make([]string, 0, len(selected)) + for _, item := range selected { + selectedNames = append(selectedNames, item.Name) + } + klog.Infof( + "storageindex: 公共基线目录过滤完成 scan_id=%s workspace_name=%s selected_top_level=%v total_candidates=%d", + scanID, workspace.WorkspaceName, selectedNames, len(signatures), + ) + + entries := make([]model.StorageIndexEntry, 0) + dirMetrics := make([]model.StorageIndexDirectoryMetric, 0) + for _, sig := range selected { + klog.Infof( + "storageindex: 开始扫描公共空间顶层子树 scan_id=%s subtree=%s actual_path=%s", + scanID, sig.LogicalPath, sig.ActualPath, + ) + + subtreeEntries, scanErr := s.scanPathDirectories(toolboxPod, scanID, workspace.WorkspaceType, workspace.WorkspaceName, sig.LogicalPath, sig.ActualPath) + if scanErr != nil { + return snapshotName, materializedSnapshotName, scanPath, nil, nil, 0, nil, 0, scanErr + } + entries = append(entries, subtreeEntries...) + + if sig.EntryType == model.StorageIndexEntryTypeDir { + subtreeMetrics, _ := buildDirectoryMetrics( + scanID, + resolvedWorkspace{ + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + LogicalPath: sig.LogicalPath, + }, + subtreeEntries, + ) + dirMetrics = append(dirMetrics, subtreeMetrics...) + } + } + + rootMetric := model.StorageIndexDirectoryMetric{ + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + ScanID: scanID, + Path: workspace.LogicalPath, + Name: path.Base(workspace.LogicalPath), + Depth: 0, + IsTopLevel: false, + } + for _, entry := range entries { + switch entry.EntryType { + case model.StorageIndexEntryTypeFile: + rootMetric.FileCount++ + rootMetric.TotalSizeBytes += entry.SizeBytes + case model.StorageIndexEntryTypeDir: + if entry.LogicalPath != workspace.LogicalPath { + rootMetric.DirectoryCount++ + } + } + } + dirMetrics = append(dirMetrics, rootMetric) + + changedPathCount, err := s.applyGrowthFromPreviousScan(ctx, workspace, baseScanID, dirMetrics) + if err != nil { + return snapshotName, materializedSnapshotName, scanPath, nil, nil, 0, nil, 0, err + } + + klog.Infof( + "storageindex: 公共基线子树扫描完成 scan_id=%s workspace_name=%s selected_subtrees=%d entry_count=%d metric_count=%d total_size_bytes=%d", + scanID, workspace.WorkspaceName, len(selected), len(entries), len(dirMetrics), rootMetric.TotalSizeBytes, + ) + + return snapshotName, materializedSnapshotName, scanPath, entries, dirMetrics, changedPathCount, nil, rootMetric.TotalSizeBytes, nil +} + +type workspaceCounts struct { + EntryCount int64 + FileCount int64 + DirectoryCount int64 + TotalSizeBytes int64 + RedundancyCount int64 + RedundancyBytes int64 +} + +func (s *Service) collectWorkspaceIncremental( + ctx context.Context, + scanID string, + workspace resolvedWorkspace, + baseScanID string, +) (*incrementalCollectResult, error) { + cfg := config.GetConfig() + prefixConfig := ceph.StoragePrefixConfig{ + User: cfg.Storage.Prefix.User, + Account: cfg.Storage.Prefix.Account, + Public: cfg.Storage.Prefix.Public, + } + + toolboxPod, err := ceph.FindCephToolboxPod(s.kubeClient, toolboxNamespace) + if err != nil { + return nil, fmt.Errorf("find ceph toolbox pod failed: %w", err) + } + + rootPath, err := ceph.ResolveCephFSPath( + s.kubeClient, + s.kubeConfig, + toolboxNamespace, + workspace.LogicalPath, + prefixConfig, + ) + if err != nil { + return nil, fmt.Errorf("resolve workspace path failed: %w", err) + } + + subvolumeRoot, err := ceph.GetCephMountRoot(s.kubeClient, s.kubeConfig, toolboxNamespace) + if err != nil { + return nil, fmt.Errorf("resolve ceph subvolume root failed: %w", err) + } + + currentScanPath, snapshotName, materializedSnapshotName, cleanup, err := s.prepareScanPath(toolboxPod, subvolumeRoot, rootPath, scanID) + if err != nil { + return nil, err + } + + baseJob, err := s.getScanJobByID(ctx, baseScanID) + if err != nil { + if cleanup != nil { + cleanup() + } + return nil, err + } + if baseJob == nil || strings.TrimSpace(baseJob.SnapshotName) == "" { + if cleanup != nil { + cleanup() + } + return nil, fmt.Errorf("base scan %s has no retained snapshot", baseScanID) + } + + previousScanPath := strings.TrimSpace(baseJob.ScanRoot) + if previousScanPath == "" { + previousScanPath, err = s.resolveExistingSnapshotScanPath(toolboxPod, subvolumeRoot, rootPath, baseJob.SnapshotName) + if err != nil { + if cleanup != nil { + cleanup() + } + return nil, fmt.Errorf("resolve previous snapshot path failed: %w", err) + } + } + + klog.Infof( + "storageindex: 开始比较快照差异 scan_id=%s workspace_type=%s workspace_name=%s base_scan_id=%s current_scan_root=%s previous_scan_root=%s", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, baseScanID, currentScanPath, previousScanPath, + ) + + previousRecordedChangedAt, err := loadCurrentWorkspaceDirectoryChangedAtMap(ctx, workspace) + if err != nil { + if cleanup != nil { + cleanup() + } + return nil, fmt.Errorf("query recorded directory rctime from latest workspace state failed: %w", err) + } + plan, changedCount, err := s.buildIncrementalPlan( + toolboxPod, + scanID, + workspace, + workspace.LogicalPath, + currentScanPath, + previousScanPath, + previousRecordedChangedAt, + ) + if err != nil { + if cleanup != nil { + cleanup() + } + return nil, err + } + existingCandidates, err := listWorkspaceCandidates(ctx, workspace) + if err != nil { + if cleanup != nil { + cleanup() + } + return nil, fmt.Errorf("query workspace candidates before incremental candidate-root diff failed: %w", err) + } + plan, changedCount, err = s.augmentIncrementalPlanWithCandidateRootDiffs( + toolboxPod, + scanID, + workspace, + currentScanPath, + previousScanPath, + plan, + changedCount, + existingCandidates, + previousRecordedChangedAt, + ) + if err != nil { + if cleanup != nil { + cleanup() + } + return nil, err + } + klog.Infof( + "storageindex: 增量子树差异统计 scan_id=%s workspace_type=%s workspace_name=%s compared_nodes=%d reused_nodes=%d pruned_dirs=%d new_nodes=%d updated_nodes=%d removed_nodes=%d rescan_targets=%d upsert_entries=%d removed_prefixes=%d", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, + plan.ComparedNodes, plan.ReusedNodes, plan.PrunedDirs, plan.NewNodes, plan.UpdatedNodes, plan.RemovedNodes, + len(plan.RescanTargets), len(plan.UpsertEntries), len(plan.RemovedPrefixes), + ) + + if changedCount > 0 { + plan = expandIncrementalPlanWithCandidateRoots(scanID, workspace, currentScanPath, plan, existingCandidates) + } + + if changedCount == 0 { + return &incrementalCollectResult{ + SnapshotName: snapshotName, + MaterializedSnapshotName: materializedSnapshotName, + ScanRoot: currentScanPath, + DiffMethod: "recursive_snapshot_diff", + ChangedPathCount: 0, + }, nil + } + + newEntries := make([]model.StorageIndexEntry, 0, len(plan.UpsertEntries)) + newMetrics := make([]model.StorageIndexDirectoryMetric, 0) + changedPrefixes := make([]string, 0, len(plan.RescanTargets)) + + for _, entry := range plan.UpsertEntries { + newEntries = appendUniqueEntry(newEntries, entry) + } + if changedCount > 0 { + rootSignature, rootErr := s.loadDirectorySignatureAtPath( + toolboxPod, + scanID, + workspace.WorkspaceType, + workspace.WorkspaceName, + workspace.LogicalPath, + "", + currentScanPath, + ) + if rootErr != nil { + if cleanup != nil { + cleanup() + } + return nil, fmt.Errorf("load workspace root signature failed: %w", rootErr) + } + newEntries = appendUniqueEntry( + newEntries, + signatureToEntry(scanID, workspace.WorkspaceType, workspace.WorkspaceName, workspace.LogicalPath, *rootSignature), + ) + } + + for _, sig := range plan.RescanTargets { + klog.Infof( + "storageindex: 开始重扫变化子树 scan_id=%s workspace_type=%s workspace_name=%s subtree=%s entry_type=%s size_bytes=%d", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, sig.LogicalPath, sig.EntryType, sig.SizeBytes, + ) + + entries, scanErr := s.scanUserWorkspaceSubtree(toolboxPod, scanID, workspace, sig) + if scanErr != nil { + if cleanup != nil { + cleanup() + } + return nil, fmt.Errorf("scan changed subtree %s failed: %w", sig.LogicalPath, scanErr) + } + newEntries = append(newEntries, entries...) + changedPrefixes = append(changedPrefixes, sig.LogicalPath) + + if sig.EntryType == model.StorageIndexEntryTypeDir { + subtreeMetrics, _ := buildDirectoryMetrics( + scanID, + resolvedWorkspace{ + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + LogicalPath: sig.LogicalPath, + }, + entries, + ) + newMetrics = append(newMetrics, subtreeMetrics...) + } + } + affectedEntryPaths := collectAffectedMetricPaths(workspace.LogicalPath, changedPrefixes, plan.RemovedPrefixes, newEntries) + for _, entryPath := range affectedEntryPaths { + if entryPath == workspace.LogicalPath || isCoveredByPrefixes(entryPath, changedPrefixes) { + continue + } + signature, signatureErr := s.loadDirectorySignatureAtPath( + toolboxPod, + scanID, + workspace.WorkspaceType, + workspace.WorkspaceName, + entryPath, + parentForDirectory(entryPath, workspace.LogicalPath), + logicalPathToActualPath(currentScanPath, workspace.LogicalPath, entryPath), + ) + if signatureErr != nil { + if cleanup != nil { + cleanup() + } + return nil, fmt.Errorf("load ancestor directory signature failed for %s: %w", entryPath, signatureErr) + } + newEntries = appendUniqueEntry( + newEntries, + signatureToEntry(scanID, workspace.WorkspaceType, workspace.WorkspaceName, workspace.LogicalPath, *signature), + ) + } + + return &incrementalCollectResult{ + SnapshotName: snapshotName, + MaterializedSnapshotName: materializedSnapshotName, + ScanRoot: currentScanPath, + DiffMethod: "recursive_snapshot_diff", + ChangedPathCount: changedCount, + ChangedPrefixes: changedPrefixes, + RemovedPrefixes: plan.RemovedPrefixes, + NewEntries: newEntries, + NewDirMetrics: newMetrics, + }, nil +} + +func (s *Service) queryWorkspaceCounts(ctx context.Context, workspace resolvedWorkspace) (workspaceCounts, error) { + return s.queryWorkspaceCountsWithDB(query.GetDB().WithContext(ctx), workspace) +} + +func (s *Service) queryWorkspaceCountsWithDB(db *gorm.DB, workspace resolvedWorkspace) (workspaceCounts, error) { + counts := workspaceCounts{} + + if err := db.Model(&model.StorageIndexEntry{}). + Where("workspace_type = ? AND workspace_name = ?", workspace.WorkspaceType, workspace.WorkspaceName). + Count(&counts.EntryCount).Error; err != nil { + return counts, fmt.Errorf("count workspace entries failed: %w", err) + } + if err := db.Model(&model.StorageIndexEntry{}). + Where("workspace_type = ? AND workspace_name = ? AND entry_type = ? AND logical_path <> ?", workspace.WorkspaceType, workspace.WorkspaceName, model.StorageIndexEntryTypeDir, workspace.LogicalPath). + Count(&counts.DirectoryCount).Error; err != nil { + return counts, fmt.Errorf("count workspace directories failed: %w", err) + } + if err := db.Model(&model.StorageIndexCandidate{}). + Where("workspace_type = ? AND workspace_name = ? AND status = ?", workspace.WorkspaceType, workspace.WorkspaceName, model.StorageIndexCandidateStatusVerified). + Count(&counts.FileCount).Error; err != nil { + return counts, fmt.Errorf("count candidates failed: %w", err) + } + if err := db.Model(&model.StorageIndexDirectoryMetric{}). + Select("COALESCE(total_size_bytes, 0)"). + Where("workspace_type = ? AND workspace_name = ? AND path = ?", workspace.WorkspaceType, workspace.WorkspaceName, workspace.LogicalPath). + Scan(&counts.TotalSizeBytes).Error; err != nil { + return counts, fmt.Errorf("query workspace root total bytes failed: %w", err) + } + if err := db.Model(&model.StorageIndexRedundancyHit{}). + Where("workspace_type = ? AND workspace_name = ?", workspace.WorkspaceType, workspace.WorkspaceName). + Count(&counts.RedundancyCount).Error; err != nil { + return counts, fmt.Errorf("count workspace redundancy hits failed: %w", err) + } + if err := db.Model(&model.StorageIndexRedundancyHit{}). + Select("COALESCE(SUM(estimated_bytes), 0)"). + Where("workspace_type = ? AND workspace_name = ?", workspace.WorkspaceType, workspace.WorkspaceName). + Scan(&counts.RedundancyBytes).Error; err != nil { + return counts, fmt.Errorf("sum workspace redundancy bytes failed: %w", err) + } + + return counts, nil +} + +func listWorkspaceCandidates(ctx context.Context, workspace resolvedWorkspace) ([]model.StorageIndexCandidate, error) { + items := make([]model.StorageIndexCandidate, 0) + if err := query.GetDB().WithContext(ctx). + Where("workspace_type = ? AND workspace_name = ?", workspace.WorkspaceType, workspace.WorkspaceName). + Order("target_path ASC"). + Find(&items).Error; err != nil { + return nil, fmt.Errorf("query workspace candidates failed: %w", err) + } + return items, nil +} + +func loadCurrentWorkspaceDirectoryChangedAtMap( + ctx context.Context, + workspace resolvedWorkspace, +) (map[string]*time.Time, error) { + result := make(map[string]*time.Time) + + type row struct { + LogicalPath string `gorm:"column:logical_path"` + ChangedAt *time.Time `gorm:"column:changed_at"` + } + rows := make([]row, 0) + if err := query.GetDB().WithContext(ctx). + Model(&model.StorageIndexEntry{}). + Select("logical_path, changed_at"). + Where( + "workspace_type = ? AND workspace_name = ? AND entry_type = ?", + workspace.WorkspaceType, + workspace.WorkspaceName, + model.StorageIndexEntryTypeDir, + ). + Find(&rows).Error; err != nil { + return nil, err + } + for _, row := range rows { + result[normalizeUnixPath(row.LogicalPath)] = row.ChangedAt + } + return result, nil +} + +func cleanupWorkspaceStateBeforeInitialScan(db *gorm.DB, workspace resolvedWorkspace) (int64, error) { + clearedRows := int64(0) + err := db.Transaction(func(tx *gorm.DB) error { + deleteScoped := func(value any) error { + result := tx. + Where("workspace_type = ? AND workspace_name = ?", workspace.WorkspaceType, workspace.WorkspaceName). + Delete(value) + if result.Error != nil { + return result.Error + } + clearedRows += result.RowsAffected + return nil + } + + if err := deleteScoped(&model.StorageIndexEntry{}); err != nil { + return err + } + if err := deleteScoped(&model.StorageIndexDirectoryMetric{}); err != nil { + return err + } + if err := deleteScoped(&model.StorageIndexRedundancyHit{}); err != nil { + return err + } + if err := deleteScoped(&model.StorageIndexCandidate{}); err != nil { + return err + } + if err := deleteScoped(&model.StorageIndexCandidateFile{}); err != nil { + return err + } + if workspace.WorkspaceType != model.StorageIndexWorkspaceTypePublic { + return nil + } + + rootResult := tx.Exec("DELETE FROM " + (&model.StorageIndexPublicRootBaseline{}).TableName()) + if rootResult.Error != nil { + return rootResult.Error + } + clearedRows += rootResult.RowsAffected + + fileResult := tx.Exec("DELETE FROM " + (&model.StorageIndexPublicFileBaseline{}).TableName()) + if fileResult.Error != nil { + return fileResult.Error + } + clearedRows += fileResult.RowsAffected + return nil + }) + if err != nil { + return 0, err + } + return clearedRows, nil +} + +func deleteWorkspacePathPrefix(tx *gorm.DB, workspace resolvedWorkspace, prefix string) error { + entryWhere := tx.Where("workspace_type = ? AND workspace_name = ?", workspace.WorkspaceType, workspace.WorkspaceName) + if err := entryWhere.Where("logical_path = ? OR logical_path LIKE ?", prefix, prefix+"/%"). + Delete(&model.StorageIndexEntry{}).Error; err != nil { + return err + } + + metricWhere := tx.Where("workspace_type = ? AND workspace_name = ?", workspace.WorkspaceType, workspace.WorkspaceName) + if err := metricWhere.Where("path = ? OR path LIKE ?", prefix, prefix+"/%"). + Delete(&model.StorageIndexDirectoryMetric{}).Error; err != nil { + return err + } + + hitWhere := tx.Where("workspace_type = ? AND workspace_name = ?", workspace.WorkspaceType, workspace.WorkspaceName) + if err := hitWhere.Where("target_path = ? OR target_path LIKE ?", prefix, prefix+"/%"). + Delete(&model.StorageIndexRedundancyHit{}).Error; err != nil { + return err + } + + return nil +} + +func deleteWorkspaceExactPath(tx *gorm.DB, workspace resolvedWorkspace, targetPath string) error { + if err := tx.Where("workspace_type = ? AND workspace_name = ? AND logical_path = ?", workspace.WorkspaceType, workspace.WorkspaceName, targetPath). + Delete(&model.StorageIndexEntry{}).Error; err != nil { + return err + } + if err := tx.Where("workspace_type = ? AND workspace_name = ? AND target_path = ?", workspace.WorkspaceType, workspace.WorkspaceName, targetPath). + Delete(&model.StorageIndexRedundancyHit{}).Error; err != nil { + return err + } + return nil +} + +func deleteWorkspaceMetricExactPath(tx *gorm.DB, workspace resolvedWorkspace, targetPath string) error { + return tx.Where("workspace_type = ? AND workspace_name = ? AND path = ?", workspace.WorkspaceType, workspace.WorkspaceName, targetPath). + Delete(&model.StorageIndexDirectoryMetric{}).Error +} + +func deleteWorkspaceRedundancyHitsForExactPaths( + tx *gorm.DB, + workspace resolvedWorkspace, + targetPaths []string, +) error { + normalized := make([]string, 0, len(targetPaths)) + for _, targetPath := range targetPaths { + cleaned := normalizeUnixPath(targetPath) + if cleaned == "" { + continue + } + normalized = append(normalized, cleaned) + } + if len(normalized) == 0 { + return nil + } + return tx. + Where("workspace_type = ? AND workspace_name = ? AND target_path IN ?", workspace.WorkspaceType, workspace.WorkspaceName, normalized). + Delete(&model.StorageIndexRedundancyHit{}).Error +} + +func deleteWorkspaceCandidateStateByPaths( + tx *gorm.DB, + workspace resolvedWorkspace, + candidatePaths []string, +) error { + for _, candidatePath := range candidatePaths { + cleaned := normalizeUnixPath(candidatePath) + if cleaned == "" { + continue + } + if err := tx. + Where("workspace_type = ? AND workspace_name = ? AND target_path = ?", workspace.WorkspaceType, workspace.WorkspaceName, cleaned). + Delete(&model.StorageIndexCandidate{}).Error; err != nil { + return err + } + if err := tx. + Where("workspace_type = ? AND workspace_name = ? AND candidate_path = ?", workspace.WorkspaceType, workspace.WorkspaceName, cleaned). + Delete(&model.StorageIndexCandidateFile{}).Error; err != nil { + return err + } + if err := tx. + Where( + "workspace_type = ? AND workspace_name = ? AND (target_path = ? OR target_path LIKE ?)", + workspace.WorkspaceType, + workspace.WorkspaceName, + cleaned, + cleaned+"/%", + ). + Delete(&model.StorageIndexRedundancyHit{}).Error; err != nil { + return err + } + } + return nil +} + +func pruneCandidateDescendants( + tx *gorm.DB, + workspace resolvedWorkspace, + candidates []model.StorageIndexCandidate, +) error { + pruned := 0 + for _, candidate := range candidates { + if candidate.TargetPath == "" { + continue + } + likePrefix := candidate.TargetPath + "/%" + dirResult := tx. + Where( + "workspace_type = ? AND workspace_name = ? AND path LIKE ?", + workspace.WorkspaceType, + workspace.WorkspaceName, + likePrefix, + ). + Delete(&model.StorageIndexDirectoryMetric{}) + if dirResult.Error != nil { + return dirResult.Error + } + pruned += int(dirResult.RowsAffected) + entryResult := tx. + Where( + "workspace_type = ? AND workspace_name = ? AND logical_path LIKE ?", + workspace.WorkspaceType, + workspace.WorkspaceName, + likePrefix, + ). + Delete(&model.StorageIndexEntry{}) + if entryResult.Error != nil { + return entryResult.Error + } + pruned += int(entryResult.RowsAffected) + } + klog.Infof( + "storageindex: 已按候选目录裁剪子目录骨架 workspace_type=%s workspace_name=%s candidate_count=%d pruned_rows=%d", + workspace.WorkspaceType, workspace.WorkspaceName, len(candidates), pruned, + ) + return nil +} + +func rebuildWorkspaceRootMetric(tx *gorm.DB, workspace resolvedWorkspace, scanID string) error { + var fileCount int64 + var directoryCount int64 + var totalSize int64 + + if err := tx.Model(&model.StorageIndexEntry{}). + Where("workspace_type = ? AND workspace_name = ? AND entry_type = ?", workspace.WorkspaceType, workspace.WorkspaceName, model.StorageIndexEntryTypeFile). + Count(&fileCount).Error; err != nil { + return err + } + if err := tx.Model(&model.StorageIndexEntry{}). + Where("workspace_type = ? AND workspace_name = ? AND entry_type = ? AND logical_path <> ?", workspace.WorkspaceType, workspace.WorkspaceName, model.StorageIndexEntryTypeDir, workspace.LogicalPath). + Count(&directoryCount).Error; err != nil { + return err + } + if err := tx.Model(&model.StorageIndexEntry{}). + Select("COALESCE(SUM(size_bytes), 0)"). + Where("workspace_type = ? AND workspace_name = ? AND entry_type = ?", workspace.WorkspaceType, workspace.WorkspaceName, model.StorageIndexEntryTypeFile). + Scan(&totalSize).Error; err != nil { + return err + } + + if err := tx.Where("workspace_type = ? AND workspace_name = ? AND path = ?", workspace.WorkspaceType, workspace.WorkspaceName, workspace.LogicalPath). + Delete(&model.StorageIndexDirectoryMetric{}).Error; err != nil { + return err + } + + rootMetric := &model.StorageIndexDirectoryMetric{ + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + ScanID: scanID, + Path: workspace.LogicalPath, + ParentPath: "", + Name: path.Base(workspace.LogicalPath), + Depth: 0, + IsTopLevel: false, + FileCount: fileCount, + DirectoryCount: directoryCount, + TotalSizeBytes: totalSize, + LatestGrowth: 0, + } + return tx.Create(rootMetric).Error +} + +func collectAffectedMetricPaths( + workspaceRoot string, + changedPrefixes []string, + removedPrefixes []string, + newEntries []model.StorageIndexEntry, +) []string { + set := make(map[string]struct{}) + addPathAndAncestors := func(target string, includeSelf bool) { + current := normalizeUnixPath(target) + root := normalizeUnixPath(workspaceRoot) + for current != "" { + if includeSelf || current != target { + set[current] = struct{}{} + } + if current == root { + break + } + next := path.Dir(current) + if next == current || next == "." || next == "/" { + break + } + current = next + } + } + + for _, prefix := range changedPrefixes { + addPathAndAncestors(prefix, false) + } + for _, prefix := range removedPrefixes { + addPathAndAncestors(path.Dir(prefix), true) + } + for _, entry := range newEntries { + if entry.EntryType == model.StorageIndexEntryTypeDir { + addPathAndAncestors(entry.LogicalPath, true) + } else { + addPathAndAncestors(entry.ParentPath, true) + } + } + + result := make([]string, 0, len(set)) + for item := range set { + if item != "" { + result = append(result, item) + } + } + sort.Slice(result, func(i, j int) bool { + return depthFromRoot(result[i], workspaceRoot) > depthFromRoot(result[j], workspaceRoot) + }) + return result +} + +func expandIncrementalPlanWithCandidateRoots( + scanID string, + workspace resolvedWorkspace, + currentScanRoot string, + plan *incrementalPlan, + existingCandidates []model.StorageIndexCandidate, +) *incrementalPlan { + if plan == nil || len(existingCandidates) == 0 { + return plan + } + + impactedRoots := collectImpactedCandidateRoots(plan, existingCandidates) + if len(impactedRoots) == 0 { + return plan + } + + rescanRoots := make([]string, 0, len(impactedRoots)) + for _, root := range impactedRoots { + if isCoveredByPrefixes(root, plan.RemovedPrefixes) { + continue + } + rescanRoots = appendUniquePrefix(rescanRoots, root) + } + if len(rescanRoots) == 0 { + return plan + } + + expandedRoots := collapsePathPrefixes(impactedRoots) + rescanRoots = collapsePathPrefixes(rescanRoots) + + expanded := &incrementalPlan{ + RescanTargets: filterRescanTargetsOutsidePrefixes(plan.RescanTargets, expandedRoots), + UpsertEntries: filterEntriesOutsidePrefixes(plan.UpsertEntries, expandedRoots), + RemovedPrefixes: append([]string{}, plan.RemovedPrefixes...), + ComparedNodes: plan.ComparedNodes, + PrunedDirs: plan.PrunedDirs, + NewNodes: plan.NewNodes, + UpdatedNodes: plan.UpdatedNodes, + RemovedNodes: plan.RemovedNodes, + ReusedNodes: plan.ReusedNodes, + } + for _, root := range rescanRoots { + expanded.RescanTargets = appendUniqueSignature( + expanded.RescanTargets, + buildCandidateRootRescanSignature(scanID, workspace, currentScanRoot, root), + ) + } + return expanded +} + +func collectImpactedCandidateRoots( + plan *incrementalPlan, + existingCandidates []model.StorageIndexCandidate, +) []string { + if plan == nil { + return nil + } + + relatedPaths := make([]string, 0, len(plan.RescanTargets)+len(plan.UpsertEntries)+len(plan.RemovedPrefixes)) + for _, sig := range plan.RescanTargets { + relatedPaths = appendUniquePrefix(relatedPaths, sig.LogicalPath) + } + for _, entry := range plan.UpsertEntries { + relatedPaths = appendUniquePrefix(relatedPaths, entry.LogicalPath) + } + for _, prefix := range plan.RemovedPrefixes { + relatedPaths = appendUniquePrefix(relatedPaths, prefix) + } + + impacted := make([]string, 0) + for _, candidate := range existingCandidates { + targetPath := normalizeUnixPath(candidate.TargetPath) + if targetPath == "" { + continue + } + for _, changedPath := range relatedPaths { + if pathOverlaps(targetPath, changedPath) { + impacted = appendUniquePrefix(impacted, targetPath) + break + } + } + } + return collapsePathPrefixes(impacted) +} + +func collectAffectedCandidatePaths( + existingCandidates []model.StorageIndexCandidate, + affectedMetricPaths []string, + prefixesToDelete []string, +) []string { + affectedSet := make(map[string]struct{}, len(affectedMetricPaths)) + for _, metricPath := range affectedMetricPaths { + cleaned := normalizeUnixPath(metricPath) + if cleaned == "" { + continue + } + affectedSet[cleaned] = struct{}{} + } + + paths := make([]string, 0) + for _, candidate := range existingCandidates { + targetPath := normalizeUnixPath(candidate.TargetPath) + if targetPath == "" { + continue + } + if _, ok := affectedSet[targetPath]; ok || isCoveredByPrefixes(targetPath, prefixesToDelete) { + paths = appendUniquePrefix(paths, targetPath) + } + } + return paths +} + +func (s *Service) augmentIncrementalPlanWithCandidateRootDiffs( + toolboxPod *corev1.Pod, + scanID string, + workspace resolvedWorkspace, + currentScanRoot string, + previousScanRoot string, + plan *incrementalPlan, + changedCount int64, + existingCandidates []model.StorageIndexCandidate, + previousRecordedChangedAt map[string]*time.Time, +) (*incrementalPlan, int64, error) { + if plan == nil || len(existingCandidates) == 0 { + return plan, changedCount, nil + } + + candidateRoots := make([]string, 0, len(existingCandidates)) + for _, candidate := range existingCandidates { + if candidate.Status != model.StorageIndexCandidateStatusVerified { + continue + } + targetPath := normalizeUnixPath(candidate.TargetPath) + if targetPath == "" || strings.TrimSpace(candidate.PublicPath) == "" { + continue + } + candidateRoots = appendUniquePrefix(candidateRoots, targetPath) + } + + for _, candidateRoot := range collapsePathPrefixes(candidateRoots) { + if planTouchesPath(plan, candidateRoot) { + continue + } + + currentExists, err := s.directoryExistsInSnapshot(toolboxPod, logicalPathToActualPath(currentScanRoot, workspace.LogicalPath, candidateRoot)) + if err != nil { + return nil, 0, err + } + previousExists, err := s.directoryExistsInSnapshot(toolboxPod, logicalPathToActualPath(previousScanRoot, workspace.LogicalPath, candidateRoot)) + if err != nil { + return nil, 0, err + } + if !currentExists && !previousExists { + continue + } + + switch { + case currentExists && !previousExists: + plan.RescanTargets = appendUniqueSignature(plan.RescanTargets, buildCandidateRootRescanSignature(scanID, workspace, currentScanRoot, candidateRoot)) + plan.NewNodes++ + changedCount++ + case !currentExists && previousExists: + plan.RemovedPrefixes = appendUniquePrefix(plan.RemovedPrefixes, candidateRoot) + plan.RemovedNodes++ + changedCount++ + default: + currentSignature, err := s.loadDirectorySignatureAtPath( + toolboxPod, + scanID, + workspace.WorkspaceType, + workspace.WorkspaceName, + candidateRoot, + parentForDirectory(candidateRoot, workspace.LogicalPath), + logicalPathToActualPath(currentScanRoot, workspace.LogicalPath, candidateRoot), + ) + if err != nil { + return nil, 0, err + } + previousSignature, err := s.loadDirectorySignatureAtPath( + toolboxPod, + scanID, + workspace.WorkspaceType, + workspace.WorkspaceName, + candidateRoot, + parentForDirectory(candidateRoot, workspace.LogicalPath), + logicalPathToActualPath(previousScanRoot, workspace.LogicalPath, candidateRoot), + ) + if err != nil { + return nil, 0, err + } + recordedChangedAt, ok := previousRecordedChangedAt[normalizeUnixPath(candidateRoot)] + if !ok || recordedChangedAt == nil { + return nil, 0, fmt.Errorf("missing recorded directory rctime for %s", candidateRoot) + } + previousSignature.ChangedAt = recordedChangedAt + if !timestampsDifferent(currentSignature.ChangedAt, previousSignature.ChangedAt) { + continue + } + plan.RescanTargets = appendUniqueSignature(plan.RescanTargets, *currentSignature) + plan.UpdatedNodes++ + changedCount++ + } + } + + return plan, changedCount, nil +} + +func mergeIncrementalPlans(dst *incrementalPlan, src *incrementalPlan) { + if dst == nil || src == nil { + return + } + for _, target := range src.RescanTargets { + dst.RescanTargets = appendUniqueSignature(dst.RescanTargets, target) + } + for _, entry := range src.UpsertEntries { + dst.UpsertEntries = appendUniqueEntry(dst.UpsertEntries, entry) + } + for _, prefix := range src.RemovedPrefixes { + dst.RemovedPrefixes = appendUniquePrefix(dst.RemovedPrefixes, prefix) + } + dst.ComparedNodes += src.ComparedNodes + dst.PrunedDirs += src.PrunedDirs + dst.NewNodes += src.NewNodes + dst.UpdatedNodes += src.UpdatedNodes + dst.RemovedNodes += src.RemovedNodes + dst.ReusedNodes += src.ReusedNodes +} + +func planTouchesPath(plan *incrementalPlan, targetPath string) bool { + if plan == nil { + return false + } + for _, target := range plan.RescanTargets { + if pathOverlaps(target.LogicalPath, targetPath) { + return true + } + } + for _, entry := range plan.UpsertEntries { + if pathOverlaps(entry.LogicalPath, targetPath) { + return true + } + } + for _, prefix := range plan.RemovedPrefixes { + if pathOverlaps(prefix, targetPath) { + return true + } + } + return false +} + +func (s *Service) directoryExistsInSnapshot(toolboxPod *corev1.Pod, actualPath string) (bool, error) { + output, err := ceph.ExecInPod( + s.kubeClient, + s.kubeConfig, + toolboxPod, + []string{"sh", "-c", fmt.Sprintf("if [ -d %s ]; then echo 1; else echo 0; fi", shellQuote(actualPath))}, + ) + if err != nil { + return false, err + } + return strings.TrimSpace(output) == "1", nil +} + +func filterRescanTargetsOutsidePrefixes( + targets []topLevelSignature, + prefixes []string, +) []topLevelSignature { + filtered := make([]topLevelSignature, 0, len(targets)) + for _, target := range targets { + if isCoveredByPrefixes(target.LogicalPath, prefixes) { + continue + } + filtered = appendUniqueSignature(filtered, target) + } + return filtered +} + +func filterEntriesOutsidePrefixes( + entries []model.StorageIndexEntry, + prefixes []string, +) []model.StorageIndexEntry { + filtered := make([]model.StorageIndexEntry, 0, len(entries)) + for _, entry := range entries { + if isCoveredByPrefixes(entry.LogicalPath, prefixes) { + continue + } + filtered = appendUniqueEntry(filtered, entry) + } + return filtered +} + +func buildCandidateRootRescanSignature( + _ string, + workspace resolvedWorkspace, + currentScanRoot string, + candidatePath string, +) topLevelSignature { + cleanedPath := normalizeUnixPath(candidatePath) + parentPath := path.Dir(cleanedPath) + if parentPath == "." || parentPath == "/" { + parentPath = workspace.LogicalPath + } + return topLevelSignature{ + Name: path.Base(cleanedPath), + LogicalPath: cleanedPath, + ParentLogicalPath: parentPath, + ActualPath: logicalPathToActualPath(currentScanRoot, workspace.LogicalPath, cleanedPath), + EntryType: model.StorageIndexEntryTypeDir, + } +} + +func logicalPathToActualPath(scanRoot, workspaceRoot, logicalPath string) string { + cleanedLogical := normalizeUnixPath(logicalPath) + cleanedRoot := normalizeUnixPath(workspaceRoot) + relative := strings.TrimPrefix(cleanedLogical, cleanedRoot) + relative = strings.TrimPrefix(relative, "/") + if relative == "" { + return normalizeUnixPath(scanRoot) + } + return normalizeUnixPath(path.Join(scanRoot, relative)) +} + +func isCoveredByPrefixes(target string, prefixes []string) bool { + for _, prefix := range prefixes { + if target == prefix || strings.HasPrefix(target, prefix+"/") { + return true + } + } + return false +} + +func rebuildDirectoryMetric(tx *gorm.DB, workspace resolvedWorkspace, scanID string, metricPath string) error { + metricPath = normalizeUnixPath(metricPath) + rootPath := normalizeUnixPath(workspace.LogicalPath) + + if metricPath != rootPath { + var exists int64 + if err := tx.Model(&model.StorageIndexEntry{}). + Where( + "workspace_type = ? AND workspace_name = ? AND logical_path = ? AND entry_type = ?", + workspace.WorkspaceType, + workspace.WorkspaceName, + metricPath, + model.StorageIndexEntryTypeDir, + ). + Count(&exists).Error; err != nil { + return err + } + if exists == 0 { + return nil + } + } + + likePrefix := metricPath + "/%" + var fileCount int64 + var directoryCount int64 + var totalSize int64 + + if err := tx.Model(&model.StorageIndexEntry{}). + Where( + "workspace_type = ? AND workspace_name = ? AND entry_type = ? AND (logical_path = ? OR logical_path LIKE ?)", + workspace.WorkspaceType, + workspace.WorkspaceName, + model.StorageIndexEntryTypeFile, + metricPath, + likePrefix, + ). + Count(&fileCount).Error; err != nil { + return err + } + + if err := tx.Model(&model.StorageIndexEntry{}). + Where( + "workspace_type = ? AND workspace_name = ? AND entry_type = ? AND logical_path LIKE ?", + workspace.WorkspaceType, + workspace.WorkspaceName, + model.StorageIndexEntryTypeDir, + likePrefix, + ). + Count(&directoryCount).Error; err != nil { + return err + } + + if err := tx.Model(&model.StorageIndexEntry{}). + Select("COALESCE(SUM(size_bytes), 0)"). + Where( + "workspace_type = ? AND workspace_name = ? AND entry_type = ? AND (logical_path = ? OR logical_path LIKE ?)", + workspace.WorkspaceType, + workspace.WorkspaceName, + model.StorageIndexEntryTypeFile, + metricPath, + likePrefix, + ). + Scan(&totalSize).Error; err != nil { + return err + } + + metric := &model.StorageIndexDirectoryMetric{ + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + ScanID: scanID, + Path: metricPath, + ParentPath: parentForDirectory(metricPath, rootPath), + Name: path.Base(metricPath), + Depth: depthFromRoot(metricPath, rootPath), + IsTopLevel: isTopLevelPath(metricPath, rootPath), + FileCount: fileCount, + DirectoryCount: directoryCount, + TotalSizeBytes: totalSize, + LatestGrowth: 0, + } + return tx.Create(metric).Error +} + +func (s *Service) prepareScanPath( + toolboxPod *corev1.Pod, + subvolumeRoot string, + rootPath string, + scanID string, +) (string, string, string, func(), error) { + relativeSuffix := relativeUnixPath(subvolumeRoot, rootPath) + if csi, ok := parseCephCSIWorkspacePath(subvolumeRoot, relativeSuffix); ok { + return s.prepareSubvolumeSnapshotScanPath(toolboxPod, csi, scanID) + } + + snapshotName := "index-" + time.Now().Format("20060102150405") + "-" + strings.ReplaceAll(scanID[:8], "-", "") + snapshotPath := path.Join(rootPath, ".snap", snapshotName) + + if _, err := ceph.ExecInPod( + s.kubeClient, + s.kubeConfig, + toolboxPod, + []string{"mkdir", snapshotPath}, + ); err != nil { + klog.Warningf("storageindex: 创建目录快照失败，回退为 live scan path=%s err=%v", rootPath, err) + return rootPath, "", "", nil, nil + } + + cleanup := func() { + if _, err := ceph.ExecInPod( + s.kubeClient, + s.kubeConfig, + toolboxPod, + []string{"rmdir", snapshotPath}, + ); err != nil { + klog.Warningf("storageindex: 清理目录快照失败 snapshot_path=%s err=%v", snapshotPath, err) + } + } + + return snapshotPath, snapshotName, snapshotName, cleanup, nil +} + +func (s *Service) prepareSubvolumeSnapshotScanPath( + toolboxPod *corev1.Pod, + csi cephCSIWorkspacePath, + scanID string, +) (string, string, string, func(), error) { + snapshotName := "index-" + time.Now().Format("20060102150405") + "-" + strings.ReplaceAll(scanID[:8], "-", "") + + if _, err := ceph.ExecInPod( + s.kubeClient, + s.kubeConfig, + toolboxPod, + []string{"ceph", "fs", "subvolume", "snapshot", "create", cephFSVolumeName, csi.SubvolumeName, snapshotName, "--group_name", csi.GroupName}, + ); err != nil { + klog.Warningf( + "storageindex: 创建 subvolume 快照失败，回退为 live scan group=%s subvolume=%s root=%s err=%v", + csi.GroupName, csi.SubvolumeName, csi.WorkspaceRoot, err, + ) + return csi.WorkspaceRoot, "", "", nil, nil + } + + out, err := ceph.ExecInPod( + s.kubeClient, + s.kubeConfig, + toolboxPod, + []string{"ceph", "fs", "subvolume", "getpath", cephFSVolumeName, csi.SubvolumeName, "--group_name", csi.GroupName}, + ) + if err != nil { + _, _ = ceph.ExecInPod( + s.kubeClient, + s.kubeConfig, + toolboxPod, + []string{"ceph", "fs", "subvolume", "snapshot", "rm", cephFSVolumeName, csi.SubvolumeName, snapshotName, "--group_name", csi.GroupName, "--force"}, + ) + klog.Warningf( + "storageindex: 获取 subvolume 路径失败，回退为 live scan group=%s subvolume=%s snapshot=%s err=%v", + csi.GroupName, csi.SubvolumeName, snapshotName, err, + ) + return csi.WorkspaceRoot, "", "", nil, nil + } + + subvolumeRoot := strings.TrimSpace(out) + if subvolumeRoot == "" { + _, _ = ceph.ExecInPod( + s.kubeClient, + s.kubeConfig, + toolboxPod, + []string{"ceph", "fs", "subvolume", "snapshot", "rm", cephFSVolumeName, csi.SubvolumeName, snapshotName, "--group_name", csi.GroupName, "--force"}, + ) + klog.Warningf( + "storageindex: subvolume 路径为空，回退为 live scan group=%s subvolume=%s snapshot=%s", + csi.GroupName, csi.SubvolumeName, snapshotName, + ) + return csi.WorkspaceRoot, "", "", nil, nil + } + + materializedSnapshotName, err := s.resolveMaterializedSnapshotName(toolboxPod, csi.MountRoot, subvolumeRoot, snapshotName) + if err != nil { + _, _ = ceph.ExecInPod( + s.kubeClient, + s.kubeConfig, + toolboxPod, + []string{"ceph", "fs", "subvolume", "snapshot", "rm", cephFSVolumeName, csi.SubvolumeName, snapshotName, "--group_name", csi.GroupName, "--force"}, + ) + klog.Warningf( + "storageindex: 解析物化快照目录名失败，回退为 live scan group=%s subvolume=%s snapshot=%s err=%v", + csi.GroupName, csi.SubvolumeName, snapshotName, err, + ) + return csi.WorkspaceRoot, "", "", nil, nil + } + + scanRoot := normalizeUnixPath(path.Join(csi.MountRoot, subvolumeRoot, ".snap", materializedSnapshotName, csi.RelativeSuffix)) + cleanup := func() { + if _, err := ceph.ExecInPod( + s.kubeClient, + s.kubeConfig, + toolboxPod, + []string{"ceph", "fs", "subvolume", "snapshot", "rm", cephFSVolumeName, csi.SubvolumeName, snapshotName, "--group_name", csi.GroupName, "--force"}, + ); err != nil { + klog.Warningf( + "storageindex: 清理 subvolume 快照失败 group=%s subvolume=%s snapshot=%s err=%v", + csi.GroupName, csi.SubvolumeName, snapshotName, err, + ) + } + } + + klog.Infof( + "storageindex: 使用 subvolume 快照扫描 group=%s subvolume=%s snapshot=%s materialized_snapshot=%s subvolume_root=%s scan_root=%s", + csi.GroupName, csi.SubvolumeName, snapshotName, materializedSnapshotName, subvolumeRoot, scanRoot, + ) + + return scanRoot, snapshotName, materializedSnapshotName, cleanup, nil +} + +func (s *Service) resolveWorkspace( + ctx context.Context, + workspaceType model.StorageIndexWorkspaceType, + workspaceName string, +) (resolvedWorkspace, error) { + name := strings.TrimSpace(workspaceName) + switch workspaceType { + case model.StorageIndexWorkspaceTypeUser: + if name == "" { + return resolvedWorkspace{}, fmt.Errorf("workspace_name is required for user workspace") + } + var user struct { + Name string `gorm:"column:name"` + Space string `gorm:"column:space"` + } + if err := query.GetDB().WithContext(ctx). + Raw("SELECT name, space FROM users WHERE name = ? AND deleted_at IS NULL", name). + Scan(&user).Error; err != nil { + return resolvedWorkspace{}, fmt.Errorf("query user workspace failed: %w", err) + } + if user.Name == "" { + return resolvedWorkspace{}, fmt.Errorf("user workspace %s not found", name) + } + return resolvedWorkspace{ + WorkspaceType: workspaceType, + WorkspaceName: user.Name, + LogicalPath: normalizeUserWorkspacePath(user.Space), + }, nil + case model.StorageIndexWorkspaceTypeAccount: + if name == "" { + return resolvedWorkspace{}, fmt.Errorf("workspace_name is required for account workspace") + } + var account struct { + Name string `gorm:"column:name"` + Space string `gorm:"column:space"` + } + if err := query.GetDB().WithContext(ctx). + Raw("SELECT name, space FROM accounts WHERE name = ? AND deleted_at IS NULL", name). + Scan(&account).Error; err != nil { + return resolvedWorkspace{}, fmt.Errorf("query account workspace failed: %w", err) + } + if account.Name == "" { + return resolvedWorkspace{}, fmt.Errorf("account workspace %s not found", name) + } + return resolvedWorkspace{ + WorkspaceType: workspaceType, + WorkspaceName: account.Name, + LogicalPath: normalizeAccountWorkspacePath(account.Space), + }, nil + case model.StorageIndexWorkspaceTypePublic: + if name == "" { + name = "public" + } + return resolvedWorkspace{ + WorkspaceType: workspaceType, + WorkspaceName: name, + LogicalPath: "/public", + }, nil + default: + return resolvedWorkspace{}, fmt.Errorf("unsupported workspace type: %s", workspaceType) + } +} + +func normalizeUserWorkspacePath(space string) string { + cleaned := strings.TrimSpace(space) + if cleaned == "" { + return "" + } + if strings.HasPrefix(cleaned, "/user/") { + return path.Clean(cleaned) + } + return path.Clean("/user/" + strings.TrimPrefix(cleaned, "/")) +} + +func normalizeAccountWorkspacePath(space string) string { + cleaned := strings.TrimSpace(space) + if cleaned == "" { + return "" + } + if strings.HasPrefix(cleaned, "/") { + return path.Clean(cleaned) + } + return path.Clean("/account/" + strings.TrimPrefix(cleaned, "/")) +} + +type cephCSIWorkspacePath struct { + MountRoot string + GroupName string + SubvolumeName string + RelativeSuffix string + WorkspaceRoot string +} + +func parseCephCSIWorkspacePath(subvolumeRoot string, relativeSuffix string) (cephCSIWorkspacePath, bool) { + normalized := normalizeUnixPath(subvolumeRoot) + marker := "/volumes/" + idx := strings.Index(normalized, marker) + if idx < 0 { + return cephCSIWorkspacePath{}, false + } + + mountRoot := normalized[:idx] + suffix := strings.TrimPrefix(normalized[idx+len(marker):], "/") + parts := strings.Split(suffix, "/") + if len(parts) < 2 { + return cephCSIWorkspacePath{}, false + } + + return cephCSIWorkspacePath{ + MountRoot: mountRoot, + GroupName: parts[0], + SubvolumeName: parts[1], + RelativeSuffix: relativeSuffix, + WorkspaceRoot: path.Join(normalized, relativeSuffix), + }, true +} + +func (s *Service) resolveMaterializedSnapshotName( + toolboxPod *corev1.Pod, + mountRoot string, + subvolumeRoot string, + requestedSnapshotName string, +) (string, error) { + snapRoot := normalizeUnixPath(path.Join(mountRoot, subvolumeRoot, ".snap")) + out, err := ceph.ExecInPod( + s.kubeClient, + s.kubeConfig, + toolboxPod, + []string{"ls", "-1", snapRoot}, + ) + if err != nil { + return "", fmt.Errorf("list snapshot directory failed: %w", err) + } + + lines := strings.Split(strings.TrimSpace(out), "\n") + if len(lines) == 0 || (len(lines) == 1 && strings.TrimSpace(lines[0]) == "") { + return "", fmt.Errorf("snapshot directory %s is empty", snapRoot) + } + + exact := "" + prefixed := "" + contains := "" + for _, line := range lines { + name := strings.TrimSpace(line) + if name == "" || name == "." || name == ".." { + continue + } + if name == requestedSnapshotName { + exact = name + break + } + if strings.HasPrefix(name, "_"+requestedSnapshotName+"_") { + prefixed = name + } + if contains == "" && strings.Contains(name, requestedSnapshotName) { + contains = name + } + } + + switch { + case exact != "": + return exact, nil + case prefixed != "": + return prefixed, nil + case contains != "": + return contains, nil + default: + return "", fmt.Errorf("requested snapshot %s not materialized under %s (entries=%v)", requestedSnapshotName, snapRoot, lines) + } +} + +func (s *Service) findLatestCompletedScanID(ctx context.Context, workspace resolvedWorkspace) (string, error) { + var job model.StorageIndexScanJob + if err := query.GetDB().WithContext(ctx). + Where("workspace_type = ? AND workspace_name = ? AND status = ?", workspace.WorkspaceType, workspace.WorkspaceName, model.StorageIndexScanStatusDone). + Order("finished_at DESC, updated_at DESC"). + First(&job).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return "", nil + } + return "", fmt.Errorf("query latest completed metadata scan failed: %w", err) + } + return job.ScanID, nil +} + +func (s *Service) applyGrowthFromPreviousScan( + ctx context.Context, + workspace resolvedWorkspace, + baseScanID string, + dirMetrics []model.StorageIndexDirectoryMetric, +) (int64, error) { + if baseScanID == "" || len(dirMetrics) == 0 { + return 0, nil + } + + var previousMetrics []model.StorageIndexDirectoryMetric + if err := query.GetDB().WithContext(ctx). + Where("workspace_type = ? AND workspace_name = ? AND scan_id = ?", workspace.WorkspaceType, workspace.WorkspaceName, baseScanID). + Find(&previousMetrics).Error; err != nil { + return 0, fmt.Errorf("query previous directory metrics failed: %w", err) + } + + previousByPath := make(map[string]model.StorageIndexDirectoryMetric, len(previousMetrics)) + for _, metric := range previousMetrics { + previousByPath[metric.Path] = metric + } + + changedCount := int64(0) + for i := range dirMetrics { + current := &dirMetrics[i] + previous, ok := previousByPath[current.Path] + if !ok { + current.LatestGrowth = current.TotalSizeBytes + if current.TotalSizeBytes > 0 { + changedCount++ + } + continue + } + + current.LatestGrowth = current.TotalSizeBytes - previous.TotalSizeBytes + if current.LatestGrowth != 0 { + changedCount++ + } + } + + return changedCount, nil +} + +func (s *Service) getScanJobByID(ctx context.Context, scanID string) (*model.StorageIndexScanJob, error) { + var job model.StorageIndexScanJob + if err := query.GetDB().WithContext(ctx).Where("scan_id = ?", scanID).First(&job).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + return nil, fmt.Errorf("query metadata scan job %s failed: %w", scanID, err) + } + return &job, nil +} + +func (s *Service) resolveExistingSnapshotScanPath( + toolboxPod *corev1.Pod, + subvolumeRoot string, + rootPath string, + snapshotName string, +) (string, error) { + if snapshotName == "" { + return "", fmt.Errorf("snapshot name is empty") + } + + relativeSuffix := relativeUnixPath(subvolumeRoot, rootPath) + if csi, ok := parseCephCSIWorkspacePath(subvolumeRoot, relativeSuffix); ok { + out, err := ceph.ExecInPod( + s.kubeClient, + s.kubeConfig, + toolboxPod, + []string{"ceph", "fs", "subvolume", "getpath", cephFSVolumeName, csi.SubvolumeName, "--group_name", csi.GroupName}, + ) + if err != nil { + return "", fmt.Errorf("getpath for existing subvolume snapshot failed: %w", err) + } + + subvolumePath := strings.TrimSpace(out) + if subvolumePath == "" { + return "", fmt.Errorf("subvolume getpath returned empty path") + } + + materializedSnapshotName, err := s.resolveMaterializedSnapshotName(toolboxPod, csi.MountRoot, subvolumePath, snapshotName) + if err != nil { + return "", err + } + return normalizeUnixPath(path.Join(csi.MountRoot, subvolumePath, ".snap", materializedSnapshotName, csi.RelativeSuffix)), nil + } + + return normalizeUnixPath(path.Join(rootPath, ".snap", snapshotName)), nil +} + +func (s *Service) listImmediateSignatures( + toolboxPod *corev1.Pod, + scanID string, + workspaceType model.StorageIndexWorkspaceType, + workspaceName string, + logicalBase string, + scanRoot string, +) (map[string]topLevelSignature, error) { + script := fmt.Sprintf( + "find %s -mindepth 1 -maxdepth 1 -printf %s", + shellQuote(scanRoot), + shellQuote(`%y\037%s\037%T@\037%C@\037%U\037%G\037%m\037%n\037%p\0`), + ) + + output, err := ceph.ExecInPod( + s.kubeClient, + s.kubeConfig, + toolboxPod, + []string{"sh", "-c", script}, + ) + if err != nil { + return nil, fmt.Errorf("list top-level signatures failed: %w", err) + } + + signatures := make(map[string]topLevelSignature) + records := strings.Split(output, findRecordSeparator) + for _, record := range records { + if record == "" { + continue + } + fields := strings.Split(record, findFieldSeparator) + if len(fields) != 9 { + continue + } + + actualPath := normalizeUnixPath(fields[8]) + relativePath := relativeUnixPath(scanRoot, actualPath) + if relativePath == "." || relativePath == "" || strings.Contains(relativePath, "/") { + continue + } + + entryType := parseEntryType(fields[0]) + sizeBytes := parseInt64(fields[1]) + changedAt := parseUnixTimestamp(fields[3]) + if entryType == model.StorageIndexEntryTypeDir { + sizeBytes, err = s.getDirectoryRBytes(toolboxPod, actualPath) + if err != nil { + klog.Warningf("storageindex: 获取目录 ceph.dir.rbytes 失败 scan_id=%s path=%s err=%v", scanID, actualPath, err) + } + recursiveChangedAt, rctimeErr := s.getDirectoryRecursiveChangedAt(toolboxPod, actualPath) + if rctimeErr != nil { + klog.Warningf("storageindex: 获取目录 ceph.dir.rctime 失败，回退为 stat ctime scan_id=%s path=%s err=%v", scanID, actualPath, rctimeErr) + } else if recursiveChangedAt != nil { + changedAt = recursiveChangedAt + } + } + + signatures[relativePath] = topLevelSignature{ + Name: relativePath, + LogicalPath: path.Join(logicalBase, relativePath), + ParentLogicalPath: logicalBase, + ActualPath: actualPath, + EntryType: entryType, + SizeBytes: sizeBytes, + ModifiedAt: parseUnixTimestamp(fields[2]), + ChangedAt: changedAt, + OwnerUID: parseInt64(fields[4]), + OwnerGID: parseInt64(fields[5]), + Mode: strings.TrimSpace(fields[6]), + LinkCount: parseInt64(fields[7]), + } + } + + klog.Infof( + "storageindex: 直接子项签名采集完成 scan_id=%s workspace_type=%s workspace_name=%s logical_base=%s signature_count=%d", + scanID, workspaceType, workspaceName, logicalBase, len(signatures), + ) + return signatures, nil +} + +func (s *Service) getDirectoryRBytes(toolboxPod *corev1.Pod, actualPath string) (int64, error) { + output, err := ceph.ExecInPod( + s.kubeClient, + s.kubeConfig, + toolboxPod, + []string{"getfattr", "-n", "ceph.dir.rbytes", actualPath}, + ) + if err != nil { + return 0, err + } + + for _, line := range strings.Split(output, "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "ceph.dir.rbytes=") { + sizeStr := strings.Trim(strings.TrimPrefix(line, "ceph.dir.rbytes="), "\"") + size, parseErr := strconv.ParseInt(sizeStr, 10, 64) + if parseErr != nil { + return 0, fmt.Errorf("parse ceph.dir.rbytes failed: %w", parseErr) + } + return size, nil + } + } + + return 0, fmt.Errorf("ceph.dir.rbytes not found for %s", actualPath) +} + +func (s *Service) getDirectoryRecursiveChangedAt(toolboxPod *corev1.Pod, actualPath string) (*time.Time, error) { + output, err := ceph.ExecInPod( + s.kubeClient, + s.kubeConfig, + toolboxPod, + []string{"getfattr", "-n", "ceph.dir.rctime", actualPath}, + ) + if err != nil { + return nil, err + } + + for _, line := range strings.Split(output, "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "ceph.dir.rctime=") { + value := strings.Trim(strings.TrimPrefix(line, "ceph.dir.rctime="), "\"") + parsed := parseUnixTimestamp(value) + if parsed == nil { + return nil, fmt.Errorf("parse ceph.dir.rctime failed for %s: %q", actualPath, value) + } + return parsed, nil + } + } + + return nil, fmt.Errorf("ceph.dir.rctime not found for %s", actualPath) +} + +func diffTopLevelSignatures( + current map[string]topLevelSignature, + previous map[string]topLevelSignature, +) ([]topLevelSignature, []string, int64) { + changedCurrent := make([]topLevelSignature, 0) + removedPrefixes := make([]string, 0) + changedCount := int64(0) + + keys := make(map[string]struct{}, len(current)+len(previous)) + for key := range current { + keys[key] = struct{}{} + } + for key := range previous { + keys[key] = struct{}{} + } + + allKeys := make([]string, 0, len(keys)) + for key := range keys { + allKeys = append(allKeys, key) + } + sort.Strings(allKeys) + + for _, key := range allKeys { + cur, curOK := current[key] + prev, prevOK := previous[key] + switch { + case curOK && !prevOK: + changedCurrent = append(changedCurrent, cur) + changedCount++ + case !curOK && prevOK: + removedPrefixes = append(removedPrefixes, prev.LogicalPath) + changedCount++ + case curOK && prevOK: + if cur.EntryType != prev.EntryType { + changedCurrent = append(changedCurrent, cur) + changedCount++ + continue + } + if cur.EntryType == model.StorageIndexEntryTypeDir { + if timestampsDifferent(cur.ChangedAt, prev.ChangedAt) { + changedCurrent = append(changedCurrent, cur) + changedCount++ + } + continue + } + if cur.SizeBytes != prev.SizeBytes || + timestampsDifferent(cur.ModifiedAt, prev.ModifiedAt) || + timestampsDifferent(cur.ChangedAt, prev.ChangedAt) || + cur.Mode != prev.Mode || + cur.LinkCount != prev.LinkCount { + changedCurrent = append(changedCurrent, cur) + changedCount++ + } + } + } + + return changedCurrent, removedPrefixes, changedCount +} + +func signatureToEntry( + scanID string, + workspaceType model.StorageIndexWorkspaceType, + workspaceName string, + workspaceRoot string, + sig topLevelSignature, +) model.StorageIndexEntry { + relativePath := relativeLogicalPath(workspaceRoot, sig.LogicalPath) + isTopLevel := path.Clean(sig.ParentLogicalPath) == path.Clean(workspaceRoot) + return model.StorageIndexEntry{ + WorkspaceType: workspaceType, + WorkspaceName: workspaceName, + ScanID: scanID, + LogicalPath: sig.LogicalPath, + RelativePath: relativePath, + ParentPath: sig.ParentLogicalPath, + Name: sig.Name, + EntryType: sig.EntryType, + SizeBytes: sig.SizeBytes, + OwnerUID: sig.OwnerUID, + OwnerGID: sig.OwnerGID, + Mode: sig.Mode, + LinkCount: sig.LinkCount, + ModifiedAt: sig.ModifiedAt, + ChangedAt: sig.ChangedAt, + IsTopLevel: isTopLevel, + } +} + +func appendUniquePrefix(items []string, value string) []string { + for _, item := range items { + if item == value { + return items + } + } + return append(items, value) +} + +func appendUniqueEntry(items []model.StorageIndexEntry, entry model.StorageIndexEntry) []model.StorageIndexEntry { + for i, item := range items { + if item.LogicalPath == entry.LogicalPath { + items[i] = entry + return items + } + } + return append(items, entry) +} + +func appendUniqueSignature(items []topLevelSignature, sig topLevelSignature) []topLevelSignature { + for i, item := range items { + if item.LogicalPath == sig.LogicalPath { + items[i] = sig + return items + } + } + return append(items, sig) +} + +func (s *Service) buildIncrementalPlan( + toolboxPod *corev1.Pod, + scanID string, + workspace resolvedWorkspace, + currentLogicalBase string, + currentActualDir string, + previousActualDir string, + previousRecordedChangedAt map[string]*time.Time, +) (*incrementalPlan, int64, error) { + currentSigns, err := s.listImmediateSignatures(toolboxPod, scanID, workspace.WorkspaceType, workspace.WorkspaceName, currentLogicalBase, currentActualDir) + if err != nil { + return nil, 0, err + } + previousSigns, err := s.listImmediateSignatures(toolboxPod, scanID, workspace.WorkspaceType, workspace.WorkspaceName, currentLogicalBase, previousActualDir) + if err != nil { + return nil, 0, err + } + if shouldApplyTopLevelModelCopyPrefilter(workspace) && normalizeUnixPath(currentLogicalBase) == normalizeUnixPath(workspace.LogicalPath) { + filteredCurrent, skippedCurrent := filterTopLevelSignaturesForModelCopyScan(currentSigns) + filteredPrevious, skippedPrevious := filterTopLevelSignaturesForModelCopyScan(previousSigns) + currentSigns = make(map[string]topLevelSignature, len(filteredCurrent)) + for _, sig := range filteredCurrent { + currentSigns[path.Base(sig.LogicalPath)] = sig + } + previousSigns = make(map[string]topLevelSignature, len(filteredPrevious)) + for _, sig := range filteredPrevious { + previousSigns[path.Base(sig.LogicalPath)] = sig + } + + skippedNames := make([]string, 0, len(skippedCurrent)+len(skippedPrevious)) + seenSkipped := make(map[string]struct{}, len(skippedCurrent)+len(skippedPrevious)) + for _, sig := range skippedCurrent { + if _, ok := seenSkipped[sig.Name]; ok { + continue + } + seenSkipped[sig.Name] = struct{}{} + skippedNames = append(skippedNames, sig.Name) + } + for _, sig := range skippedPrevious { + if _, ok := seenSkipped[sig.Name]; ok { + continue + } + seenSkipped[sig.Name] = struct{}{} + skippedNames = append(skippedNames, sig.Name) + } + sort.Strings(skippedNames) + if len(skippedNames) > 0 { + klog.Infof( + "storageindex: 增量对比顶层目录过滤完成 scan_id=%s workspace_type=%s workspace_name=%s logical_base=%s skipped_top_level=%v", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, currentLogicalBase, skippedNames, + ) + } + } + plan := &incrementalPlan{ + RescanTargets: make([]topLevelSignature, 0), + UpsertEntries: make([]model.StorageIndexEntry, 0), + RemovedPrefixes: make([]string, 0), + } + + changedCount := int64(0) + keys := make(map[string]struct{}, len(currentSigns)+len(previousSigns)) + for key := range currentSigns { + keys[key] = struct{}{} + } + for key := range previousSigns { + keys[key] = struct{}{} + } + + allKeys := make([]string, 0, len(keys)) + for key := range keys { + allKeys = append(allKeys, key) + } + sort.Strings(allKeys) + plan.ComparedNodes += int64(len(allKeys)) + + for _, key := range allKeys { + cur, curOK := currentSigns[key] + prev, prevOK := previousSigns[key] + switch { + case curOK && !prevOK: + changedCount++ + plan.NewNodes++ + if cur.EntryType == model.StorageIndexEntryTypeDir { + plan.RescanTargets = appendUniqueSignature(plan.RescanTargets, cur) + } else { + plan.UpsertEntries = appendUniqueEntry(plan.UpsertEntries, signatureToEntry(scanID, workspace.WorkspaceType, workspace.WorkspaceName, workspace.LogicalPath, cur)) + } + case !curOK && prevOK: + changedCount++ + plan.RemovedNodes++ + plan.RemovedPrefixes = appendUniquePrefix(plan.RemovedPrefixes, prev.LogicalPath) + case curOK && prevOK: + if cur.EntryType != prev.EntryType { + changedCount++ + plan.UpdatedNodes++ + plan.RemovedPrefixes = appendUniquePrefix(plan.RemovedPrefixes, prev.LogicalPath) + if cur.EntryType == model.StorageIndexEntryTypeDir { + plan.RescanTargets = appendUniqueSignature(plan.RescanTargets, cur) + } else { + plan.UpsertEntries = appendUniqueEntry(plan.UpsertEntries, signatureToEntry(scanID, workspace.WorkspaceType, workspace.WorkspaceName, workspace.LogicalPath, cur)) + } + continue + } + + if cur.EntryType != model.StorageIndexEntryTypeDir { + if cur.SizeBytes != prev.SizeBytes || timestampsDifferent(cur.ModifiedAt, prev.ModifiedAt) || timestampsDifferent(cur.ChangedAt, prev.ChangedAt) || cur.Mode != prev.Mode || cur.LinkCount != prev.LinkCount { + changedCount++ + plan.UpdatedNodes++ + plan.UpsertEntries = appendUniqueEntry(plan.UpsertEntries, signatureToEntry(scanID, workspace.WorkspaceType, workspace.WorkspaceName, workspace.LogicalPath, cur)) + } else { + plan.ReusedNodes++ + } + continue + } + + recorded, ok := previousRecordedChangedAt[normalizeUnixPath(prev.LogicalPath)] + if !ok || recorded == nil { + return nil, 0, fmt.Errorf("missing recorded directory rctime for %s", prev.LogicalPath) + } + prev.ChangedAt = recorded + + dirChanged := timestampsDifferent(cur.ChangedAt, prev.ChangedAt) + klog.Infof( + "storageindex: 目录签名比较 scan_id=%s workspace_type=%s workspace_name=%s path=%s current_rbytes=%d previous_rbytes=%d current_rctime=%s previous_rctime=%s previous_rctime_source=db recurse=%t", + scanID, + workspace.WorkspaceType, + workspace.WorkspaceName, + cur.LogicalPath, + cur.SizeBytes, + prev.SizeBytes, + formatTimeForLog(cur.ChangedAt), + formatTimeForLog(prev.ChangedAt), + dirChanged, + ) + + if !dirChanged { + plan.ReusedNodes++ + plan.PrunedDirs++ + continue + } + + childPlan, childChanged, childErr := s.buildIncrementalPlan( + toolboxPod, + scanID, + workspace, + cur.LogicalPath, + cur.ActualPath, + prev.ActualPath, + previousRecordedChangedAt, + ) + if childErr != nil { + return nil, 0, childErr + } + + changedCount += childChanged + plan.ComparedNodes += childPlan.ComparedNodes + plan.PrunedDirs += childPlan.PrunedDirs + plan.NewNodes += childPlan.NewNodes + plan.UpdatedNodes += childPlan.UpdatedNodes + plan.RemovedNodes += childPlan.RemovedNodes + plan.ReusedNodes += childPlan.ReusedNodes + if childChanged == 0 { + plan.ReusedNodes++ + continue + } + plan.UpdatedNodes++ + plan.UpsertEntries = appendUniqueEntry(plan.UpsertEntries, signatureToEntry(scanID, workspace.WorkspaceType, workspace.WorkspaceName, workspace.LogicalPath, cur)) + for _, target := range childPlan.RescanTargets { + plan.RescanTargets = appendUniqueSignature(plan.RescanTargets, target) + } + for _, entry := range childPlan.UpsertEntries { + plan.UpsertEntries = appendUniqueEntry(plan.UpsertEntries, entry) + } + for _, removed := range childPlan.RemovedPrefixes { + plan.RemovedPrefixes = appendUniquePrefix(plan.RemovedPrefixes, removed) + } + } + } + + return plan, changedCount, nil +} + +func timestampsDifferent(a, b *time.Time) bool { + switch { + case a == nil && b == nil: + return false + case a == nil || b == nil: + return true + default: + return !a.Equal(*b) + } +} + +func (s *Service) scanWorkspaceEntries( + toolboxPod *corev1.Pod, + scanID string, + workspace resolvedWorkspace, + scanRoot string, +) ([]model.StorageIndexEntry, error) { + return s.scanPathDirectories(toolboxPod, scanID, workspace.WorkspaceType, workspace.WorkspaceName, workspace.LogicalPath, scanRoot) +} + +func (s *Service) loadDirectorySignatureAtPath( + toolboxPod *corev1.Pod, + _ string, + _ model.StorageIndexWorkspaceType, + _ string, + logicalPath string, + parentLogicalPath string, + actualPath string, +) (*topLevelSignature, error) { + sizeBytes, err := s.getDirectoryRBytes(toolboxPod, actualPath) + if err != nil { + return nil, err + } + changedAt, err := s.getDirectoryRecursiveChangedAt(toolboxPod, actualPath) + if err != nil { + return nil, err + } + + script := fmt.Sprintf( + "find %s -mindepth 0 -maxdepth 0 -printf %s", + shellQuote(actualPath), + shellQuote(`%T@\037%U\037%G\037%m\037%n\037%p\0`), + ) + output, err := ceph.ExecInPod( + s.kubeClient, + s.kubeConfig, + toolboxPod, + []string{"sh", "-c", script}, + ) + if err != nil { + return nil, fmt.Errorf("load directory signature at path failed: %w", err) + } + + records := strings.Split(output, findRecordSeparator) + for _, record := range records { + if record == "" { + continue + } + fields := strings.Split(record, findFieldSeparator) + if len(fields) != 6 { + continue + } + return &topLevelSignature{ + Name: path.Base(logicalPath), + LogicalPath: normalizeUnixPath(logicalPath), + ParentLogicalPath: normalizeUnixPath(parentLogicalPath), + ActualPath: normalizeUnixPath(actualPath), + EntryType: model.StorageIndexEntryTypeDir, + SizeBytes: sizeBytes, + ModifiedAt: parseUnixTimestamp(fields[0]), + ChangedAt: changedAt, + OwnerUID: parseInt64(fields[1]), + OwnerGID: parseInt64(fields[2]), + Mode: strings.TrimSpace(fields[3]), + LinkCount: parseInt64(fields[4]), + }, nil + } + + return nil, fmt.Errorf("directory signature output missing for %s", actualPath) +} + +func buildDirectoryScanScript(scanRoot string, pruneActualPaths []string) string { + pruneMatchers := []string{ + fmt.Sprintf("-path %s", shellQuote(path.Join(scanRoot, ".snap"))), + } + for _, name := range sortedPrunableNestedDirNames() { + pruneMatchers = append(pruneMatchers, fmt.Sprintf("-name %s", shellQuote(name))) + } + for _, prunePath := range pruneActualPaths { + normalized := normalizeUnixPath(prunePath) + if normalized == "" || normalized == "." { + continue + } + pruneMatchers = append(pruneMatchers, fmt.Sprintf("-path %s", shellQuote(normalized))) + } + + return fmt.Sprintf( + `find %s \( %s \) -prune -o -type d -print0 | xargs -0 -r -n 32 sh -c 'for d in "$@"; do m=$(stat -c %%Y "$d" 2>/dev/null || echo 0); s=$(getfattr --only-values -n ceph.dir.rbytes "$d" 2>/dev/null || echo 0); rc=$(getfattr --only-values -n ceph.dir.rctime "$d" 2>/dev/null || stat -c %%Z "$d" 2>/dev/null || echo 0); printf "%%s\037%%s\037%%s\037%%s\0" "$m" "$s" "$rc" "$d"; done' sh`, + shellQuote(scanRoot), + strings.Join(pruneMatchers, " -o "), + ) +} + +func sortedPrunableNestedDirNames() []string { + names := make([]string, 0, len(definitelyNotPublicModelCopyNestedDirSet)) + for name := range definitelyNotPublicModelCopyNestedDirSet { + names = append(names, name) + } + sort.Strings(names) + return names +} + +func (s *Service) scanPathDirectories( + toolboxPod *corev1.Pod, + scanID string, + workspaceType model.StorageIndexWorkspaceType, + workspaceName string, + logicalBase string, + scanRoot string, +) ([]model.StorageIndexEntry, error) { + return s.scanPathDirectoriesWithPrunedChildren( + toolboxPod, + scanID, + workspaceType, + workspaceName, + logicalBase, + scanRoot, + nil, + ) +} + +func (s *Service) scanPathDirectoriesWithPrunedChildren( + toolboxPod *corev1.Pod, + scanID string, + workspaceType model.StorageIndexWorkspaceType, + workspaceName string, + logicalBase string, + scanRoot string, + pruneActualPaths []string, +) ([]model.StorageIndexEntry, error) { + klog.Infof( + "storageindex: 正在执行目录骨架扫描 scan_id=%s workspace_type=%s workspace_name=%s logical_base=%s scan_root=%s", + scanID, workspaceType, workspaceName, logicalBase, scanRoot, + ) + + script := buildDirectoryScanScript(scanRoot, pruneActualPaths) + + var stdout strings.Builder + var stderr strings.Builder + progress := &findProgressWriter{ + scanID: scanID, + workspaceType: workspaceType, + workspaceName: workspaceName, + everyRecords: scanProgressLogEveryRecord, + } + + err := ceph.ExecInPodStream( + s.kubeClient, + s.kubeConfig, + toolboxPod, + []string{"sh", "-c", script}, + io.MultiWriter(&stdout, progress), + &stderr, + ) + if err != nil { + return nil, fmt.Errorf("scan workspace directories failed: %w, stderr: %s", err, stderr.String()) + } + + records := strings.Split(stdout.String(), findRecordSeparator) + klog.Infof( + "storageindex: 目录骨架扫描完成 scan_id=%s workspace_type=%s workspace_name=%s logical_base=%s raw_record_count=%d stderr_len=%d", + scanID, workspaceType, workspaceName, logicalBase, len(records), stderr.Len(), + ) + + entries := make([]model.StorageIndexEntry, 0, len(records)) + for _, record := range records { + if record == "" { + continue + } + fields := strings.Split(record, findFieldSeparator) + if len(fields) != 4 { + continue + } + + fullPath := normalizeUnixPath(fields[3]) + relativePath := relativeUnixPath(scanRoot, fullPath) + logicalPath := logicalBase + if relativePath != "." { + logicalPath = path.Join(logicalBase, relativePath) + } + parentPath := "" + if logicalPath != logicalBase { + parentPath = path.Dir(logicalPath) + if parentPath == "." { + parentPath = logicalBase + } + } + + entry := model.StorageIndexEntry{ + WorkspaceType: workspaceType, + WorkspaceName: workspaceName, + ScanID: scanID, + LogicalPath: logicalPath, + RelativePath: relativeLogicalPath(logicalBase, logicalPath), + ParentPath: parentPath, + Name: path.Base(logicalPath), + EntryType: model.StorageIndexEntryTypeDir, + SizeBytes: parseInt64(fields[1]), + ModifiedAt: parseUnixTimestamp(fields[0]), + ChangedAt: parseUnixTimestamp(fields[2]), + IsTopLevel: isTopLevelRelative(relativeLogicalPath(logicalBase, logicalPath)), + } + entries = append(entries, entry) + } + + sort.Slice(entries, func(i, j int) bool { + return entries[i].LogicalPath < entries[j].LogicalPath + }) + + klog.Infof( + "storageindex: 目录骨架规范化完成 scan_id=%s workspace_type=%s workspace_name=%s logical_base=%s dir_count=%d", + scanID, workspaceType, workspaceName, logicalBase, len(entries), + ) + return entries, nil +} + +func (s *Service) scanPathEntries( + toolboxPod *corev1.Pod, + scanID string, + workspaceType model.StorageIndexWorkspaceType, + workspaceName string, + logicalBase string, + scanRoot string, +) ([]model.StorageIndexEntry, error) { + klog.Infof( + "storageindex: 正在执行 find 扫描 scan_id=%s workspace_type=%s workspace_name=%s logical_base=%s scan_root=%s", + scanID, workspaceType, workspaceName, logicalBase, scanRoot, + ) + + script := fmt.Sprintf( + "find %s \\( %s \\) -prune -o -printf %s", + shellQuote(scanRoot), + strings.Join(append([]string{fmt.Sprintf("-path %s", shellQuote(path.Join(scanRoot, ".snap")))}, func() []string { + items := make([]string, 0, len(sortedPrunableNestedDirNames())) + for _, name := range sortedPrunableNestedDirNames() { + items = append(items, fmt.Sprintf("-name %s", shellQuote(name))) + } + return items + }()...), " -o "), + shellQuote(`%y\037%i\037%s\037%T@\037%C@\037%A@\037%U\037%G\037%m\037%n\037%p\0`), + ) + + var stdout strings.Builder + var stderr strings.Builder + progress := &findProgressWriter{ + scanID: scanID, + workspaceType: workspaceType, + workspaceName: workspaceName, + everyRecords: scanProgressLogEveryRecord, + } + + err := ceph.ExecInPodStream( + s.kubeClient, + s.kubeConfig, + toolboxPod, + []string{"sh", "-c", script}, + io.MultiWriter(&stdout, progress), + &stderr, + ) + if err != nil { + return nil, fmt.Errorf("scan workspace entries failed: %w, stderr: %s", err, stderr.String()) + } + + output := stdout.String() + records := strings.Split(output, findRecordSeparator) + klog.Infof( + "storageindex: find 扫描完成 scan_id=%s workspace_type=%s workspace_name=%s logical_base=%s raw_record_count=%d stderr_len=%d", + scanID, workspaceType, workspaceName, logicalBase, len(records), stderr.Len(), + ) + entries := make([]model.StorageIndexEntry, 0, len(records)) + + for _, record := range records { + if record == "" { + continue + } + fields := strings.Split(record, findFieldSeparator) + if len(fields) != 11 { + continue + } + + fullPath := normalizeUnixPath(fields[10]) + relativePath := relativeUnixPath(scanRoot, fullPath) + logicalPath := logicalBase + if relativePath != "." { + logicalPath = path.Join(logicalBase, relativePath) + } + + parentPath := "" + if logicalPath != logicalBase { + parentPath = path.Dir(logicalPath) + if parentPath == "." { + parentPath = logicalBase + } + } + + entry := model.StorageIndexEntry{ + WorkspaceType: workspaceType, + WorkspaceName: workspaceName, + ScanID: scanID, + LogicalPath: logicalPath, + RelativePath: relativePath, + ParentPath: parentPath, + Name: path.Base(logicalPath), + EntryType: parseEntryType(fields[0]), + SizeBytes: parseInt64(fields[2]), + OwnerUID: parseInt64(fields[6]), + OwnerGID: parseInt64(fields[7]), + Mode: strings.TrimSpace(fields[8]), + LinkCount: parseInt64(fields[9]), + ModifiedAt: parseUnixTimestamp(fields[3]), + ChangedAt: parseUnixTimestamp(fields[4]), + AccessedAt: parseUnixTimestamp(fields[5]), + IsTopLevel: isTopLevelRelative(relativePath), + } + entries = append(entries, entry) + } + + sort.Slice(entries, func(i, j int) bool { + return entries[i].LogicalPath < entries[j].LogicalPath + }) + + klog.Infof( + "storageindex: 规范化条目完成 scan_id=%s workspace_type=%s workspace_name=%s logical_base=%s normalized_entry_count=%d", + scanID, workspaceType, workspaceName, logicalBase, len(entries), + ) + + return entries, nil +} + +func buildDirectoryMetrics( + scanID string, + workspace resolvedWorkspace, + entries []model.StorageIndexEntry, +) ([]model.StorageIndexDirectoryMetric, int64) { + metricsByPath := make(map[string]*model.StorageIndexDirectoryMetric) + immediateChildDirs := make(map[string][]string) + immediateChildFileCount := make(map[string]int64) + latestModifiedByPath := make(map[string]*time.Time) + + ensureMetric := func(entryPath, parentPath, name string, depth int, isTopLevel bool) *model.StorageIndexDirectoryMetric { + if metric, ok := metricsByPath[entryPath]; ok { + return metric + } + metric := &model.StorageIndexDirectoryMetric{ + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + ScanID: scanID, + Path: entryPath, + ParentPath: parentPath, + Name: name, + Depth: depth, + IsTopLevel: isTopLevel, + } + metricsByPath[entryPath] = metric + return metric + } + + ensureMetric(workspace.LogicalPath, "", path.Base(workspace.LogicalPath), 0, false) + + for _, entry := range entries { + if entry.EntryType == model.StorageIndexEntryTypeDir { + metric := ensureMetric( + entry.LogicalPath, + entry.ParentPath, + entry.Name, + depthFromRelative(entry.RelativePath), + entry.IsTopLevel, + ) + metric.TotalSizeBytes = entry.SizeBytes + if entry.ParentPath != "" { + immediateChildDirs[entry.ParentPath] = append(immediateChildDirs[entry.ParentPath], entry.Name) + } + if entry.ModifiedAt != nil { + latestModifiedByPath[entry.LogicalPath] = entry.ModifiedAt + } + } + } + + for _, entry := range entries { + switch entry.EntryType { + case model.StorageIndexEntryTypeFile: + startPath := entry.ParentPath + if startPath == "" { + startPath = workspace.LogicalPath + } + for _, ancestor := range ancestorDirectories(startPath, workspace.LogicalPath) { + metric := ensureMetric(ancestor, parentForDirectory(ancestor, workspace.LogicalPath), path.Base(ancestor), depthFromRoot(ancestor, workspace.LogicalPath), isTopLevelPath(ancestor, workspace.LogicalPath)) + metric.TotalSizeBytes += entry.SizeBytes + metric.FileCount++ + } + case model.StorageIndexEntryTypeDir: + if entry.LogicalPath == workspace.LogicalPath { + continue + } + for _, ancestor := range ancestorDirectories(path.Dir(entry.LogicalPath), workspace.LogicalPath) { + metric := ensureMetric(ancestor, parentForDirectory(ancestor, workspace.LogicalPath), path.Base(ancestor), depthFromRoot(ancestor, workspace.LogicalPath), isTopLevelPath(ancestor, workspace.LogicalPath)) + metric.DirectoryCount++ + } + } + } + + metrics := make([]model.StorageIndexDirectoryMetric, 0, len(metricsByPath)) + rootTotal := int64(0) + for _, metric := range metricsByPath { + childDirs := immediateChildDirs[metric.Path] + sort.Strings(childDirs) + metric.ImmediateChildDirCount = int64(len(childDirs)) + metric.ImmediateChildFileCount = immediateChildFileCount[metric.Path] + metric.LatestModifiedAt = latestModifiedByPath[metric.Path] + metric.Signature = buildDirectorySignature(metric, childDirs) + metric.CategoryHint = classifyDirectory(metric.Path) + metric.CandidateScore = computeCandidateScore(metric) + metrics = append(metrics, *metric) + if metric.Path == workspace.LogicalPath { + rootTotal = metric.TotalSizeBytes + } + } + + sort.Slice(metrics, func(i, j int) bool { + return metrics[i].Path < metrics[j].Path + }) + + return metrics, rootTotal +} + +func (s *Service) detectRedundancy( + ctx context.Context, + toolboxPod *corev1.Pod, + prefixConfig ceph.StoragePrefixConfig, + scanID string, + workspace resolvedWorkspace, + entries []model.StorageIndexEntry, + dirMetrics []model.StorageIndexDirectoryMetric, +) ([]model.StorageIndexRedundancyHit, error) { + _ = toolboxPod + _ = prefixConfig + _ = entries + if workspace.WorkspaceType == model.StorageIndexWorkspaceTypePublic { + return nil, nil + } + + db := query.GetDB().WithContext(ctx) + + var publicDirs []model.StorageIndexDirectoryMetric + if err := db. + Where("workspace_type = ? AND is_top_level = ?", model.StorageIndexWorkspaceTypePublic, true). + Find(&publicDirs).Error; err != nil { + return nil, fmt.Errorf("query public directory baseline failed: %w", err) + } + if len(publicDirs) == 0 { + return nil, nil + } + + dirBaseline := make(map[string][]model.StorageIndexDirectoryMetric) + for _, item := range publicDirs { + key := redundancyDirectoryKey(item.Name, item.TotalSizeBytes) + dirBaseline[key] = append(dirBaseline[key], item) + } + + hits := make([]model.StorageIndexRedundancyHit, 0) + seen := make(map[string]struct{}) + + for _, metric := range dirMetrics { + if !metric.IsTopLevel || metric.TotalSizeBytes <= 0 || metric.Path == workspace.LogicalPath { + continue + } + key := redundancyDirectoryKey(metric.Name, metric.TotalSizeBytes) + candidates := dirBaseline[key] + if len(candidates) == 0 { + continue + } + publicMetric := candidates[0] + hitKey := metric.Path + "->" + publicMetric.Path + if _, ok := seen[hitKey]; ok { + continue + } + seen[hitKey] = struct{}{} + hits = append(hits, model.StorageIndexRedundancyHit{ + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + ScanID: scanID, + TargetType: model.StorageIndexRedundancyTargetTypeDirectory, + TargetPath: metric.Path, + PublicPath: publicMetric.Path, + MatchKey: key, + Evidence: "目录名与目录总大小匹配公共空间基线，疑似重复保存的模型或数据集目录", + Confidence: redundancyConfidenceHigh, + VerificationStatus: model.StorageIndexVerificationStatusSuspected, + VerificationMode: verificationModeMetadata, + EstimatedBytes: metric.TotalSizeBytes, + }) + } + + sort.Slice(hits, func(i, j int) bool { + if hits[i].EstimatedBytes == hits[j].EstimatedBytes { + return hits[i].TargetPath < hits[j].TargetPath + } + return hits[i].EstimatedBytes > hits[j].EstimatedBytes + }) + + return hits, nil +} + +func (s *Service) buildCandidates( + ctx context.Context, + toolboxPod *corev1.Pod, + prefixConfig ceph.StoragePrefixConfig, + scanID string, + workspace resolvedWorkspace, + dirMetrics []model.StorageIndexDirectoryMetric, + hits []model.StorageIndexRedundancyHit, + existingCandidates []model.StorageIndexCandidate, +) ([]model.StorageIndexCandidate, []model.StorageIndexCandidateFile, []model.StorageIndexRedundancyHit, error) { + if workspace.WorkspaceType == model.StorageIndexWorkspaceTypePublic { + return nil, nil, nil, nil + } + + var publicRoots []model.StorageIndexPublicRootBaseline + if err := query.GetDB().WithContext(ctx). + Where("category <> ''"). + Find(&publicRoots).Error; err != nil { + return nil, nil, nil, fmt.Errorf("query public root baseline failed: %w", err) + } + if len(publicRoots) == 0 { + return nil, nil, nil, nil + } + publicRootLookup := buildPublicRootLookup(publicRoots) + + candidates := make([]model.StorageIndexCandidate, 0) + sort.Slice(dirMetrics, func(i, j int) bool { + if dirMetrics[i].Depth == dirMetrics[j].Depth { + return dirMetrics[i].Path < dirMetrics[j].Path + } + return dirMetrics[i].Depth < dirMetrics[j].Depth + }) + matchedCandidateRoots := make([]string, 0) + skippedCandidateRoots := make([]string, 0) + for _, metric := range dirMetrics { + if metric.Path == workspace.LogicalPath { + continue + } + if isCoveredByPrefixes(metric.Path, skippedCandidateRoots) { + continue + } + if shouldSkipTopLevelModelCopyCandidate(metric) { + skippedCandidateRoots = appendUniquePrefix(skippedCandidateRoots, metric.Path) + continue + } + if isCoveredByPrefixes(metric.Path, matchedCandidateRoots) { + continue + } + + matchedPublicPath := "" + evidence := "" + score := metric.CandidateScore + for _, publicRoot := range publicRootLookup[strings.ToLower(strings.TrimSpace(metric.Name))] { + if metric.TotalSizeBytes <= 0 { + continue + } + if !roughlySameSize(metric.TotalSizeBytes, publicRoot.TotalSizeBytes) { + continue + } + if publicRoot.Category == metric.CategoryHint || metric.CategoryHint == "" { + matchedPublicPath = publicRoot.LogicalPath + evidence = "目录名称与大小接近公共空间基线，疑似为冗余目录" + if score < 80 { + score = 80 + } + break + } + } + + if matchedPublicPath == "" { + continue + } + if score < 10 { + continue + } + if evidence == "" { + evidence = "目录类别提示命中，作为冗余候选目录保留待验证" + } + + candidate := model.StorageIndexCandidate{ + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + ScanID: scanID, + CandidateType: loCoalesce(metric.CategoryHint, inferCandidateTypeFromPublicPath(matchedPublicPath)), + TargetPath: metric.Path, + PublicPath: matchedPublicPath, + Evidence: evidence, + CandidateScore: score, + Status: model.StorageIndexCandidateStatusSuspected, + } + candidates = append(candidates, candidate) + if matchedPublicPath != "" { + matchedCandidateRoots = appendUniquePrefix(matchedCandidateRoots, candidate.TargetPath) + } + } + candidates = mergeExistingCandidateBindings(scanID, workspace, candidates, existingCandidates) + + candidateFiles := make([]model.StorageIndexCandidateFile, 0) + verifiedFileHits := make([]model.StorageIndexRedundancyHit, 0) + verifyEligibleCount := 0 + filteredCandidates := make([]model.StorageIndexCandidate, 0, len(candidates)) + for i := range candidates { + if candidates[i].PublicPath == "" { + continue + } + verifyEligibleCount++ + files, verifiedHits, err := s.verifyCandidateDirectory( + toolboxPod, + prefixConfig, + scanID, + workspace, + candidates[i].TargetPath, + candidates[i].PublicPath, + candidates[i].CandidateType, + ) + if err != nil { + klog.Warningf( + "storageindex: 候选目录关键文件校验失败 scan_id=%s candidate=%s public=%s err=%v", + scanID, candidates[i].TargetPath, candidates[i].PublicPath, err, + ) + continue + } + candidateFiles = append(candidateFiles, files...) + if !allCandidateFilesVerified(files) { + klog.Infof( + "storageindex: 候选目录校验未通过，保留为疑似候选 scan_id=%s workspace_type=%s workspace_name=%s candidate=%s public=%s matched_file_count=%d verified_hit_count=%d", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, candidates[i].TargetPath, candidates[i].PublicPath, len(files), len(verifiedHits), + ) + if len(files) == 0 { + candidates[i].Evidence = "候选目录命中公共基线，但当前目录内容与公共目录不再完全一致" + } else { + candidates[i].Evidence = "候选目录命中公共基线，但关键文件未全部校验通过" + } + filteredCandidates = append(filteredCandidates, candidates[i]) + continue + } + verifiedFileHits = append(verifiedFileHits, verifiedHits...) + candidates[i].Status = model.StorageIndexCandidateStatusVerified + candidates[i].Evidence = "候选目录中的关键文件与公共空间资源哈希一致，确认存在冗余" + if candidates[i].CandidateScore < 100 { + candidates[i].CandidateScore = 100 + } + hits = append(hits, verifiedHits...) + filteredCandidates = append(filteredCandidates, candidates[i]) + } + candidates = filteredCandidates + + sort.Slice(candidates, func(i, j int) bool { + if candidates[i].CandidateScore == candidates[j].CandidateScore { + return candidates[i].TargetPath < candidates[j].TargetPath + } + return candidates[i].CandidateScore > candidates[j].CandidateScore + }) + + klog.Infof( + "storageindex: 候选目录识别完成 scan_id=%s workspace_type=%s workspace_name=%s candidate_count=%d candidate_file_count=%d", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, len(candidates), len(candidateFiles), + ) + klog.Infof( + "storageindex: candidate rebuild stats scan_id=%s workspace_type=%s workspace_name=%s candidate_count=%d verify_eligible_count=%d candidate_file_count=%d verified_hit_count=%d", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, len(candidates), verifyEligibleCount, len(candidateFiles), len(verifiedFileHits), + ) + return candidates, candidateFiles, verifiedFileHits, nil +} + +func mergeExistingCandidateBindings( + scanID string, + workspace resolvedWorkspace, + candidates []model.StorageIndexCandidate, + existingCandidates []model.StorageIndexCandidate, +) []model.StorageIndexCandidate { + if len(existingCandidates) == 0 { + return candidates + } + + indexByPath := make(map[string]int, len(candidates)) + for i := range candidates { + indexByPath[normalizeUnixPath(candidates[i].TargetPath)] = i + } + + for _, existing := range existingCandidates { + targetPath := normalizeUnixPath(existing.TargetPath) + publicPath := normalizeUnixPath(existing.PublicPath) + if targetPath == "" || publicPath == "" { + continue + } + + if idx, ok := indexByPath[targetPath]; ok { + if strings.TrimSpace(candidates[idx].PublicPath) == "" { + candidates[idx].PublicPath = publicPath + } + if strings.TrimSpace(candidates[idx].CandidateType) == "" { + candidates[idx].CandidateType = existing.CandidateType + } + if strings.TrimSpace(candidates[idx].Evidence) == "" { + candidates[idx].Evidence = existing.Evidence + } + if candidates[idx].CandidateScore < existing.CandidateScore { + candidates[idx].CandidateScore = existing.CandidateScore + } + continue + } + + evidence := strings.TrimSpace(existing.Evidence) + if evidence == "" { + evidence = "沿用上一次候选命中的公共基线，执行增量重校验" + } + candidateType := strings.TrimSpace(existing.CandidateType) + if candidateType == "" { + candidateType = inferCandidateTypeFromPublicPath(publicPath) + } + candidates = append(candidates, model.StorageIndexCandidate{ + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + ScanID: scanID, + CandidateType: candidateType, + TargetPath: targetPath, + PublicPath: publicPath, + Evidence: evidence, + CandidateScore: existing.CandidateScore, + Status: model.StorageIndexCandidateStatusSuspected, + }) + indexByPath[targetPath] = len(candidates) - 1 + } + + return candidates +} + +func allCandidateFilesVerified(files []model.StorageIndexCandidateFile) bool { + if len(files) == 0 { + return false + } + for _, file := range files { + if file.VerificationStatus != model.StorageIndexVerificationStatusVerified { + return false + } + } + return true +} + +func (s *Service) rebuildCandidates( + ctx context.Context, + scanID string, + workspace resolvedWorkspace, +) error { + db := query.GetDB().WithContext(ctx) + cfg := config.GetConfig() + prefixConfig := ceph.StoragePrefixConfig{ + User: cfg.Storage.Prefix.User, + Account: cfg.Storage.Prefix.Account, + Public: cfg.Storage.Prefix.Public, + } + toolboxPod, err := ceph.FindCephToolboxPod(s.kubeClient, toolboxNamespace) + if err != nil { + return fmt.Errorf("find ceph toolbox pod for candidate rebuild failed: %w", err) + } + + var dirMetrics []model.StorageIndexDirectoryMetric + if err := db. + Where("workspace_type = ? AND workspace_name = ?", workspace.WorkspaceType, workspace.WorkspaceName). + Find(&dirMetrics).Error; err != nil { + return fmt.Errorf("query workspace directory metrics failed: %w", err) + } + + var hits []model.StorageIndexRedundancyHit + if err := db. + Where("workspace_type = ? AND workspace_name = ?", workspace.WorkspaceType, workspace.WorkspaceName). + Find(&hits).Error; err != nil { + return fmt.Errorf("query workspace redundancy hits failed: %w", err) + } + + candidates, candidateFiles, verifiedHits, err := s.buildCandidates(ctx, toolboxPod, prefixConfig, scanID, workspace, dirMetrics, hits, nil) + if err != nil { + return err + } + allHits := cloneRedundancyHitsForScan(scanID, hits) + allHits = append(allHits, verifiedHits...) + + return db.Transaction(func(tx *gorm.DB) error { + if err := tx.Where("workspace_type = ? AND workspace_name = ?", workspace.WorkspaceType, workspace.WorkspaceName). + Delete(&model.StorageIndexRedundancyHit{}).Error; err != nil { + return err + } + if err := tx.Where("workspace_type = ? AND workspace_name = ?", workspace.WorkspaceType, workspace.WorkspaceName). + Delete(&model.StorageIndexCandidate{}).Error; err != nil { + return err + } + if err := tx.Where("workspace_type = ? AND workspace_name = ?", workspace.WorkspaceType, workspace.WorkspaceName). + Delete(&model.StorageIndexCandidateFile{}).Error; err != nil { + return err + } + if len(allHits) > 0 { + if err := insertRedundancyHitsInChunks(tx, scanID, workspace, allHits, insertBatchSize); err != nil { + return err + } + } + if len(candidates) > 0 { + if err := insertCandidatesInChunks(tx, scanID, workspace, candidates, insertBatchSize); err != nil { + return err + } + } + if len(candidateFiles) > 0 { + if err := insertCandidateFilesInChunks(tx, scanID, workspace, candidateFiles, insertBatchSize); err != nil { + return err + } + } + return nil + }) +} + +func (s *Service) refreshIncrementalDerivedState( + ctx context.Context, + scanID string, + workspace resolvedWorkspace, + result *incrementalCollectResult, +) error { + if workspace.WorkspaceType == model.StorageIndexWorkspaceTypePublic { + return nil + } + + affectedMetricPaths := collectAffectedMetricPaths(workspace.LogicalPath, result.ChangedPrefixes, result.RemovedPrefixes, result.NewEntries) + prefixesToDelete := append([]string{}, result.ChangedPrefixes...) + prefixesToDelete = append(prefixesToDelete, result.RemovedPrefixes...) + + affectedCandidates, err := listWorkspaceCandidates(ctx, workspace) + if err != nil { + return err + } + affectedCandidatePaths := collectAffectedCandidatePaths(affectedCandidates, affectedMetricPaths, prefixesToDelete) + + dirMetrics := make([]model.StorageIndexDirectoryMetric, 0, len(affectedMetricPaths)) + if len(affectedMetricPaths) > 0 { + if err := query.GetDB().WithContext(ctx). + Where("workspace_type = ? AND workspace_name = ? AND path IN ?", workspace.WorkspaceType, workspace.WorkspaceName, affectedMetricPaths). + Find(&dirMetrics).Error; err != nil { + return fmt.Errorf("query affected directory metrics failed: %w", err) + } + } + + cfg := config.GetConfig() + prefixConfig := ceph.StoragePrefixConfig{ + User: cfg.Storage.Prefix.User, + Account: cfg.Storage.Prefix.Account, + Public: cfg.Storage.Prefix.Public, + } + toolboxPod, err := ceph.FindCephToolboxPod(s.kubeClient, toolboxNamespace) + if err != nil { + return fmt.Errorf("find ceph toolbox pod for incremental derived state failed: %w", err) + } + + directoryHits, err := s.detectRedundancy(ctx, toolboxPod, prefixConfig, scanID, workspace, nil, dirMetrics) + if err != nil { + return err + } + candidates, candidateFiles, verifiedHits, err := s.buildCandidates(ctx, toolboxPod, prefixConfig, scanID, workspace, dirMetrics, nil, affectedCandidates) + if err != nil { + return err + } + + allHits := append([]model.StorageIndexRedundancyHit{}, directoryHits...) + allHits = append(allHits, verifiedHits...) + + klog.Infof( + "storageindex: 增量派生结果刷新准备完成 scan_id=%s workspace_type=%s workspace_name=%s metric_count=%d affected_candidate_count=%d new_candidate_count=%d new_candidate_file_count=%d new_hit_count=%d", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, len(dirMetrics), len(affectedCandidatePaths), len(candidates), len(candidateFiles), len(allHits), + ) + + return query.GetDB().WithContext(ctx).Transaction(func(tx *gorm.DB) error { + if err := deleteWorkspaceRedundancyHitsForExactPaths(tx, workspace, affectedMetricPaths); err != nil { + return err + } + if err := deleteWorkspaceCandidateStateByPaths(tx, workspace, affectedCandidatePaths); err != nil { + return err + } + if len(allHits) > 0 { + if err := insertRedundancyHitsInChunks(tx, scanID, workspace, allHits, insertBatchSize); err != nil { + return err + } + } + if len(candidates) > 0 { + if err := insertCandidatesInChunks(tx, scanID, workspace, candidates, insertBatchSize); err != nil { + return err + } + } + if len(candidateFiles) > 0 { + if err := insertCandidateFilesInChunks(tx, scanID, workspace, candidateFiles, insertBatchSize); err != nil { + return err + } + } + return nil + }) +} + +func cloneRedundancyHitsForScan( + scanID string, + hits []model.StorageIndexRedundancyHit, +) []model.StorageIndexRedundancyHit { + if len(hits) == 0 { + return nil + } + + cloned := make([]model.StorageIndexRedundancyHit, 0, len(hits)) + for _, hit := range hits { + hit.ID = 0 + hit.ScanID = scanID + hit.CreatedAt = time.Time{} + hit.UpdatedAt = time.Time{} + cloned = append(cloned, hit) + } + return cloned +} + +func (s *Service) rebuildPublicFileBaseline( + ctx context.Context, + scanID string, + workspace resolvedWorkspace, +) (int, error) { + if workspace.WorkspaceType != model.StorageIndexWorkspaceTypePublic { + return 0, nil + } + + db := query.GetDB().WithContext(ctx) + cfg := config.GetConfig() + prefixConfig := ceph.StoragePrefixConfig{ + User: cfg.Storage.Prefix.User, + Account: cfg.Storage.Prefix.Account, + Public: cfg.Storage.Prefix.Public, + } + toolboxPod, err := ceph.FindCephToolboxPod(s.kubeClient, toolboxNamespace) + if err != nil { + return 0, fmt.Errorf("find ceph toolbox pod for public baseline failed: %w", err) + } + + resourceRoots, err := s.listPublicResourceRoots(ctx, prefixConfig) + if err != nil { + return 0, err + } + klog.Infof("storageindex: 公共资源登记表加载完成 scan_id=%s root_count=%d", scanID, len(resourceRoots)) + + buildResult := &publicBaselineBuildResult{ + Roots: make([]model.StorageIndexPublicRootBaseline, 0, len(resourceRoots)), + Files: make([]model.StorageIndexPublicFileBaseline, 0), + } + + for _, root := range resourceRoots { + actualDir, pathErr := ceph.ResolveCephFSPath(s.kubeClient, s.kubeConfig, toolboxNamespace, root.LogicalPath, prefixConfig) + if pathErr != nil { + klog.Warningf("storageindex: 解析公共基线路径失败 scan_id=%s path=%s err=%v", scanID, root.LogicalPath, pathErr) + continue + } + totalSize, sizeErr := ceph.GetCephDirectorySize(s.kubeClient, s.kubeConfig, toolboxNamespace, root.LogicalPath, prefixConfig) + if sizeErr != nil { + klog.Warningf("storageindex: 获取公共基线目录大小失败 scan_id=%s path=%s err=%v", scanID, root.LogicalPath, sizeErr) + continue + } + files, scanErr := s.scanComparableFilesByCategory(toolboxPod, actualDir, root.Category) + if scanErr != nil { + klog.Warningf("storageindex: 扫描公共基线关键文件失败 scan_id=%s path=%s err=%v", scanID, root.LogicalPath, scanErr) + continue + } + buildResult.Roots = append(buildResult.Roots, model.StorageIndexPublicRootBaseline{ + ScanID: scanID, + ResourceName: root.Name, + LogicalPath: root.LogicalPath, + RootHash: hashString(root.LogicalPath), + Category: root.Category, + TotalSizeBytes: totalSize, + KeyFileCount: int64(len(files)), + Signature: hashString(strings.ToLower(root.Name) + "|" + root.Category + "|" + strconv.FormatInt(totalSize, 10) + "|" + strconv.Itoa(len(files))), + }) + for _, file := range files { + matchKey := file.RelativePath + "|" + strconv.FormatInt(file.SizeBytes, 10) + buildResult.Files = append(buildResult.Files, model.StorageIndexPublicFileBaseline{ + ScanID: scanID, + PublicRootPath: root.LogicalPath, + PublicRootHash: hashString(root.LogicalPath), + FilePath: path.Join(root.LogicalPath, file.RelativePath), + FileName: file.FileName, + RelativePath: file.RelativePath, + SizeBytes: file.SizeBytes, + MatchKey: matchKey, + MatchKeyHash: hashString(matchKey), + HashAlgorithm: "", + FileHash: "", + }) + } + } + + err = db.Transaction(func(tx *gorm.DB) error { + if err := tx.Exec("DELETE FROM " + (&model.StorageIndexPublicRootBaseline{}).TableName()).Error; err != nil { + return err + } + if err := tx.Exec("DELETE FROM " + (&model.StorageIndexPublicFileBaseline{}).TableName()).Error; err != nil { + return err + } + if len(buildResult.Roots) > 0 { + if err := insertPublicRootBaselinesInChunks(tx, scanID, buildResult.Roots, insertBatchSize); err != nil { + return err + } + } + if len(buildResult.Files) == 0 { + return nil + } + return insertPublicBaselineFilesInChunks(tx, scanID, buildResult.Files, insertBatchSize) + }) + if err != nil { + return 0, err + } + + klog.Infof( + "storageindex: 公共资源基线重建完成 scan_id=%s root_count=%d keyfile_count=%d", + scanID, len(buildResult.Roots), len(buildResult.Files), + ) + return len(buildResult.Files), nil +} + +func longestCandidatePrefix(targetPath string, candidates []model.StorageIndexCandidate) string { + longest := "" + for _, candidate := range candidates { + if targetPath == candidate.TargetPath || strings.HasPrefix(targetPath, candidate.TargetPath+"/") { + if len(candidate.TargetPath) > len(longest) { + longest = candidate.TargetPath + } + } + } + return longest +} + +func roughlySameSize(a, b int64) bool { + if a == b { + return true + } + if a <= 0 || b <= 0 { + return false + } + diff := a - b + if diff < 0 { + diff = -diff + } + threshold := int64(float64(maxInt64(a, b)) * 0.05) + if threshold < 100*1024*1024 { + threshold = 100 * 1024 * 1024 + } + return diff <= threshold +} + +func buildPublicRootLookup( + publicRoots []model.StorageIndexPublicRootBaseline, +) map[string][]model.StorageIndexPublicRootBaseline { + lookup := make(map[string][]model.StorageIndexPublicRootBaseline) + seen := make(map[string]map[string]struct{}) + appendKey := func(key string, item model.StorageIndexPublicRootBaseline) { + normalized := strings.ToLower(strings.TrimSpace(key)) + if normalized == "" { + return + } + if _, ok := seen[normalized]; !ok { + seen[normalized] = make(map[string]struct{}) + } + identity := normalizeUnixPath(item.LogicalPath) + if _, ok := seen[normalized][identity]; ok { + return + } + seen[normalized][identity] = struct{}{} + lookup[normalized] = append(lookup[normalized], item) + } + for _, item := range publicRoots { + appendKey(item.ResourceName, item) + appendKey(path.Base(item.LogicalPath), item) + } + return lookup +} + +func maxInt64(a, b int64) int64 { + if a > b { + return a + } + return b +} + +func minInt(a, b int) int { + if a < b { + return a + } + return b +} + +var candidateFileNameAllowList = []string{ + "config.json", + "tokenizer.json", + "tokenizer_config.json", + "generation_config.json", + "dataset_info.json", +} + +var candidateFileSuffixAllowList = []string{ + ".safetensors", + ".bin", + ".pt", + ".pth", + ".ckpt", + ".index.json", +} + +type candidateFileProbe struct { + RelativePath string + FileName string + ActualPath string + SizeBytes int64 +} + +func (s *Service) verifyCandidateDirectory( + toolboxPod *corev1.Pod, + prefixConfig ceph.StoragePrefixConfig, + scanID string, + workspace resolvedWorkspace, + targetPath string, + publicPath string, + candidateType string, +) ([]model.StorageIndexCandidateFile, []model.StorageIndexRedundancyHit, error) { + if toolboxPod != nil { + if strings.TrimSpace(candidateType) == "dataset_dir" { + return s.verifyCandidateDirectoryByNameSize(toolboxPod, prefixConfig, scanID, workspace, targetPath, publicPath, candidateType) + } + return s.verifyCandidateDirectoryOptimized(toolboxPod, prefixConfig, scanID, workspace, targetPath, publicPath) + } + + targetActual, err := ceph.ResolveCephFSPath(s.kubeClient, s.kubeConfig, toolboxNamespace, targetPath, prefixConfig) + if err != nil { + return nil, nil, err + } + missingTargetFiles, missingPublicFiles, err := s.ensureDirectoryFileSetMatches(toolboxPod, prefixConfig, targetActual, publicPath) + if err != nil { + return nil, nil, err + } + if len(missingTargetFiles) > 0 || len(missingPublicFiles) > 0 { + klog.Infof( + "storageindex: candidate hash compare rejected due to file set drift scan_id=%s workspace_type=%s workspace_name=%s candidate=%s public=%s missing_target=%v missing_public=%v", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, targetPath, publicPath, missingTargetFiles, missingPublicFiles, + ) + return nil, nil, nil + } + + targetFiles, err := s.scanCandidateKeyFiles(toolboxPod, targetActual) + if err != nil { + return nil, nil, err + } + + publicRootHash := hashString(publicPath) + matchKeyHashes := make([]string, 0, len(targetFiles)) + keyByHash := make(map[string]string, len(targetFiles)) + for _, target := range targetFiles { + matchKey := target.RelativePath + "|" + strconv.FormatInt(target.SizeBytes, 10) + matchKeyHash := hashString(matchKey) + matchKeyHashes = append(matchKeyHashes, matchKeyHash) + keyByHash[matchKeyHash] = matchKey + } + + var publicFiles []model.StorageIndexPublicFileBaseline + if len(matchKeyHashes) > 0 { + if err := query.GetDB().WithContext(context.Background()). + Where("public_root_hash = ? AND match_key_hash IN ?", publicRootHash, matchKeyHashes). + Find(&publicFiles).Error; err != nil { + return nil, nil, fmt.Errorf("query public file baseline failed: %w", err) + } + } + publicByKey := make(map[string]model.StorageIndexPublicFileBaseline, len(publicFiles)) + for _, item := range publicFiles { + publicByKey[item.MatchKeyHash] = item + } + + candidateFiles := make([]model.StorageIndexCandidateFile, 0) + verifiedHits := make([]model.StorageIndexRedundancyHit, 0) + for _, target := range targetFiles { + key := target.RelativePath + "|" + strconv.FormatInt(target.SizeBytes, 10) + keyHash := hashString(key) + publicCandidate, ok := publicByKey[keyHash] + if !ok { + continue + } + + targetHash, publicHash, hashErr := s.computePublicBaselineAwareHashes(toolboxPod, target.ActualPath, publicCandidate) + status := model.StorageIndexVerificationStatusSuspected + if hashErr == nil && targetHash == publicHash { + status = model.StorageIndexVerificationStatusVerified + } + + targetLogicalPath := path.Join(targetPath, target.RelativePath) + publicLogicalPath := publicCandidate.FilePath + candidateFiles = append(candidateFiles, model.StorageIndexCandidateFile{ + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + ScanID: scanID, + CandidatePath: targetPath, + FilePath: targetLogicalPath, + FileName: target.FileName, + RelativePath: target.RelativePath, + SizeBytes: target.SizeBytes, + MatchedPublicFile: publicLogicalPath, + HashAlgorithm: hashAlgorithmSHA256, + FileHash: targetHash, + VerificationStatus: status, + }) + + if status == model.StorageIndexVerificationStatusVerified { + verifiedHits = append(verifiedHits, model.StorageIndexRedundancyHit{ + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + ScanID: scanID, + TargetType: model.StorageIndexRedundancyTargetTypeFile, + TargetPath: targetLogicalPath, + PublicPath: publicLogicalPath, + MatchKey: key, + Evidence: "候选目录中的关键文件相对路径、大小与公共空间一致，且 SHA256 校验一致", + Confidence: redundancyConfidenceHigh, + VerificationStatus: model.StorageIndexVerificationStatusVerified, + VerificationMode: hashAlgorithmSHA256, + HashAlgorithm: hashAlgorithmSampledSHA256, + TargetHash: targetHash, + PublicHash: publicHash, + EstimatedBytes: target.SizeBytes, + }) + } + } + + return candidateFiles, verifiedHits, nil +} + +func (s *Service) verifyCandidateDirectoryOptimized( + toolboxPod *corev1.Pod, + prefixConfig ceph.StoragePrefixConfig, + scanID string, + workspace resolvedWorkspace, + targetPath string, + publicPath string, +) ([]model.StorageIndexCandidateFile, []model.StorageIndexRedundancyHit, error) { + targetActual, err := ceph.ResolveCephFSPath(s.kubeClient, s.kubeConfig, toolboxNamespace, targetPath, prefixConfig) + if err != nil { + return nil, nil, err + } + missingTargetFiles, missingPublicFiles, err := s.ensureDirectoryFileSetMatches( + toolboxPod, + prefixConfig, + targetActual, + publicPath, + ) + if err != nil { + return nil, nil, err + } + if len(missingTargetFiles) > 0 || len(missingPublicFiles) > 0 { + klog.Infof( + "storageindex: candidate hash compare rejected due to file set drift scan_id=%s workspace_type=%s workspace_name=%s candidate=%s public=%s missing_target=%v missing_public=%v", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, targetPath, publicPath, missingTargetFiles, missingPublicFiles, + ) + return nil, nil, nil + } + + targetFiles, err := s.scanCandidateKeyFiles(toolboxPod, targetActual) + if err != nil { + return nil, nil, err + } + + publicRootHash := hashString(publicPath) + matchKeyHashes := make([]string, 0, len(targetFiles)) + for _, target := range targetFiles { + matchKey := target.RelativePath + "|" + strconv.FormatInt(target.SizeBytes, 10) + matchKeyHashes = append(matchKeyHashes, hashString(matchKey)) + } + + var publicFiles []model.StorageIndexPublicFileBaseline + if len(matchKeyHashes) > 0 { + if err := query.GetDB().WithContext(context.Background()). + Where("public_root_hash = ? AND match_key_hash IN ?", publicRootHash, matchKeyHashes). + Find(&publicFiles).Error; err != nil { + return nil, nil, fmt.Errorf("query public file baseline failed: %w", err) + } + } + publicByKey := make(map[string]model.StorageIndexPublicFileBaseline, len(publicFiles)) + for _, item := range publicFiles { + publicByKey[item.MatchKeyHash] = item + } + + type matchedCandidateFile struct { + target candidateFileProbe + publicCandidate model.StorageIndexPublicFileBaseline + key string + publicActual string + } + + matches := make([]matchedCandidateFile, 0) + publicRootActual := "" + for _, target := range targetFiles { + key := target.RelativePath + "|" + strconv.FormatInt(target.SizeBytes, 10) + keyHash := hashString(key) + publicCandidate, ok := publicByKey[keyHash] + if !ok { + continue + } + + publicActual := "" + needPublicActual := strings.TrimSpace(publicCandidate.FileHash) == "" || strings.TrimSpace(publicCandidate.HashAlgorithm) != hashAlgorithmSampledSHA256 || isSafeTensorsFile(target.FileName) + if needPublicActual { + if publicRootActual == "" { + publicRootActual, err = ceph.ResolveCephFSPath(s.kubeClient, s.kubeConfig, toolboxNamespace, publicPath, prefixConfig) + if err != nil { + return nil, nil, err + } + } + publicActual = normalizeUnixPath(path.Join(publicRootActual, publicCandidate.RelativePath)) + } + matches = append(matches, matchedCandidateFile{ + target: target, + publicCandidate: publicCandidate, + key: key, + publicActual: publicActual, + }) + } + fallbackMatchCount := 0 + if len(matches) == 0 { + fallbackMatches, fallbackPublicRootActual, err := buildFallbackCandidateMatches(targetFiles, publicFiles) + if err != nil { + return nil, nil, err + } + if len(fallbackMatches) > 0 { + if publicRootActual == "" { + publicRootActual = fallbackPublicRootActual + } + for _, match := range fallbackMatches { + if match.publicActual == "" && (strings.TrimSpace(match.publicCandidate.FileHash) == "" || isSafeTensorsFile(match.target.FileName)) { + if publicRootActual == "" { + publicRootActual, err = ceph.ResolveCephFSPath(s.kubeClient, s.kubeConfig, toolboxNamespace, publicPath, prefixConfig) + if err != nil { + return nil, nil, err + } + } + match.publicActual = normalizeUnixPath(path.Join(publicRootActual, match.publicCandidate.RelativePath)) + } + matches = append(matches, match) + } + fallbackMatchCount = len(fallbackMatches) + } + } + if len(matches) == 0 { + klog.Infof( + "storageindex: candidate hash compare skipped scan_id=%s workspace_type=%s workspace_name=%s candidate=%s public=%s target_key_file_count=%d matched_key_file_count=0 fallback_match_count=0", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, targetPath, publicPath, len(targetFiles), + ) + return nil, nil, nil + } + + targetActualPaths := make([]string, 0, len(matches)) + targetActualSizes := make(map[string]int64, len(matches)) + missingPublicActualPaths := make([]string, 0) + publicActualSizes := make(map[string]int64) + for _, match := range matches { + targetActualPaths = append(targetActualPaths, match.target.ActualPath) + targetActualSizes[match.target.ActualPath] = match.target.SizeBytes + if match.publicActual != "" { + missingPublicActualPaths = append(missingPublicActualPaths, match.publicActual) + publicActualSizes[match.publicActual] = match.publicCandidate.SizeBytes + } + } + + targetHashes, err := s.computeActualFileHashesBatchWithSizes(toolboxPod, targetActualPaths, targetActualSizes) + if err != nil { + return nil, nil, err + } + publicHashes := make(map[string]string) + if len(missingPublicActualPaths) > 0 { + publicHashes, err = s.computeActualFileHashesBatchWithSizes(toolboxPod, missingPublicActualPaths, publicActualSizes) + if err != nil { + return nil, nil, err + } + for _, match := range matches { + if match.publicCandidate.ID == 0 || match.publicActual == "" { + continue + } + publicHash := publicHashes[match.publicActual] + if publicHash == "" { + continue + } + _ = query.GetDB(). + Model(&model.StorageIndexPublicFileBaseline{}). + Where("id = ?", match.publicCandidate.ID). + Updates(map[string]any{ + "hash_algorithm": hashAlgorithmSampledSHA256, + "file_hash": publicHash, + }).Error + } + } + + candidateFiles := make([]model.StorageIndexCandidateFile, 0, len(matches)) + verifiedHits := make([]model.StorageIndexRedundancyHit, 0) + for _, match := range matches { + targetHash := targetHashes[match.target.ActualPath] + publicHash := strings.TrimSpace(match.publicCandidate.FileHash) + if (publicHash == "" || strings.TrimSpace(match.publicCandidate.HashAlgorithm) != hashAlgorithmSampledSHA256) && match.publicActual != "" { + publicHash = publicHashes[match.publicActual] + } + status := model.StorageIndexVerificationStatusSuspected + verificationMode := hashAlgorithmSampledSHA256 + if isSafeTensorsFile(match.target.FileName) { + headersMatch, headerErr := s.compareSafetensorsHeaders(toolboxPod, match.target.ActualPath, match.publicActual) + if headerErr == nil && headersMatch { + verificationMode = verificationModeSafeTensorsHdrAndSampledSHA + } else { + targetHash = "" + publicHash = "" + } + } + if targetHash != "" && publicHash != "" && targetHash == publicHash { + status = model.StorageIndexVerificationStatusVerified + } + + targetLogicalPath := path.Join(targetPath, match.target.RelativePath) + publicLogicalPath := match.publicCandidate.FilePath + candidateFiles = append(candidateFiles, model.StorageIndexCandidateFile{ + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + ScanID: scanID, + CandidatePath: targetPath, + FilePath: targetLogicalPath, + FileName: match.target.FileName, + RelativePath: match.target.RelativePath, + SizeBytes: match.target.SizeBytes, + MatchedPublicFile: publicLogicalPath, + HashAlgorithm: hashAlgorithmSampledSHA256, + FileHash: targetHash, + VerificationStatus: status, + }) + + if status == model.StorageIndexVerificationStatusVerified { + verifiedHits = append(verifiedHits, model.StorageIndexRedundancyHit{ + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + ScanID: scanID, + TargetType: model.StorageIndexRedundancyTargetTypeFile, + TargetPath: targetLogicalPath, + PublicPath: publicLogicalPath, + MatchKey: match.key, + Evidence: "候选目录中的关键文件相对路径、大小与公共空间一致，且 SHA256 校验一致", + Confidence: redundancyConfidenceHigh, + VerificationStatus: model.StorageIndexVerificationStatusVerified, + VerificationMode: verificationMode, + HashAlgorithm: hashAlgorithmSampledSHA256, + TargetHash: targetHash, + PublicHash: publicHash, + EstimatedBytes: match.target.SizeBytes, + }) + } + } + + klog.Infof( + "storageindex: candidate hash compare finished scan_id=%s workspace_type=%s workspace_name=%s candidate=%s public=%s target_key_file_count=%d matched_key_file_count=%d fallback_match_count=%d verified_hit_count=%d missing_public_hash_count=%d", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, targetPath, publicPath, len(targetFiles), len(matches), fallbackMatchCount, len(verifiedHits), len(missingPublicActualPaths), + ) + return candidateFiles, verifiedHits, nil +} + +func (s *Service) verifyCandidateDirectoryByNameSize( + toolboxPod *corev1.Pod, + prefixConfig ceph.StoragePrefixConfig, + scanID string, + workspace resolvedWorkspace, + targetPath string, + publicPath string, + candidateType string, +) ([]model.StorageIndexCandidateFile, []model.StorageIndexRedundancyHit, error) { + targetActual, err := ceph.ResolveCephFSPath(s.kubeClient, s.kubeConfig, toolboxNamespace, targetPath, prefixConfig) + if err != nil { + return nil, nil, err + } + missingTargetFiles, missingPublicFiles, err := s.ensureDirectoryFileSetMatches(toolboxPod, prefixConfig, targetActual, publicPath) + if err != nil { + return nil, nil, err + } + if len(missingTargetFiles) > 0 || len(missingPublicFiles) > 0 { + klog.Infof( + "storageindex: candidate keyfile compare rejected due to file set drift scan_id=%s workspace_type=%s workspace_name=%s candidate=%s public=%s missing_target=%v missing_public=%v", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, targetPath, publicPath, missingTargetFiles, missingPublicFiles, + ) + return nil, nil, nil + } + + targetFiles, err := s.scanComparableFilesByCategory(toolboxPod, targetActual, candidateType) + if err != nil { + return nil, nil, err + } + + publicRootHash := hashString(publicPath) + matchKeyHashes := make([]string, 0, len(targetFiles)) + for _, target := range targetFiles { + matchKey := target.RelativePath + "|" + strconv.FormatInt(target.SizeBytes, 10) + matchKeyHashes = append(matchKeyHashes, hashString(matchKey)) + } + + var publicFiles []model.StorageIndexPublicFileBaseline + if len(matchKeyHashes) > 0 { + if err := query.GetDB().WithContext(context.Background()). + Where("public_root_hash = ? AND match_key_hash IN ?", publicRootHash, matchKeyHashes). + Find(&publicFiles).Error; err != nil { + return nil, nil, fmt.Errorf("query public file baseline failed: %w", err) + } + } + + type matchedCandidateFile struct { + target candidateFileProbe + publicCandidate model.StorageIndexPublicFileBaseline + key string + } + + publicByKey := make(map[string]model.StorageIndexPublicFileBaseline, len(publicFiles)) + for _, item := range publicFiles { + publicByKey[item.MatchKeyHash] = item + } + + matches := make([]matchedCandidateFile, 0) + for _, target := range targetFiles { + key := target.RelativePath + "|" + strconv.FormatInt(target.SizeBytes, 10) + keyHash := hashString(key) + publicCandidate, ok := publicByKey[keyHash] + if !ok { + continue + } + matches = append(matches, matchedCandidateFile{ + target: target, + publicCandidate: publicCandidate, + key: key, + }) + } + + fallbackMatchCount := 0 + if len(matches) == 0 { + fallbackMatches := buildFallbackCandidateMatchesByNameSize(targetFiles, publicFiles) + fallbackMatchCount = len(fallbackMatches) + for _, match := range fallbackMatches { + matches = append(matches, matchedCandidateFile{ + target: match.target, + publicCandidate: match.publicCandidate, + key: match.key, + }) + } + } + if len(matches) == 0 { + klog.Infof( + "storageindex: candidate keyfile compare skipped scan_id=%s workspace_type=%s workspace_name=%s candidate=%s public=%s target_key_file_count=%d matched_key_file_count=0 fallback_match_count=0", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, targetPath, publicPath, len(targetFiles), + ) + return nil, nil, nil + } + + candidateFiles := make([]model.StorageIndexCandidateFile, 0, len(matches)) + verifiedHits := make([]model.StorageIndexRedundancyHit, 0, len(matches)) + for _, match := range matches { + targetLogicalPath := path.Join(targetPath, match.target.RelativePath) + publicLogicalPath := match.publicCandidate.FilePath + candidateFiles = append(candidateFiles, model.StorageIndexCandidateFile{ + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + ScanID: scanID, + CandidatePath: targetPath, + FilePath: targetLogicalPath, + FileName: match.target.FileName, + RelativePath: match.target.RelativePath, + SizeBytes: match.target.SizeBytes, + MatchedPublicFile: publicLogicalPath, + HashAlgorithm: "", + FileHash: "", + VerificationStatus: model.StorageIndexVerificationStatusVerified, + }) + + verifiedHits = append(verifiedHits, model.StorageIndexRedundancyHit{ + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + ScanID: scanID, + TargetType: model.StorageIndexRedundancyTargetTypeFile, + TargetPath: targetLogicalPath, + PublicPath: publicLogicalPath, + MatchKey: match.key, + Evidence: "候选目录中的关键文件名与大小和公共空间资源一致，确认存在冗余", + Confidence: redundancyConfidenceHigh, + VerificationStatus: model.StorageIndexVerificationStatusVerified, + VerificationMode: verificationModeFileName, + HashAlgorithm: "", + TargetHash: "", + PublicHash: "", + EstimatedBytes: match.target.SizeBytes, + }) + } + + klog.Infof( + "storageindex: candidate keyfile compare finished scan_id=%s workspace_type=%s workspace_name=%s candidate=%s public=%s target_key_file_count=%d matched_key_file_count=%d fallback_match_count=%d verified_hit_count=%d", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, targetPath, publicPath, len(targetFiles), len(matches), fallbackMatchCount, len(verifiedHits), + ) + return candidateFiles, verifiedHits, nil +} + +func (s *Service) scanCandidateKeyFiles( + toolboxPod *corev1.Pod, + actualDir string, +) ([]candidateFileProbe, error) { + script := fmt.Sprintf( + `find %s -type f \( -name '*.safetensors' -o -name '*.bin' -o -name '*.pt' -o -name '*.pth' -o -name '*.ckpt' -o -name '*.index.json' -o -name 'config.json' -o -name 'tokenizer.json' -o -name 'tokenizer_config.json' -o -name 'generation_config.json' -o -name 'dataset_info.json' \) -printf '%%P\037%%f\037%%s\037%%p\0'`, + shellQuote(actualDir), + ) + return s.scanCandidateFilesWithScript(toolboxPod, script, "scan candidate key files failed") +} + +func (s *Service) scanModelComparableFiles( + toolboxPod *corev1.Pod, + actualDir string, +) ([]candidateFileProbe, error) { + return s.scanCandidateKeyFiles(toolboxPod, actualDir) +} + +func (s *Service) scanDatasetComparableFiles( + toolboxPod *corev1.Pod, + actualDir string, +) ([]candidateFileProbe, error) { + script := fmt.Sprintf( + `find %s -name .snap -prune -o -type f -printf '%%P\037%%f\037%%s\037%%p\0'`, + shellQuote(actualDir), + ) + return s.scanCandidateFilesWithScript(toolboxPod, script, "scan dataset comparable files failed") +} + +func (s *Service) scanCandidateFilesWithScript( + toolboxPod *corev1.Pod, + script string, + errorMessage string, +) ([]candidateFileProbe, error) { + output, err := ceph.ExecInPod( + s.kubeClient, + s.kubeConfig, + toolboxPod, + []string{"sh", "-c", script}, + ) + if err != nil { + return nil, fmt.Errorf("%s: %w", errorMessage, err) + } + + records := strings.Split(output, findRecordSeparator) + result := make([]candidateFileProbe, 0, len(records)) + for _, record := range records { + if record == "" { + continue + } + fields := strings.Split(record, findFieldSeparator) + if len(fields) != 4 { + continue + } + result = append(result, candidateFileProbe{ + RelativePath: normalizeUnixPath(fields[0]), + FileName: fields[1], + SizeBytes: parseInt64(fields[2]), + ActualPath: normalizeUnixPath(fields[3]), + }) + } + return result, nil +} + +func (s *Service) ensureDirectoryFileSetMatches( + toolboxPod *corev1.Pod, + prefixConfig ceph.StoragePrefixConfig, + targetActual string, + publicPath string, +) ([]string, []string, error) { + publicActual, err := ceph.ResolveCephFSPath(s.kubeClient, s.kubeConfig, toolboxNamespace, publicPath, prefixConfig) + if err != nil { + return nil, nil, err + } + targetFiles, err := s.scanDatasetComparableFiles(toolboxPod, targetActual) + if err != nil { + return nil, nil, err + } + publicFiles, err := s.scanDatasetComparableFiles(toolboxPod, publicActual) + if err != nil { + return nil, nil, err + } + missingTarget, missingPublic := compareStrictDirectoryFileSets(targetFiles, publicFiles) + return missingTarget, missingPublic, nil +} + +func (s *Service) scanComparableFilesByCategory( + toolboxPod *corev1.Pod, + actualDir string, + category string, +) ([]candidateFileProbe, error) { + if strings.TrimSpace(category) == "dataset_dir" { + return s.scanDatasetComparableFiles(toolboxPod, actualDir) + } + return s.scanCandidateKeyFiles(toolboxPod, actualDir) +} + +func (s *Service) computeActualFileHashes( + toolboxPod *corev1.Pod, + targetActualPath string, + publicActualPath string, +) (string, string, error) { + targetHash, err := s.computeActualFileHash(toolboxPod, targetActualPath) + if err != nil { + return "", "", err + } + publicHash, err := s.computeActualFileHash(toolboxPod, publicActualPath) + if err != nil { + return "", "", err + } + return targetHash, publicHash, nil +} + +func (s *Service) computePublicBaselineAwareHashes( + toolboxPod *corev1.Pod, + targetActualPath string, + publicBaseline model.StorageIndexPublicFileBaseline, +) (string, string, error) { + targetHash, err := s.computeActualFileHash(toolboxPod, targetActualPath) + if err != nil { + return "", "", err + } + + publicHash := strings.TrimSpace(publicBaseline.FileHash) + if publicHash != "" { + return targetHash, publicHash, nil + } + + cfg := config.GetConfig() + prefixConfig := ceph.StoragePrefixConfig{ + User: cfg.Storage.Prefix.User, + Account: cfg.Storage.Prefix.Account, + Public: cfg.Storage.Prefix.Public, + } + publicActualPath, err := ceph.ResolveCephFSPath(s.kubeClient, s.kubeConfig, toolboxNamespace, publicBaseline.FilePath, prefixConfig) + if err != nil { + return "", "", err + } + publicHash, err = s.computeActualFileHash(toolboxPod, publicActualPath) + if err != nil { + return "", "", err + } + + if publicBaseline.ID != 0 { + _ = query.GetDB(). + Model(&model.StorageIndexPublicFileBaseline{}). + Where("id = ?", publicBaseline.ID). + Updates(map[string]any{ + "hash_algorithm": hashAlgorithmSHA256, + "file_hash": publicHash, + }).Error + } + + return targetHash, publicHash, nil +} + +func (s *Service) computeActualFileHash( + toolboxPod *corev1.Pod, + actualPath string, +) (string, error) { + output, err := ceph.ExecInPod( + s.kubeClient, + s.kubeConfig, + toolboxPod, + []string{"sha256sum", actualPath}, + ) + if err != nil { + return "", err + } + fields := strings.Fields(strings.TrimSpace(output)) + if len(fields) == 0 { + return "", fmt.Errorf("empty sha256sum output for %s", actualPath) + } + return fields[0], nil +} + +func (s *Service) computeActualFileHashesBatch( + toolboxPod *corev1.Pod, + actualPaths []string, +) (map[string]string, error) { + return s.computeActualFileHashesBatchWithSizes(toolboxPod, actualPaths, nil) +} + +func (s *Service) computeActualFileHashesBatchWithSizes( + toolboxPod *corev1.Pod, + actualPaths []string, + sizeByPath map[string]int64, +) (map[string]string, error) { + uniquePaths := make([]string, 0, len(actualPaths)) + seen := make(map[string]struct{}, len(actualPaths)) + for _, actualPath := range actualPaths { + normalized := normalizeUnixPath(actualPath) + if normalized == "" { + continue + } + if _, ok := seen[normalized]; ok { + continue + } + seen[normalized] = struct{}{} + uniquePaths = append(uniquePaths, normalized) + } + if len(uniquePaths) == 0 { + return map[string]string{}, nil + } + + result := make(map[string]string, len(uniquePaths)) + for _, actualPath := range uniquePaths { + sizeBytes := int64(0) + if sizeByPath != nil { + sizeBytes = sizeByPath[actualPath] + } + hashValue, err := s.computeActualFileSampledHash(toolboxPod, actualPath, sizeBytes) + if err != nil { + return nil, err + } + result[actualPath] = hashValue + } + + return result, nil +} + +func (s *Service) computeActualFileFullHashesBatch( + toolboxPod *corev1.Pod, + actualPaths []string, +) (map[string]string, error) { + uniquePaths := make([]string, 0, len(actualPaths)) + seen := make(map[string]struct{}, len(actualPaths)) + for _, actualPath := range actualPaths { + normalized := normalizeUnixPath(actualPath) + if normalized == "" { + continue + } + if _, ok := seen[normalized]; ok { + continue + } + seen[normalized] = struct{}{} + uniquePaths = append(uniquePaths, normalized) + } + if len(uniquePaths) == 0 { + return map[string]string{}, nil + } + + result := make(map[string]string, len(uniquePaths)) + for _, chunk := range chunkStrings(uniquePaths, 128) { + args := make([]string, 0, len(chunk)+2) + args = append(args, "sha256sum", "--") + args = append(args, chunk...) + output, err := ceph.ExecInPod( + s.kubeClient, + s.kubeConfig, + toolboxPod, + args, + ) + if err != nil { + return nil, err + } + for path, hashValue := range parseSha256sumOutput(output) { + result[path] = hashValue + } + } + + for _, actualPath := range uniquePaths { + if strings.TrimSpace(result[actualPath]) == "" { + return nil, fmt.Errorf("missing full sha256 output for %s", actualPath) + } + } + + return result, nil +} + +func chunkStrings(items []string, size int) [][]string { + if size <= 0 || len(items) == 0 { + return nil + } + + chunks := make([][]string, 0, (len(items)+size-1)/size) + for start := 0; start < len(items); start += size { + end := start + size + if end > len(items) { + end = len(items) + } + chunks = append(chunks, items[start:end]) + } + return chunks +} + +func parseSha256sumOutput(output string) map[string]string { + result := make(map[string]string) + for _, line := range strings.Split(strings.TrimSpace(output), "\n") { + trimmed := strings.TrimSpace(line) + if trimmed == "" { + continue + } + if len(trimmed) < 64 { + continue + } + hashValue := trimmed[:64] + actualPath := strings.TrimSpace(trimmed[64:]) + actualPath = strings.TrimPrefix(actualPath, "*") + actualPath = normalizeUnixPath(actualPath) + if actualPath == "" { + continue + } + result[actualPath] = hashValue + } + return result +} + +func isSafeTensorsFile(fileName string) bool { + return strings.HasSuffix(strings.ToLower(strings.TrimSpace(fileName)), ".safetensors") +} + +type sampledRegion struct { + Offset int64 + Count int64 +} + +func buildSampledRegions(sizeBytes int64) []sampledRegion { + if sizeBytes <= 0 { + return nil + } + if sizeBytes <= sampledHashSegmentBytes { + return []sampledRegion{{Offset: 0, Count: sizeBytes}} + } + + offsets := []int64{ + sizeBytes / 4, + sizeBytes / 2, + (sizeBytes * 3) / 4, + } + regions := make([]sampledRegion, 0, len(offsets)) + seen := make(map[int64]struct{}, len(offsets)) + for _, offset := range offsets { + if offset > sizeBytes-sampledHashSegmentBytes { + offset = sizeBytes - sampledHashSegmentBytes + } + if offset < 0 { + offset = 0 + } + if _, ok := seen[offset]; ok { + continue + } + seen[offset] = struct{}{} + regions = append(regions, sampledRegion{ + Offset: offset, + Count: sampledHashSegmentBytes, + }) + } + return regions +} + +func (s *Service) computeActualFileSampledHash( + toolboxPod *corev1.Pod, + actualPath string, + sizeBytes int64, +) (string, error) { + regions := buildSampledRegions(sizeBytes) + if len(regions) == 0 { + return "", fmt.Errorf("invalid file size for sampled hash: %d", sizeBytes) + } + + commands := make([]string, 0, len(regions)) + for _, region := range regions { + commands = append( + commands, + fmt.Sprintf( + "dd if=%s skip=%d count=%d iflag=skip_bytes,count_bytes 2>/dev/null", + shellQuote(actualPath), + region.Offset, + region.Count, + ), + ) + } + script := "{ " + strings.Join(commands, "; ") + "; } | sha256sum" + output, err := ceph.ExecInPod( + s.kubeClient, + s.kubeConfig, + toolboxPod, + []string{"sh", "-c", script}, + ) + if err != nil { + return "", err + } + fields := strings.Fields(strings.TrimSpace(output)) + if len(fields) == 0 { + return "", fmt.Errorf("empty sampled sha256sum output for %s", actualPath) + } + return fields[0], nil +} + +func (s *Service) compareSafetensorsHeaders( + toolboxPod *corev1.Pod, + targetActualPath string, + publicActualPath string, +) (bool, error) { + if targetActualPath == "" || publicActualPath == "" { + return false, fmt.Errorf("empty safetensors path for header compare") + } + targetSkeleton, err := s.readSafeTensorsHeaderSkeleton(toolboxPod, targetActualPath) + if err != nil { + return false, err + } + publicSkeleton, err := s.readSafeTensorsHeaderSkeleton(toolboxPod, publicActualPath) + if err != nil { + return false, err + } + if len(targetSkeleton) != len(publicSkeleton) { + return false, nil + } + for i := range targetSkeleton { + if targetSkeleton[i] != publicSkeleton[i] { + return false, nil + } + } + return true, nil +} + +func (s *Service) readSafeTensorsHeaderSkeleton( + toolboxPod *corev1.Pod, + actualPath string, +) ([]string, error) { + headerLength, err := s.readSafeTensorsHeaderLength(toolboxPod, actualPath) + if err != nil { + return nil, err + } + if headerLength <= 0 || headerLength > safetensorsHeaderMaxBytes { + return nil, fmt.Errorf("invalid safetensors header length %d for %s", headerLength, actualPath) + } + + script := fmt.Sprintf( + "dd if=%s skip=8 count=%d iflag=skip_bytes,count_bytes 2>/dev/null", + shellQuote(actualPath), + headerLength, + ) + output, err := ceph.ExecInPod( + s.kubeClient, + s.kubeConfig, + toolboxPod, + []string{"sh", "-c", script}, + ) + if err != nil { + return nil, err + } + return parseSafeTensorsHeaderSkeleton(output) +} + +func (s *Service) readSafeTensorsHeaderLength( + toolboxPod *corev1.Pod, + actualPath string, +) (int64, error) { + script := fmt.Sprintf( + "dd if=%s count=8 iflag=count_bytes 2>/dev/null | od -An -v -t x1", + shellQuote(actualPath), + ) + output, err := ceph.ExecInPod( + s.kubeClient, + s.kubeConfig, + toolboxPod, + []string{"sh", "-c", script}, + ) + if err != nil { + return 0, err + } + fields := strings.Fields(strings.TrimSpace(output)) + if len(fields) != 8 { + return 0, fmt.Errorf("unexpected safetensors header length bytes for %s: %q", actualPath, output) + } + headerBytes := make([]byte, 8) + for i, field := range fields { + value, parseErr := strconv.ParseUint(field, 16, 8) + if parseErr != nil { + return 0, fmt.Errorf("parse safetensors header length byte failed: %w", parseErr) + } + headerBytes[i] = byte(value) + } + return int64(binary.LittleEndian.Uint64(headerBytes)), nil +} + +func parseSafeTensorsHeaderSkeleton(headerJSON string) ([]string, error) { + var payload map[string]json.RawMessage + if err := json.Unmarshal([]byte(headerJSON), &payload); err != nil { + return nil, err + } + + type tensorShapePayload struct { + Shape []int64 `json:"shape"` + } + + skeleton := make([]string, 0, len(payload)) + for name, raw := range payload { + if name == "__metadata__" { + continue + } + var tensor tensorShapePayload + if err := json.Unmarshal(raw, &tensor); err != nil { + return nil, err + } + shapeParts := make([]string, 0, len(tensor.Shape)) + for _, dim := range tensor.Shape { + shapeParts = append(shapeParts, strconv.FormatInt(dim, 10)) + } + skeleton = append(skeleton, name+"="+strings.Join(shapeParts, ",")) + } + sort.Strings(skeleton) + return skeleton, nil +} + +func buildFallbackCandidateMatches( + targetFiles []candidateFileProbe, + publicFiles []model.StorageIndexPublicFileBaseline, +) ([]struct { + target candidateFileProbe + publicCandidate model.StorageIndexPublicFileBaseline + key string + publicActual string +}, string, error) { + type matchedCandidateFile struct { + target candidateFileProbe + publicCandidate model.StorageIndexPublicFileBaseline + key string + publicActual string + } + + targetByNameSize := make(map[string][]candidateFileProbe) + for _, target := range targetFiles { + key := strings.ToLower(strings.TrimSpace(target.FileName)) + "|" + strconv.FormatInt(target.SizeBytes, 10) + targetByNameSize[key] = append(targetByNameSize[key], target) + } + publicByNameSize := make(map[string][]model.StorageIndexPublicFileBaseline) + for _, item := range publicFiles { + key := strings.ToLower(strings.TrimSpace(item.FileName)) + "|" + strconv.FormatInt(item.SizeBytes, 10) + publicByNameSize[key] = append(publicByNameSize[key], item) + } + + matches := make([]matchedCandidateFile, 0) + for key, targets := range targetByNameSize { + if len(targets) != 1 { + continue + } + publicCandidates := publicByNameSize[key] + if len(publicCandidates) != 1 { + continue + } + matchKey := targets[0].RelativePath + "|" + strconv.FormatInt(targets[0].SizeBytes, 10) + matches = append(matches, matchedCandidateFile{ + target: targets[0], + publicCandidate: publicCandidates[0], + key: matchKey, + publicActual: "", + }) + } + + result := make([]struct { + target candidateFileProbe + publicCandidate model.StorageIndexPublicFileBaseline + key string + publicActual string + }, 0, len(matches)) + for _, match := range matches { + result = append(result, struct { + target candidateFileProbe + publicCandidate model.StorageIndexPublicFileBaseline + key string + publicActual string + }{ + target: match.target, + publicCandidate: match.publicCandidate, + key: match.key, + publicActual: match.publicActual, + }) + } + return result, "", nil +} + +func buildFallbackCandidateMatchesByNameSize( + targetFiles []candidateFileProbe, + publicFiles []model.StorageIndexPublicFileBaseline, +) []struct { + target candidateFileProbe + publicCandidate model.StorageIndexPublicFileBaseline + key string +} { + targetByNameSize := make(map[string][]candidateFileProbe) + for _, target := range targetFiles { + key := strings.ToLower(strings.TrimSpace(target.FileName)) + "|" + strconv.FormatInt(target.SizeBytes, 10) + targetByNameSize[key] = append(targetByNameSize[key], target) + } + publicByNameSize := make(map[string][]model.StorageIndexPublicFileBaseline) + for _, item := range publicFiles { + key := strings.ToLower(strings.TrimSpace(item.FileName)) + "|" + strconv.FormatInt(item.SizeBytes, 10) + publicByNameSize[key] = append(publicByNameSize[key], item) + } + + matches := make([]struct { + target candidateFileProbe + publicCandidate model.StorageIndexPublicFileBaseline + key string + }, 0) + for key, targets := range targetByNameSize { + if len(targets) != 1 { + continue + } + publicCandidates := publicByNameSize[key] + if len(publicCandidates) != 1 { + continue + } + matchKey := targets[0].RelativePath + "|" + strconv.FormatInt(targets[0].SizeBytes, 10) + matches = append(matches, struct { + target candidateFileProbe + publicCandidate model.StorageIndexPublicFileBaseline + key string + }{ + target: targets[0], + publicCandidate: publicCandidates[0], + key: matchKey, + }) + } + return matches +} + +func compareStrictDirectoryFileSets( + targetFiles []candidateFileProbe, + publicFiles []candidateFileProbe, +) ([]string, []string) { + targetByKey := make(map[string]int, len(targetFiles)) + for _, item := range targetFiles { + key := normalizeUnixPath(item.RelativePath) + "|" + strconv.FormatInt(item.SizeBytes, 10) + targetByKey[key]++ + } + publicByKey := make(map[string]int, len(publicFiles)) + for _, item := range publicFiles { + key := normalizeUnixPath(item.RelativePath) + "|" + strconv.FormatInt(item.SizeBytes, 10) + publicByKey[key]++ + } + + keys := make(map[string]struct{}, len(targetByKey)+len(publicByKey)) + for key := range targetByKey { + keys[key] = struct{}{} + } + for key := range publicByKey { + keys[key] = struct{}{} + } + + missingTarget := make([]string, 0) + missingPublic := make([]string, 0) + for key := range keys { + targetCount := targetByKey[key] + publicCount := publicByKey[key] + relativePath := strings.SplitN(key, "|", 2)[0] + if targetCount > publicCount { + for i := 0; i < targetCount-publicCount; i++ { + missingPublic = append(missingPublic, relativePath) + } + } + if publicCount > targetCount { + for i := 0; i < publicCount-targetCount; i++ { + missingTarget = append(missingTarget, relativePath) + } + } + } + sort.Strings(missingTarget) + sort.Strings(missingPublic) + return missingTarget, missingPublic +} + +func pairCandidateFilesForComparison( + leftFiles []candidateFileProbe, + rightFiles []candidateFileProbe, +) ([]struct { + left candidateFileProbe + right candidateFileProbe +}, int, int, []string, []string) { + type comparePair struct { + left candidateFileProbe + right candidateFileProbe + } + + leftByExact := make(map[string]candidateFileProbe, len(leftFiles)) + rightByExact := make(map[string]candidateFileProbe, len(rightFiles)) + for _, item := range leftFiles { + leftByExact[item.RelativePath+"|"+strconv.FormatInt(item.SizeBytes, 10)] = item + } + for _, item := range rightFiles { + rightByExact[item.RelativePath+"|"+strconv.FormatInt(item.SizeBytes, 10)] = item + } + + pairs := make([]comparePair, 0) + matchedLeft := make(map[string]struct{}, len(leftFiles)) + matchedRight := make(map[string]struct{}, len(rightFiles)) + exactMatchCount := 0 + for key, left := range leftByExact { + right, ok := rightByExact[key] + if !ok { + continue + } + pairs = append(pairs, comparePair{left: left, right: right}) + matchedLeft[left.RelativePath] = struct{}{} + matchedRight[right.RelativePath] = struct{}{} + exactMatchCount++ + } + + unmatchedLeft := make([]candidateFileProbe, 0) + for _, item := range leftFiles { + if _, ok := matchedLeft[item.RelativePath]; ok { + continue + } + unmatchedLeft = append(unmatchedLeft, item) + } + unmatchedRight := make([]candidateFileProbe, 0) + for _, item := range rightFiles { + if _, ok := matchedRight[item.RelativePath]; ok { + continue + } + unmatchedRight = append(unmatchedRight, item) + } + + leftFallback := make(map[string][]candidateFileProbe) + for _, item := range unmatchedLeft { + key := strings.ToLower(strings.TrimSpace(item.FileName)) + "|" + strconv.FormatInt(item.SizeBytes, 10) + leftFallback[key] = append(leftFallback[key], item) + } + rightFallback := make(map[string][]candidateFileProbe) + for _, item := range unmatchedRight { + key := strings.ToLower(strings.TrimSpace(item.FileName)) + "|" + strconv.FormatInt(item.SizeBytes, 10) + rightFallback[key] = append(rightFallback[key], item) + } + + fallbackMatchCount := 0 + for key, leftItems := range leftFallback { + if len(leftItems) != 1 { + continue + } + rightItems := rightFallback[key] + if len(rightItems) != 1 { + continue + } + left := leftItems[0] + right := rightItems[0] + pairs = append(pairs, comparePair{left: left, right: right}) + matchedLeft[left.RelativePath] = struct{}{} + matchedRight[right.RelativePath] = struct{}{} + fallbackMatchCount++ + } + + missingLeft := make([]string, 0) + for _, item := range leftFiles { + if _, ok := matchedLeft[item.RelativePath]; ok { + continue + } + missingLeft = append(missingLeft, item.RelativePath) + } + sort.Strings(missingLeft) + missingRight := make([]string, 0) + for _, item := range rightFiles { + if _, ok := matchedRight[item.RelativePath]; ok { + continue + } + missingRight = append(missingRight, item.RelativePath) + } + sort.Strings(missingRight) + + result := make([]struct { + left candidateFileProbe + right candidateFileProbe + }, 0, len(pairs)) + sort.Slice(pairs, func(i, j int) bool { + if pairs[i].left.RelativePath == pairs[j].left.RelativePath { + return pairs[i].right.RelativePath < pairs[j].right.RelativePath + } + return pairs[i].left.RelativePath < pairs[j].left.RelativePath + }) + for _, pair := range pairs { + result = append(result, struct { + left candidateFileProbe + right candidateFileProbe + }{ + left: pair.left, + right: pair.right, + }) + } + + return result, exactMatchCount, fallbackMatchCount, missingLeft, missingRight +} + +func pairDirectoryFilesByNameAndSize( + leftFiles []candidateFileProbe, + rightFiles []candidateFileProbe, +) ([]struct { + left candidateFileProbe + right candidateFileProbe +}, int, int, []string, []string) { + type comparePair struct { + left candidateFileProbe + right candidateFileProbe + } + + leftByNameSize := make(map[string][]candidateFileProbe) + for _, item := range leftFiles { + key := strings.ToLower(strings.TrimSpace(item.FileName)) + "|" + strconv.FormatInt(item.SizeBytes, 10) + leftByNameSize[key] = append(leftByNameSize[key], item) + } + rightByNameSize := make(map[string][]candidateFileProbe) + for _, item := range rightFiles { + key := strings.ToLower(strings.TrimSpace(item.FileName)) + "|" + strconv.FormatInt(item.SizeBytes, 10) + rightByNameSize[key] = append(rightByNameSize[key], item) + } + + keys := make(map[string]struct{}, len(leftByNameSize)+len(rightByNameSize)) + for key := range leftByNameSize { + keys[key] = struct{}{} + } + for key := range rightByNameSize { + keys[key] = struct{}{} + } + + pairs := make([]comparePair, 0) + missingLeft := make([]string, 0) + missingRight := make([]string, 0) + fallbackMatchCount := 0 + for key := range keys { + leftItems := leftByNameSize[key] + rightItems := rightByNameSize[key] + sort.Slice(leftItems, func(i, j int) bool { + return leftItems[i].RelativePath < leftItems[j].RelativePath + }) + sort.Slice(rightItems, func(i, j int) bool { + return rightItems[i].RelativePath < rightItems[j].RelativePath + }) + limit := minInt(len(leftItems), len(rightItems)) + for i := 0; i < limit; i++ { + pairs = append(pairs, comparePair{ + left: leftItems[i], + right: rightItems[i], + }) + fallbackMatchCount++ + } + for i := limit; i < len(leftItems); i++ { + missingLeft = append(missingLeft, leftItems[i].RelativePath) + } + for i := limit; i < len(rightItems); i++ { + missingRight = append(missingRight, rightItems[i].RelativePath) + } + } + + sort.Slice(pairs, func(i, j int) bool { + if pairs[i].left.FileName == pairs[j].left.FileName { + return pairs[i].left.RelativePath < pairs[j].left.RelativePath + } + return pairs[i].left.FileName < pairs[j].left.FileName + }) + sort.Strings(missingLeft) + sort.Strings(missingRight) + + result := make([]struct { + left candidateFileProbe + right candidateFileProbe + }, 0, len(pairs)) + for _, pair := range pairs { + result = append(result, struct { + left candidateFileProbe + right candidateFileProbe + }{ + left: pair.left, + right: pair.right, + }) + } + + return result, 0, fallbackMatchCount, missingLeft, missingRight +} + +func (s *Service) buildFileRedundancyHit( + toolboxPod *corev1.Pod, + prefixConfig ceph.StoragePrefixConfig, + scanID string, + workspace resolvedWorkspace, + entry model.StorageIndexEntry, + publicEntry model.StorageIndexEntry, + matchKey string, +) (*model.StorageIndexRedundancyHit, bool, error) { + baseHit := &model.StorageIndexRedundancyHit{ + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + ScanID: scanID, + TargetType: model.StorageIndexRedundancyTargetTypeFile, + TargetPath: entry.LogicalPath, + PublicPath: publicEntry.LogicalPath, + MatchKey: matchKey, + EstimatedBytes: entry.SizeBytes, + VerificationStatus: model.StorageIndexVerificationStatusSuspected, + VerificationMode: verificationModeMetadata, + Evidence: "文件名与文件大小匹配公共空间基线，疑似重复保存的公共资源文件", + Confidence: redundancyConfidenceMedium, + } + + targetHash, publicHash, err := s.computeCandidateHashes(toolboxPod, prefixConfig, entry.LogicalPath, publicEntry.LogicalPath) + if err != nil { + baseHit.VerificationMode = hashAlgorithmSHA256 + baseHit.HashAlgorithm = hashAlgorithmSHA256 + baseHit.Evidence = "文件名与文件大小匹配公共空间基线；已尝试进行 SHA256 校验，但校验过程失败，当前仍为疑似命中" + return baseHit, false, err + } + + if targetHash != publicHash { + return nil, false, nil + } + + baseHit.VerificationStatus = model.StorageIndexVerificationStatusVerified + baseHit.VerificationMode = hashAlgorithmSHA256 + baseHit.HashAlgorithm = hashAlgorithmSHA256 + baseHit.TargetHash = targetHash + baseHit.PublicHash = publicHash + baseHit.Evidence = "文件名与文件大小匹配公共空间基线，且 SHA256 校验一致，确认与公共空间资源重复" + baseHit.Confidence = redundancyConfidenceHigh + + return baseHit, true, nil +} + +func (s *Service) computeCandidateHashes( + toolboxPod *corev1.Pod, + prefixConfig ceph.StoragePrefixConfig, + targetLogicalPath string, + publicLogicalPath string, +) (string, string, error) { + targetHash, err := s.computeFileHash(toolboxPod, prefixConfig, targetLogicalPath) + if err != nil { + return "", "", err + } + publicHash, err := s.computeFileHash(toolboxPod, prefixConfig, publicLogicalPath) + if err != nil { + return "", "", err + } + return targetHash, publicHash, nil +} + +func (s *Service) computeFileHash( + toolboxPod *corev1.Pod, + prefixConfig ceph.StoragePrefixConfig, + logicalPath string, +) (string, error) { + actualPath, err := ceph.ResolveCephFSPath( + s.kubeClient, + s.kubeConfig, + toolboxNamespace, + logicalPath, + prefixConfig, + ) + if err != nil { + return "", fmt.Errorf("resolve logical path %s failed: %w", logicalPath, err) + } + + output, err := ceph.ExecInPod( + s.kubeClient, + s.kubeConfig, + toolboxPod, + []string{"sha256sum", actualPath}, + ) + if err != nil { + return "", fmt.Errorf("sha256sum %s failed: %w", logicalPath, err) + } + + fields := strings.Fields(strings.TrimSpace(output)) + if len(fields) == 0 { + return "", fmt.Errorf("empty sha256sum output for %s", logicalPath) + } + return fields[0], nil +} + +func shellQuote(value string) string { + return "'" + strings.ReplaceAll(value, "'", `'\''`) + "'" +} + +func parseEntryType(value string) model.StorageIndexEntryType { + switch strings.TrimSpace(value) { + case "f": + return model.StorageIndexEntryTypeFile + case "d": + return model.StorageIndexEntryTypeDir + case "l": + return model.StorageIndexEntryTypeSymlink + default: + return model.StorageIndexEntryTypeOther + } +} + +func parseInt64(value string) int64 { + parsed, err := strconv.ParseInt(strings.TrimSpace(value), 10, 64) + if err != nil { + return 0 + } + return parsed +} + +func parseUnixTimestamp(value string) *time.Time { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + return nil + } + floatValue, err := strconv.ParseFloat(trimmed, 64) + if err != nil { + return nil + } + seconds := int64(floatValue) + nanos := int64((floatValue - float64(seconds)) * float64(time.Second)) + timestamp := time.Unix(seconds, nanos).UTC().Truncate(time.Microsecond) + return ×tamp +} + +func formatTimeForLog(value *time.Time) string { + if value == nil { + return "nil" + } + return value.UTC().Format(time.RFC3339Nano) +} + +func normalizeUnixPath(value string) string { + cleaned := strings.ReplaceAll(strings.TrimSpace(value), "\\", "/") + if cleaned == "" { + return "" + } + return path.Clean(cleaned) +} + +func normalizeCompareLogicalPath(value string) string { + cleaned := normalizeUnixPath(value) + if cleaned == "" { + return "" + } + if !strings.HasPrefix(cleaned, "/") { + cleaned = "/" + cleaned + } + switch { + case strings.HasPrefix(cleaned, "/admin-user/"): + cleaned = "/user/" + strings.TrimPrefix(cleaned, "/admin-user/") + case cleaned == "/admin-user": + cleaned = "/user" + case strings.HasPrefix(cleaned, "/admin-account/"): + cleaned = "/account/" + strings.TrimPrefix(cleaned, "/admin-account/") + case cleaned == "/admin-account": + cleaned = "/account" + case strings.HasPrefix(cleaned, "/admin-public/"): + cleaned = "/public/" + strings.TrimPrefix(cleaned, "/admin-public/") + case cleaned == "/admin-public": + cleaned = "/public" + } + return normalizeUnixPath(cleaned) +} + +func normalizeDirectoryCompareType(value string) string { + switch strings.ToLower(strings.TrimSpace(value)) { + case "", "auto": + return "auto" + case "dataset", "dataset_dir": + return "dataset" + default: + return "model" + } +} + +func normalizeDirectoryCompareMode(value string) string { + switch strings.ToLower(strings.TrimSpace(value)) { + case "", compareModeOptimized: + return compareModeOptimized + case "baseline", "full-hash", compareModeFullHash: + return compareModeFullHash + default: + return compareModeOptimized + } +} + +func inferDirectoryCompareTypeFromFiles(leftFiles []candidateFileProbe, rightFiles []candidateFileProbe) string { + allFiles := append([]candidateFileProbe{}, leftFiles...) + allFiles = append(allFiles, rightFiles...) + for _, file := range allFiles { + lower := strings.ToLower(strings.TrimSpace(file.FileName)) + if isSafeTensorsFile(lower) || + strings.HasSuffix(lower, ".bin") || + strings.HasSuffix(lower, ".pt") || + strings.HasSuffix(lower, ".pth") || + strings.HasSuffix(lower, ".ckpt") || + strings.HasSuffix(lower, ".gguf") || + strings.HasSuffix(lower, ".index.json") || + lower == "config.json" || + lower == "tokenizer.json" || + lower == "tokenizer.model" || + lower == "tokenizer_config.json" || + lower == "generation_config.json" || + lower == "special_tokens_map.json" || + lower == "processor_config.json" || + lower == "preprocessor_config.json" || + lower == "adapter_config.json" || + lower == "model_index.json" { + return "model" + } + } + return "dataset" +} + +func (s *Service) scanFilesForDirectoryCompare( + toolboxPod *corev1.Pod, + leftActual string, + rightActual string, + compareType string, +) (string, []candidateFileProbe, []candidateFileProbe, error) { + normalizedType := normalizeDirectoryCompareType(compareType) + if normalizedType == "model" { + leftFiles, err := s.scanModelComparableFiles(toolboxPod, leftActual) + if err != nil { + return "", nil, nil, fmt.Errorf("scan left model files failed: %w", err) + } + rightFiles, err := s.scanModelComparableFiles(toolboxPod, rightActual) + if err != nil { + return "", nil, nil, fmt.Errorf("scan right model files failed: %w", err) + } + return normalizedType, leftFiles, rightFiles, nil + } + if normalizedType == "dataset" { + leftFiles, err := s.scanDatasetComparableFiles(toolboxPod, leftActual) + if err != nil { + return "", nil, nil, fmt.Errorf("scan left dataset files failed: %w", err) + } + rightFiles, err := s.scanDatasetComparableFiles(toolboxPod, rightActual) + if err != nil { + return "", nil, nil, fmt.Errorf("scan right dataset files failed: %w", err) + } + return normalizedType, leftFiles, rightFiles, nil + } + + leftModelFiles, err := s.scanModelComparableFiles(toolboxPod, leftActual) + if err != nil { + return "", nil, nil, fmt.Errorf("scan left model probe files failed: %w", err) + } + rightModelFiles, err := s.scanModelComparableFiles(toolboxPod, rightActual) + if err != nil { + return "", nil, nil, fmt.Errorf("scan right model probe files failed: %w", err) + } + inferredType := inferDirectoryCompareTypeFromFiles(leftModelFiles, rightModelFiles) + if inferredType == "model" { + return inferredType, leftModelFiles, rightModelFiles, nil + } + + leftFiles, err := s.scanDatasetComparableFiles(toolboxPod, leftActual) + if err != nil { + return "", nil, nil, fmt.Errorf("scan left dataset files failed: %w", err) + } + rightFiles, err := s.scanDatasetComparableFiles(toolboxPod, rightActual) + if err != nil { + return "", nil, nil, fmt.Errorf("scan right dataset files failed: %w", err) + } + return inferredType, leftFiles, rightFiles, nil +} + +func relativeUnixPath(rootPath, fullPath string) string { + root := strings.TrimSuffix(normalizeUnixPath(rootPath), "/") + full := normalizeUnixPath(fullPath) + if full == root { + return "." + } + return strings.TrimPrefix(full, root+"/") +} + +func relativeLogicalPath(rootPath, fullPath string) string { + return relativeUnixPath(rootPath, fullPath) +} + +func isTopLevelRelative(relativePath string) bool { + if relativePath == "" || relativePath == "." { + return false + } + return !strings.Contains(relativePath, "/") +} + +func depthFromRelative(relativePath string) int { + if relativePath == "" || relativePath == "." { + return 0 + } + return len(strings.Split(relativePath, "/")) +} + +func depthFromRoot(targetPath, rootPath string) int { + if normalizeUnixPath(targetPath) == normalizeUnixPath(rootPath) { + return 0 + } + relative := strings.TrimPrefix(normalizeUnixPath(targetPath), strings.TrimSuffix(normalizeUnixPath(rootPath), "/")+"/") + return depthFromRelative(relative) +} + +func isTopLevelPath(targetPath, rootPath string) bool { + return depthFromRoot(targetPath, rootPath) == 1 +} + +func parentForDirectory(targetPath, rootPath string) string { + cleanedTarget := normalizeUnixPath(targetPath) + cleanedRoot := normalizeUnixPath(rootPath) + if cleanedTarget == cleanedRoot { + return "" + } + parentPath := path.Dir(cleanedTarget) + if parentPath == "." { + return cleanedRoot + } + return parentPath +} + +func ancestorDirectories(targetPath, rootPath string) []string { + current := normalizeUnixPath(targetPath) + root := normalizeUnixPath(rootPath) + if current == "" || root == "" { + return nil + } + + result := make([]string, 0) + for { + result = append(result, current) + if current == root { + break + } + next := path.Dir(current) + if next == current || next == "." || next == "/" { + break + } + current = next + } + + return result +} + +func collapsePathPrefixes(paths []string) []string { + if len(paths) == 0 { + return nil + } + + sorted := append([]string{}, paths...) + sort.Slice(sorted, func(i, j int) bool { + depthI := strings.Count(strings.TrimPrefix(normalizeUnixPath(sorted[i]), "/"), "/") + depthJ := strings.Count(strings.TrimPrefix(normalizeUnixPath(sorted[j]), "/"), "/") + if depthI == depthJ { + return sorted[i] < sorted[j] + } + return depthI < depthJ + }) + + collapsed := make([]string, 0, len(sorted)) + for _, item := range sorted { + cleaned := normalizeUnixPath(item) + if cleaned == "" || isCoveredByPrefixes(cleaned, collapsed) { + continue + } + collapsed = append(collapsed, cleaned) + } + return collapsed +} + +func pathOverlaps(leftPath, rightPath string) bool { + left := normalizeUnixPath(leftPath) + right := normalizeUnixPath(rightPath) + if left == "" || right == "" { + return false + } + return left == right || strings.HasPrefix(left, right+"/") || strings.HasPrefix(right, left+"/") +} + +func redundancyDirectoryKey(name string, size int64) string { + return strings.ToLower(strings.TrimSpace(name)) + "|" + strconv.FormatInt(size, 10) +} + +func redundancyFileKey(name string, size int64) string { + return strings.ToLower(strings.TrimSpace(name)) + "|" + strconv.FormatInt(size, 10) +} + +func coveredByDirectoryHit(targetPath string, prefixes []string) bool { + for _, prefix := range prefixes { + if targetPath == prefix || strings.HasPrefix(targetPath, prefix+"/") { + return true + } + } + return false +} + +func buildDirectorySignature(metric *model.StorageIndexDirectoryMetric, childDirs []string) string { + mtime := "0" + if metric.LatestModifiedAt != nil { + mtime = metric.LatestModifiedAt.UTC().Format(time.RFC3339) + } + childSet := strings.Join(childDirs, ",") + raw := fmt.Sprintf( + "%s|%d|%d|%d|%s|%s", + strings.ToLower(metric.Name), + metric.TotalSizeBytes, + metric.ImmediateChildDirCount, + metric.ImmediateChildFileCount, + mtime, + childSet, + ) + sum := sha1.Sum([]byte(raw)) + return hex.EncodeToString(sum[:]) +} + +func hashString(value string) string { + sum := sha1.Sum([]byte(value)) + return hex.EncodeToString(sum[:]) +} + +func shouldApplyTopLevelModelCopyPrefilter(workspace resolvedWorkspace) bool { + return workspace.WorkspaceType == model.StorageIndexWorkspaceTypeUser +} + +func shouldSkipTopLevelModelCopyName(name string) bool { + lower := strings.ToLower(strings.TrimSpace(name)) + _, blocked := definitelyNotPublicModelCopyTopLevelDirSet[lower] + return blocked +} + +func shouldSkipTopLevelModelCopySignature(sig topLevelSignature) bool { + if sig.EntryType != model.StorageIndexEntryTypeDir { + return false + } + return shouldSkipTopLevelModelCopyName(sig.Name) +} + +func filterTopLevelSignaturesForModelCopyScan( + signatures map[string]topLevelSignature, +) ([]topLevelSignature, []topLevelSignature) { + selected := make([]topLevelSignature, 0, len(signatures)) + skipped := make([]topLevelSignature, 0) + for _, sig := range signatures { + if shouldSkipTopLevelModelCopySignature(sig) { + skipped = append(skipped, sig) + continue + } + selected = append(selected, sig) + } + sort.Slice(selected, func(i, j int) bool { + return selected[i].LogicalPath < selected[j].LogicalPath + }) + sort.Slice(skipped, func(i, j int) bool { + return skipped[i].LogicalPath < skipped[j].LogicalPath + }) + return selected, skipped +} + +func shouldSkipTopLevelModelCopyCandidate(metric model.StorageIndexDirectoryMetric) bool { + if !metric.IsTopLevel { + return false + } + return shouldSkipTopLevelModelCopyName(metric.Name) +} + +func immediateSubtreeAllowListForTopLevel(name string) (map[string]struct{}, bool) { + allowList, ok := selectiveTopLevelSubtreeAllowLists[strings.ToLower(strings.TrimSpace(name))] + return allowList, ok +} + +func filterImmediateSubtreesByAllowList( + signatures map[string]topLevelSignature, + allowList map[string]struct{}, +) ([]topLevelSignature, []topLevelSignature) { + selected := make([]topLevelSignature, 0, len(signatures)) + skipped := make([]topLevelSignature, 0) + for _, sig := range signatures { + if sig.EntryType != model.StorageIndexEntryTypeDir { + continue + } + if _, ok := allowList[strings.ToLower(strings.TrimSpace(sig.Name))]; ok { + selected = append(selected, sig) + continue + } + skipped = append(skipped, sig) + } + sort.Slice(selected, func(i, j int) bool { + return selected[i].LogicalPath < selected[j].LogicalPath + }) + sort.Slice(skipped, func(i, j int) bool { + return skipped[i].LogicalPath < skipped[j].LogicalPath + }) + return selected, skipped +} + +func classifyDirectory(targetPath string) string { + lower := strings.ToLower(targetPath) + switch { + case strings.Contains(lower, "models"), + strings.Contains(lower, "model"), + strings.Contains(lower, "checkpoint"), + strings.Contains(lower, "checkpoints"), + strings.Contains(lower, "huggingface"), + strings.Contains(lower, "transformers"): + return "model_dir" + case strings.Contains(lower, "datasets"), + strings.Contains(lower, "dataset"), + strings.Contains(lower, "/data"): + return "dataset_dir" + default: + return "" + } +} + +func computeCandidateScore(metric *model.StorageIndexDirectoryMetric) float64 { + score := 0.0 + if metric.CategoryHint != "" { + score += 10 + } + if metric.TotalSizeBytes > 0 { + score += float64(metric.TotalSizeBytes) / float64(1024*1024*1024) + } + if metric.ImmediateChildDirCount > 0 { + score += float64(metric.ImmediateChildDirCount) * 0.5 + } + return score +} + +func inferCandidateTypeFromPublicPath(publicPath string) string { + return classifyDirectory(publicPath) +} + +func datasetTypeToCandidateType(dataType model.DataType) string { + switch dataType { + case model.DataTypeModel: + return "model_dir" + case model.DataTypeDataset: + return "dataset_dir" + default: + return "" + } +} + +func logicalPublicPathFromDatasetURL(prefixConfig ceph.StoragePrefixConfig, rawURL string) string { + normalized := normalizeUnixPath(rawURL) + publicPrefix := normalizeUnixPath(prefixConfig.Public) + publicRoot := normalizeUnixPath("/public") + + if normalized == publicPrefix { + return publicRoot + } + if strings.HasPrefix(normalized, publicPrefix+"/") { + return normalizeUnixPath(publicRoot + strings.TrimPrefix(normalized, publicPrefix)) + } + if strings.HasPrefix(normalized, "/"+publicPrefix+"/") { + return normalizeUnixPath(publicRoot + strings.TrimPrefix(normalized, "/"+publicPrefix)) + } + if strings.HasPrefix(normalized, "/public/") || normalized == "/public" { + return normalized + } + return "" +} + +func (s *Service) listPublicResourceRoots( + ctx context.Context, + prefixConfig ceph.StoragePrefixConfig, +) ([]publicResourceRoot, error) { + var datasets []model.Dataset + if err := query.GetDB().WithContext(ctx). + Where("type IN ? AND deleted_at IS NULL", []model.DataType{model.DataTypeModel, model.DataTypeDataset}). + Find(&datasets).Error; err != nil { + return nil, fmt.Errorf("query public resource registry failed: %w", err) + } + + roots := make([]publicResourceRoot, 0) + seen := make(map[string]struct{}) + for _, dataset := range datasets { + logicalPath := logicalPublicPathFromDatasetURL(prefixConfig, dataset.URL) + if logicalPath == "" || logicalPath == "/public" { + continue + } + if _, ok := seen[logicalPath]; ok { + continue + } + seen[logicalPath] = struct{}{} + roots = append(roots, publicResourceRoot{ + Name: loCoalesce(strings.TrimSpace(dataset.Name), path.Base(logicalPath)), + LogicalPath: logicalPath, + Category: datasetTypeToCandidateType(dataset.Type), + }) + } + + sort.Slice(roots, func(i, j int) bool { + return roots[i].LogicalPath < roots[j].LogicalPath + }) + return roots, nil +} + +func loCoalesce(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return value + } + } + return "" +} + +func insertEntriesInChunks( + tx *gorm.DB, + scanID string, + workspace resolvedWorkspace, + items []model.StorageIndexEntry, + chunkSize int, +) error { + return insertInChunks(tx, items, chunkSize, func(start, end int) error { + klog.Infof( + "storageindex: 分批写入条目 scan_id=%s workspace_type=%s workspace_name=%s start=%d end=%d total=%d", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, start, end, len(items), + ) + batch := items[start:end] + return tx.Create(&batch).Error + }) +} + +func insertDirectoryMetricsInChunks( + tx *gorm.DB, + scanID string, + workspace resolvedWorkspace, + items []model.StorageIndexDirectoryMetric, + chunkSize int, +) error { + return insertInChunks(tx, items, chunkSize, func(start, end int) error { + klog.Infof( + "storageindex: 分批写入目录聚合 scan_id=%s workspace_type=%s workspace_name=%s start=%d end=%d total=%d", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, start, end, len(items), + ) + batch := items[start:end] + return tx.Create(&batch).Error + }) +} + +func insertRedundancyHitsInChunks( + tx *gorm.DB, + scanID string, + workspace resolvedWorkspace, + items []model.StorageIndexRedundancyHit, + chunkSize int, +) error { + return insertInChunks(tx, items, chunkSize, func(start, end int) error { + klog.Infof( + "storageindex: 分批写入冗余命中 scan_id=%s workspace_type=%s workspace_name=%s start=%d end=%d total=%d", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, start, end, len(items), + ) + batch := items[start:end] + return tx.Create(&batch).Error + }) +} + +func insertCandidatesInChunks( + tx *gorm.DB, + scanID string, + workspace resolvedWorkspace, + items []model.StorageIndexCandidate, + chunkSize int, +) error { + return insertInChunks(tx, items, chunkSize, func(start, end int) error { + klog.Infof( + "storageindex: 分批写入候选目录 scan_id=%s workspace_type=%s workspace_name=%s start=%d end=%d total=%d", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, start, end, len(items), + ) + batch := items[start:end] + return tx.Create(&batch).Error + }) +} + +func insertCandidateFilesInChunks( + tx *gorm.DB, + scanID string, + workspace resolvedWorkspace, + items []model.StorageIndexCandidateFile, + chunkSize int, +) error { + return insertInChunks(tx, items, chunkSize, func(start, end int) error { + klog.Infof( + "storageindex: 分批写入候选关键文件 scan_id=%s workspace_type=%s workspace_name=%s start=%d end=%d total=%d", + scanID, workspace.WorkspaceType, workspace.WorkspaceName, start, end, len(items), + ) + batch := items[start:end] + return tx.Create(&batch).Error + }) +} + +func insertPublicBaselineFilesInChunks( + tx *gorm.DB, + scanID string, + items []model.StorageIndexPublicFileBaseline, + chunkSize int, +) error { + return insertInChunks(tx, items, chunkSize, func(start, end int) error { + klog.Infof( + "storageindex: 分批写入公共关键文件基线 scan_id=%s start=%d end=%d total=%d", + scanID, start, end, len(items), + ) + batch := items[start:end] + return tx.Create(&batch).Error + }) +} + +func insertPublicRootBaselinesInChunks( + tx *gorm.DB, + scanID string, + items []model.StorageIndexPublicRootBaseline, + chunkSize int, +) error { + return insertInChunks(tx, items, chunkSize, func(start, end int) error { + klog.Infof( + "storageindex: 分批写入公共资源根基线 scan_id=%s start=%d end=%d total=%d", + scanID, start, end, len(items), + ) + batch := items[start:end] + return tx.Create(&batch).Error + }) +} + +func insertInChunks[T any](_ *gorm.DB, items []T, chunkSize int, insertFn func(start, end int) error) error { + if chunkSize <= 0 { + chunkSize = insertBatchSize + } + for start := 0; start < len(items); start += chunkSize { + end := start + chunkSize + if end > len(items) { + end = len(items) + } + if err := insertFn(start, end); err != nil { + return err + } + } + return nil +} + +type findProgressWriter struct { + scanID string + workspaceType model.StorageIndexWorkspaceType + workspaceName string + everyRecords int64 + recordCount int64 + nextLogAt int64 +} + +func (w *findProgressWriter) Write(p []byte) (int, error) { + if w.everyRecords <= 0 { + w.everyRecords = scanProgressLogEveryRecord + } + if w.nextLogAt == 0 { + w.nextLogAt = w.everyRecords + } + + count := int64(bytes.Count(p, []byte(findRecordSeparator))) + if count == 0 { + return len(p), nil + } + + w.recordCount += count + for w.recordCount >= w.nextLogAt { + klog.Infof( + "storageindex: find 流式扫描进度 scan_id=%s workspace_type=%s workspace_name=%s discovered_records=%d", + w.scanID, w.workspaceType, w.workspaceName, w.recordCount, + ) + w.nextLogAt += w.everyRecords + } + + return len(p), nil +} diff --git a/backend/pkg/storageindex/service_test.go b/backend/pkg/storageindex/service_test.go new file mode 100644 index 000000000..80b3c8eb7 --- /dev/null +++ b/backend/pkg/storageindex/service_test.go @@ -0,0 +1,870 @@ +//nolint:goconst,gosec // Tests repeat literal fixture paths and guard slice lengths before fixed-index checks. +package storageindex + +import ( + "path/filepath" + "strings" + "testing" + "time" + + "gorm.io/driver/sqlite" + "gorm.io/gorm" + + "github.com/raids-lab/crater/dao/model" +) + +const aliceModelsPath = "/user/alice/models" + +func TestVerificationModesFitStorageIndexColumns(t *testing.T) { + t.Parallel() + + modes := map[string]string{ + "metadata": verificationModeMetadata, + "file_name_size": verificationModeFileName, + "sha256": hashAlgorithmSHA256, + "sampled_sha256": hashAlgorithmSampledSHA256, + "safetensors header + hash": verificationModeSafeTensorsHdrAndSampledSHA, + } + + for name, mode := range modes { + if len(mode) > 32 { + t.Fatalf("%s exceeds varchar(32): %q (%d)", name, mode, len(mode)) + } + } +} + +func TestBuildDirectoryMetricsAggregatesFilesToAncestorDirectories(t *testing.T) { + workspace := resolvedWorkspace{ + WorkspaceType: model.StorageIndexWorkspaceTypeUser, + WorkspaceName: "alice", + LogicalPath: "/user/alice-space", + } + + entries := []model.StorageIndexEntry{ + { + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + LogicalPath: workspace.LogicalPath, + RelativePath: ".", + Name: "alice-space", + EntryType: model.StorageIndexEntryTypeDir, + }, + { + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + LogicalPath: "/user/alice-space/models", + RelativePath: "models", + ParentPath: workspace.LogicalPath, + Name: "models", + EntryType: model.StorageIndexEntryTypeDir, + IsTopLevel: true, + }, + { + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + LogicalPath: "/user/alice-space/models/model.bin", + RelativePath: "models/model.bin", + ParentPath: "/user/alice-space/models", + Name: "model.bin", + EntryType: model.StorageIndexEntryTypeFile, + SizeBytes: 128, + }, + } + + metrics, totalSize := buildDirectoryMetrics("scan-1", workspace, entries) + if totalSize != 128 { + t.Fatalf("expected root total size 128, got %d", totalSize) + } + + metricByPath := make(map[string]model.StorageIndexDirectoryMetric, len(metrics)) + for _, item := range metrics { + metricByPath[item.Path] = item + } + + rootMetric, ok := metricByPath["/user/alice-space"] + if !ok { + t.Fatalf("root metric not found") + } + if rootMetric.FileCount != 1 { + t.Fatalf("expected root file count 1, got %d", rootMetric.FileCount) + } + if rootMetric.DirectoryCount != 1 { + t.Fatalf("expected root directory count 1, got %d", rootMetric.DirectoryCount) + } + if rootMetric.TotalSizeBytes != 128 { + t.Fatalf("expected root total size 128, got %d", rootMetric.TotalSizeBytes) + } + + modelsMetric, ok := metricByPath["/user/alice-space/models"] + if !ok { + t.Fatalf("models metric not found") + } + if modelsMetric.FileCount != 1 { + t.Fatalf("expected models file count 1, got %d", modelsMetric.FileCount) + } + if modelsMetric.TotalSizeBytes != 128 { + t.Fatalf("expected models total size 128, got %d", modelsMetric.TotalSizeBytes) + } + + if _, exists := metricByPath["/user/alice-space/models/model.bin"]; exists { + t.Fatalf("file path should not be materialized as directory metric") + } +} + +func TestCoveredByDirectoryHit(t *testing.T) { + prefixes := []string{"/user/alice-space/models"} + if !coveredByDirectoryHit("/user/alice-space/models/model.bin", prefixes) { + t.Fatalf("expected file to be covered by redundant directory prefix") + } + if coveredByDirectoryHit("/user/alice-space/logs/train.log", prefixes) { + t.Fatalf("did not expect unrelated path to be covered by redundant directory prefix") + } +} + +func TestShouldSkipTopLevelModelCopyCandidate(t *testing.T) { + tests := []struct { + name string + metric model.StorageIndexDirectoryMetric + want bool + }{ + { + name: "skip top level conda root", + metric: model.StorageIndexDirectoryMetric{ + Name: "conda", + IsTopLevel: true, + }, + want: true, + }, + { + name: "skip top level codex workspace", + metric: model.StorageIndexDirectoryMetric{ + Name: ".codex", + IsTopLevel: true, + }, + want: true, + }, + { + name: "skip top level local runtime dir", + metric: model.StorageIndexDirectoryMetric{ + Name: ".local", + IsTopLevel: true, + }, + want: true, + }, + { + name: "skip top level pip cache dir", + metric: model.StorageIndexDirectoryMetric{ + Name: ".pip", + IsTopLevel: true, + }, + want: true, + }, + { + name: "skip top level nvidia cache dir", + metric: model.StorageIndexDirectoryMetric{ + Name: ".nv", + IsTopLevel: true, + }, + want: true, + }, + { + name: "keep huggingface cache because it may store duplicated public models", + metric: model.StorageIndexDirectoryMetric{ + Name: ".cache", + IsTopLevel: true, + }, + want: false, + }, + { + name: "keep outputs because some jobs write checkpoints there", + metric: model.StorageIndexDirectoryMetric{ + Name: "outputs", + IsTopLevel: true, + }, + want: false, + }, + { + name: "do not skip nested env-style names", + metric: model.StorageIndexDirectoryMetric{ + Name: ".venv", + IsTopLevel: false, + }, + want: false, + }, + } + + for _, tt := range tests { + if got := shouldSkipTopLevelModelCopyCandidate(tt.metric); got != tt.want { + t.Fatalf("%s: expected %t, got %t", tt.name, tt.want, got) + } + } +} + +func TestSkippedTopLevelModelCopyRootPrunesDescendants(t *testing.T) { + skippedRoots := appendUniquePrefix(nil, "/user/alice-space/conda") + if !isCoveredByPrefixes("/user/alice-space/conda/envs/base/lib/python3.10/site-packages", skippedRoots) { + t.Fatalf("expected descendants under skipped top-level root to be pruned") + } + if isCoveredByPrefixes("/user/alice-space/models/llama-7b", skippedRoots) { + t.Fatalf("did not expect unrelated model directory to be pruned") + } +} + +func TestFilterTopLevelSignaturesForModelCopyScan(t *testing.T) { + signatures := map[string]topLevelSignature{ + "conda": { + Name: "conda", + LogicalPath: "/user/alice-space/conda", + EntryType: model.StorageIndexEntryTypeDir, + }, + "models": { + Name: "models", + LogicalPath: "/user/alice-space/models", + EntryType: model.StorageIndexEntryTypeDir, + }, + ".cache": { + Name: ".cache", + LogicalPath: "/user/alice-space/.cache", + EntryType: model.StorageIndexEntryTypeDir, + }, + ".codex": { + Name: ".codex", + LogicalPath: "/user/alice-space/.codex", + EntryType: model.StorageIndexEntryTypeDir, + }, + } + + selected, skipped := filterTopLevelSignaturesForModelCopyScan(signatures) + if len(selected) != 2 { + t.Fatalf("expected 2 selected top-level signatures, got %d", len(selected)) + } + if len(skipped) != 2 { + t.Fatalf("expected 2 skipped top-level signatures, got %d", len(skipped)) + } + skippedNames := []string{skipped[0].Name, skipped[1].Name} + if skippedNames[0] != ".codex" || skippedNames[1] != "conda" { + t.Fatalf("unexpected skipped names: %v", skippedNames) + } + selectedNames := []string{selected[0].Name, selected[1].Name} + if selectedNames[0] != ".cache" || selectedNames[1] != "models" { + t.Fatalf("unexpected selected names: %v", selectedNames) + } +} + +func TestImmediateSubtreeAllowListForCache(t *testing.T) { + allowList, ok := immediateSubtreeAllowListForTopLevel(".cache") + if !ok { + t.Fatalf("expected .cache allowlist to exist") + } + if _, ok := allowList["huggingface"]; !ok { + t.Fatalf("expected huggingface to be retained under .cache") + } + if _, ok := allowList["pip"]; ok { + t.Fatalf("did not expect pip to be retained under .cache") + } +} + +func TestFilterImmediateSubtreesByAllowList(t *testing.T) { + signatures := map[string]topLevelSignature{ + "huggingface": { + Name: "huggingface", + LogicalPath: "/user/alice-space/.cache/huggingface", + EntryType: model.StorageIndexEntryTypeDir, + }, + "modelscope": { + Name: "modelscope", + LogicalPath: "/user/alice-space/.cache/modelscope", + EntryType: model.StorageIndexEntryTypeDir, + }, + "pip": { + Name: "pip", + LogicalPath: "/user/alice-space/.cache/pip", + EntryType: model.StorageIndexEntryTypeDir, + }, + "readme.txt": { + Name: "readme.txt", + LogicalPath: "/user/alice-space/.cache/readme.txt", + EntryType: model.StorageIndexEntryTypeFile, + }, + } + + allowList, _ := immediateSubtreeAllowListForTopLevel(".cache") + selected, skipped := filterImmediateSubtreesByAllowList(signatures, allowList) + if len(selected) != 2 { + t.Fatalf("expected 2 selected cache subtrees, got %d", len(selected)) + } + if len(skipped) != 1 { + t.Fatalf("expected 1 skipped cache subtree, got %d", len(skipped)) + } + + selectedNames := []string{selected[0].Name, selected[1].Name} + if selectedNames[0] != "huggingface" || selectedNames[1] != "modelscope" { + t.Fatalf("unexpected selected cache subtree names: %v", selectedNames) + } + if skipped[0].Name != "pip" { + t.Fatalf("expected pip to be skipped, got %s", skipped[0].Name) + } +} + +func TestSortedPrunableNestedDirNamesIncludesVenv(t *testing.T) { + names := sortedPrunableNestedDirNames() + foundVenv := false + foundPycache := false + for _, name := range names { + if name == ".venv" { + foundVenv = true + } + if name == "__pycache__" { + foundPycache = true + } + } + if !foundVenv || !foundPycache { + t.Fatalf("expected recursive prune names to include .venv and __pycache__, got %v", names) + } +} + +func TestBuildDirectoryScanScriptPrunesNestedEnvDirs(t *testing.T) { + script := buildDirectoryScanScript("/user/alice-space/models", nil) + if !strings.Contains(script, "-name '.venv'") { + t.Fatalf("expected script to prune nested .venv directories, got %s", script) + } + if !strings.Contains(script, "-name '__pycache__'") { + t.Fatalf("expected script to prune nested __pycache__ directories, got %s", script) + } +} + +func TestBuildPublicRootLookupIndexesResourceAndPathBase(t *testing.T) { + publicRoots := []model.StorageIndexPublicRootBaseline{ + { + ResourceName: "Qwen2.5-7B", + LogicalPath: "/public/models/Qwen2.5-7B", + }, + } + + lookup := buildPublicRootLookup(publicRoots) + if len(lookup["qwen2.5-7b"]) != 1 { + t.Fatalf("expected deduplicated root lookup entry, got %d entries", len(lookup["qwen2.5-7b"])) + } +} + +func TestParseSha256sumOutput(t *testing.T) { + output := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa /mnt/mycephfs/models/model.bin\n" + + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb /mnt/mycephfs/models/path with spaces/config.json\n" + + parsed := parseSha256sumOutput(output) + if parsed["/mnt/mycephfs/models/model.bin"] != "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" { + t.Fatalf("failed to parse normal sha256sum output") + } + if parsed["/mnt/mycephfs/models/path with spaces/config.json"] != "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" { + t.Fatalf("failed to parse sha256sum output with spaces in path") + } +} + +func TestParseSafeTensorsHeaderSkeleton(t *testing.T) { + header := `{ + "__metadata__": {"format":"pt"}, + "model.layers.1.weight": {"dtype":"F16","shape":[3,4],"data_offsets":[100,200]}, + "model.layers.0.weight": {"dtype":"F16","shape":[1,2],"data_offsets":[0,99]} + }` + + skeleton, err := parseSafeTensorsHeaderSkeleton(header) + if err != nil { + t.Fatalf("unexpected parse error: %v", err) + } + if len(skeleton) != 2 { + t.Fatalf("expected 2 tensor entries, got %d", len(skeleton)) + } + if skeleton[0] != "model.layers.0.weight=1,2" { + t.Fatalf("unexpected first skeleton entry: %s", skeleton[0]) + } + if skeleton[1] != "model.layers.1.weight=3,4" { + t.Fatalf("unexpected second skeleton entry: %s", skeleton[1]) + } +} + +func TestBuildSampledRegions(t *testing.T) { + regions := buildSampledRegions(1024 * 1024) + if len(regions) != 3 { + t.Fatalf("expected 3 sampled regions, got %d", len(regions)) + } + if regions[0].Offset != 262144 { + t.Fatalf("unexpected 25%% offset: %d", regions[0].Offset) + } + if regions[1].Offset != 524288 { + t.Fatalf("unexpected 50%% offset: %d", regions[1].Offset) + } + if regions[2].Offset != 786432 { + t.Fatalf("unexpected 75%% offset: %d", regions[2].Offset) + } + + small := buildSampledRegions(1024) + if len(small) != 1 || small[0].Offset != 0 || small[0].Count != 1024 { + t.Fatalf("unexpected sampled region for small file: %+v", small) + } +} + +func TestCloneRedundancyHitsForScanRetagsExistingRows(t *testing.T) { + now := time.Unix(1714377600, 0) + hits := []model.StorageIndexRedundancyHit{ + { + ID: 42, + WorkspaceType: model.StorageIndexWorkspaceTypeUser, + WorkspaceName: "alice", + ScanID: "scan-old", + TargetType: model.StorageIndexRedundancyTargetTypeDirectory, + TargetPath: "/user/alice/models", + PublicPath: "/public/models/foo", + MatchKey: "models|123", + VerificationStatus: model.StorageIndexVerificationStatusSuspected, + EstimatedBytes: 123, + CreatedAt: now, + UpdatedAt: now, + }, + } + + cloned := cloneRedundancyHitsForScan("scan-new", hits) + if len(cloned) != 1 { + t.Fatalf("expected 1 cloned hit, got %d", len(cloned)) + } + if cloned[0].ID != 0 { + t.Fatalf("expected cloned hit id reset to 0, got %d", cloned[0].ID) + } + if cloned[0].ScanID != "scan-new" { + t.Fatalf("expected cloned hit scan id to be retagged, got %q", cloned[0].ScanID) + } + if !cloned[0].CreatedAt.IsZero() || !cloned[0].UpdatedAt.IsZero() { + t.Fatalf("expected cloned hit timestamps to be cleared, got created=%v updated=%v", cloned[0].CreatedAt, cloned[0].UpdatedAt) + } + if cloned[0].TargetPath != hits[0].TargetPath || cloned[0].PublicPath != hits[0].PublicPath { + t.Fatalf("expected cloned hit payload to be preserved, got %+v", cloned[0]) + } + if hits[0].ID != 42 || hits[0].ScanID != "scan-old" { + t.Fatalf("expected original hit to remain unchanged, got %+v", hits[0]) + } +} + +func TestExpandIncrementalPlanWithCandidateRootsPromotesCandidateAncestor(t *testing.T) { + workspace := resolvedWorkspace{ + WorkspaceType: model.StorageIndexWorkspaceTypeUser, + WorkspaceName: "alice", + LogicalPath: "/user/alice", + } + plan := &incrementalPlan{ + RescanTargets: []topLevelSignature{ + { + Name: "subdir", + LogicalPath: "/user/alice/models/subdir", + ParentLogicalPath: "/user/alice/models", + ActualPath: "/snap/alice/models/subdir", + EntryType: model.StorageIndexEntryTypeDir, + }, + }, + UpsertEntries: []model.StorageIndexEntry{ + { + LogicalPath: "/user/alice/models/subdir/file.bin", + EntryType: model.StorageIndexEntryTypeFile, + }, + }, + RemovedPrefixes: []string{}, + } + existingCandidates := []model.StorageIndexCandidate{ + {TargetPath: aliceModelsPath}, + {TargetPath: "/user/alice/unrelated"}, + } + + expanded := expandIncrementalPlanWithCandidateRoots("scan-1", workspace, "/snap/alice", plan, existingCandidates) + if len(expanded.RescanTargets) != 1 { + t.Fatalf("expected 1 expanded rescan target, got %d", len(expanded.RescanTargets)) + } + if expanded.RescanTargets[0].LogicalPath != aliceModelsPath { + t.Fatalf("expected candidate ancestor to become rescan target, got %s", expanded.RescanTargets[0].LogicalPath) + } + if expanded.RescanTargets[0].ActualPath != "/snap/alice/models" { + t.Fatalf("expected candidate ancestor actual path to be projected from snapshot root, got %s", expanded.RescanTargets[0].ActualPath) + } + if len(expanded.UpsertEntries) != 0 { + t.Fatalf("expected nested upsert entries to be subsumed by candidate root rescan, got %d entries", len(expanded.UpsertEntries)) + } +} + +func TestCollectAffectedCandidatePathsIncludesExactAndRemovedPrefixes(t *testing.T) { + existingCandidates := []model.StorageIndexCandidate{ + {TargetPath: "/user/alice/models"}, + {TargetPath: "/user/alice/removed/old-copy"}, + {TargetPath: "/user/alice/unrelated"}, + } + + affected := collectAffectedCandidatePaths( + existingCandidates, + []string{"/user/alice/models", "/user/alice"}, + []string{"/user/alice/removed"}, + ) + + if len(affected) != 2 { + t.Fatalf("expected 2 affected candidate paths, got %d: %v", len(affected), affected) + } + if affected[0] != "/user/alice/models" && affected[1] != "/user/alice/models" { + t.Fatalf("expected exact affected candidate path to be included, got %v", affected) + } + if affected[0] != "/user/alice/removed/old-copy" && affected[1] != "/user/alice/removed/old-copy" { + t.Fatalf("expected removed-prefix candidate path to be included, got %v", affected) + } +} + +func TestMergeExistingCandidateBindingsPreservesPublicPathAfterSizeDrift(t *testing.T) { + workspace := resolvedWorkspace{ + WorkspaceType: model.StorageIndexWorkspaceTypeUser, + WorkspaceName: "alice", + LogicalPath: "/user/alice", + } + candidates := []model.StorageIndexCandidate{ + { + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + ScanID: "scan-new", + CandidateType: "model_dir", + TargetPath: "/user/alice/models/qwen", + PublicPath: "", + Evidence: "category hint only", + CandidateScore: 48, + Status: model.StorageIndexCandidateStatusSuspected, + }, + } + existing := []model.StorageIndexCandidate{ + { + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + ScanID: "scan-old", + CandidateType: "model_dir", + TargetPath: "/user/alice/models/qwen", + PublicPath: "/public/models/qwen", + Evidence: "previous verified match", + CandidateScore: 100, + Status: model.StorageIndexCandidateStatusVerified, + }, + } + + merged := mergeExistingCandidateBindings("scan-new", workspace, candidates, existing) + if len(merged) != 1 { + t.Fatalf("expected 1 merged candidate, got %d", len(merged)) + } + if merged[0].PublicPath != "/public/models/qwen" { + t.Fatalf("expected public path binding to be preserved, got %q", merged[0].PublicPath) + } + if merged[0].CandidateScore != 100 { + t.Fatalf("expected candidate score to keep stronger historical score, got %v", merged[0].CandidateScore) + } +} + +func TestMergeExistingCandidateBindingsAppendsHistoricalCandidateForRevalidation(t *testing.T) { + workspace := resolvedWorkspace{ + WorkspaceType: model.StorageIndexWorkspaceTypeUser, + WorkspaceName: "alice", + LogicalPath: "/user/alice", + } + existing := []model.StorageIndexCandidate{ + { + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + ScanID: "scan-old", + CandidateType: "model_dir", + TargetPath: "/user/alice/models/llama", + PublicPath: "/public/models/llama", + Evidence: "previous verified match", + CandidateScore: 100, + Status: model.StorageIndexCandidateStatusVerified, + }, + } + + merged := mergeExistingCandidateBindings("scan-new", workspace, nil, existing) + if len(merged) != 1 { + t.Fatalf("expected historical candidate to be appended for revalidation, got %d", len(merged)) + } + if merged[0].TargetPath != "/user/alice/models/llama" || merged[0].PublicPath != "/public/models/llama" { + t.Fatalf("expected appended candidate to keep previous target/public binding, got %+v", merged[0]) + } +} + +func TestCompareStrictDirectoryFileSetsDetectsAddedFiles(t *testing.T) { + targetFiles := []candidateFileProbe{ + {RelativePath: "config.json", SizeBytes: 100}, + {RelativePath: "weights.bin", SizeBytes: 200}, + {RelativePath: "extra.txt", SizeBytes: 50}, + } + publicFiles := []candidateFileProbe{ + {RelativePath: "config.json", SizeBytes: 100}, + {RelativePath: "weights.bin", SizeBytes: 200}, + } + + missingTarget, missingPublic := compareStrictDirectoryFileSets(targetFiles, publicFiles) + if len(missingTarget) != 0 { + t.Fatalf("expected public baseline not to miss target-required files, got %v", missingTarget) + } + if len(missingPublic) != 1 || missingPublic[0] != "extra.txt" { + t.Fatalf("expected added target file to appear in missing public set, got %v", missingPublic) + } +} + +func TestAllCandidateFilesVerifiedRequiresEveryFileVerified(t *testing.T) { + files := []model.StorageIndexCandidateFile{ + {VerificationStatus: model.StorageIndexVerificationStatusVerified}, + {VerificationStatus: model.StorageIndexVerificationStatusSuspected}, + } + if allCandidateFilesVerified(files) { + t.Fatalf("expected mixed verification states to fail full candidate verification") + } + if !allCandidateFilesVerified([]model.StorageIndexCandidateFile{{VerificationStatus: model.StorageIndexVerificationStatusVerified}}) { + t.Fatalf("expected fully verified file list to pass") + } +} + +func TestDiffTopLevelSignaturesDetectsRecursiveChangedAtUpdate(t *testing.T) { + currentChangedAt := time.Unix(1714480000, 0) + previousChangedAt := currentChangedAt.Add(-time.Second) + current := map[string]topLevelSignature{ + "models": { + Name: "models", + LogicalPath: "/user/alice/models", + EntryType: model.StorageIndexEntryTypeDir, + SizeBytes: 1024, + ChangedAt: ¤tChangedAt, + }, + } + previous := map[string]topLevelSignature{ + "models": { + Name: "models", + LogicalPath: "/user/alice/models", + EntryType: model.StorageIndexEntryTypeDir, + SizeBytes: 1024, + ChangedAt: &previousChangedAt, + }, + } + + changed, removed, changedCount := diffTopLevelSignatures(current, previous) + if len(removed) != 0 { + t.Fatalf("expected no removed prefixes, got %v", removed) + } + if changedCount != 1 || len(changed) != 1 || changed[0].LogicalPath != "/user/alice/models" { + t.Fatalf("expected recursive changed time drift to be detected, got changed=%v count=%d", changed, changedCount) + } +} + +func TestParseUnixTimestampTruncatesToMicrosecond(t *testing.T) { + parsed := parseUnixTimestamp("1714416536.118177413") + if parsed == nil { + t.Fatalf("expected timestamp to parse") + } + if parsed.UTC().Format(time.RFC3339Nano) != "2024-04-29T18:48:56.118177Z" { + t.Fatalf("expected parsed timestamp to be truncated to microsecond precision, got %s", parsed.UTC().Format(time.RFC3339Nano)) + } +} + +func TestCleanupWorkspaceStateBeforeInitialScanClearsOnlyTargetWorkspace(t *testing.T) { + db := openStorageIndexTestDB(t) + target := resolvedWorkspace{ + WorkspaceType: model.StorageIndexWorkspaceTypeUser, + WorkspaceName: "alice", + LogicalPath: "/user/alice", + } + other := resolvedWorkspace{ + WorkspaceType: model.StorageIndexWorkspaceTypeUser, + WorkspaceName: "bob", + LogicalPath: "/user/bob", + } + + mustCreateStorageIndexState(t, db, target, "scan-target") + mustCreateStorageIndexState(t, db, other, "scan-other") + + clearedRows, err := cleanupWorkspaceStateBeforeInitialScan(db, target) + if err != nil { + t.Fatalf("cleanup workspace state failed: %v", err) + } + if clearedRows != 5 { + t.Fatalf("expected 5 cleared rows for target workspace, got %d", clearedRows) + } + + assertWorkspaceStateRowCount(t, db, target, 0) + assertWorkspaceStateRowCount(t, db, other, 5) +} + +func TestCleanupWorkspaceStateBeforeInitialScanClearsPublicBaselines(t *testing.T) { + db := openStorageIndexTestDB(t) + publicWorkspace := resolvedWorkspace{ + WorkspaceType: model.StorageIndexWorkspaceTypePublic, + WorkspaceName: "public", + LogicalPath: "/public", + } + other := resolvedWorkspace{ + WorkspaceType: model.StorageIndexWorkspaceTypeUser, + WorkspaceName: "alice", + LogicalPath: "/user/alice", + } + + mustCreateStorageIndexState(t, db, publicWorkspace, "scan-public") + mustCreatePublicBaselineState(t, db, "scan-public") + mustCreateStorageIndexState(t, db, other, "scan-other") + + clearedRows, err := cleanupWorkspaceStateBeforeInitialScan(db, publicWorkspace) + if err != nil { + t.Fatalf("cleanup public workspace state failed: %v", err) + } + if clearedRows != 7 { + t.Fatalf("expected 7 cleared rows for public workspace, got %d", clearedRows) + } + + assertWorkspaceStateRowCount(t, db, publicWorkspace, 0) + assertWorkspaceStateRowCount(t, db, other, 5) + assertTableRowCount(t, db, &model.StorageIndexPublicRootBaseline{}, 0) + assertTableRowCount(t, db, &model.StorageIndexPublicFileBaseline{}, 0) +} + +func openStorageIndexTestDB(t *testing.T) *gorm.DB { + t.Helper() + + dbPath := filepath.Join(t.TempDir(), "storageindex-test.sqlite") + db, err := gorm.Open(sqlite.Open(dbPath), &gorm.Config{}) + if err != nil { + t.Fatalf("open sqlite db failed: %v", err) + } + sqlDB, err := db.DB() + if err != nil { + t.Fatalf("get sqlite sql db failed: %v", err) + } + t.Cleanup(func() { + _ = sqlDB.Close() + }) + if err := db.AutoMigrate( + &model.StorageIndexEntry{}, + &model.StorageIndexDirectoryMetric{}, + &model.StorageIndexRedundancyHit{}, + &model.StorageIndexCandidate{}, + &model.StorageIndexCandidateFile{}, + &model.StorageIndexPublicRootBaseline{}, + &model.StorageIndexPublicFileBaseline{}, + ); err != nil { + t.Fatalf("auto migrate sqlite db failed: %v", err) + } + return db +} + +func mustCreateStorageIndexState(t *testing.T, db *gorm.DB, workspace resolvedWorkspace, scanID string) { + t.Helper() + + entry := model.StorageIndexEntry{ + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + ScanID: scanID, + LogicalPath: workspace.LogicalPath, + RelativePath: ".", + Name: filepath.Base(workspace.LogicalPath), + EntryType: model.StorageIndexEntryTypeDir, + } + metric := model.StorageIndexDirectoryMetric{ + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + ScanID: scanID, + Path: workspace.LogicalPath, + Name: filepath.Base(workspace.LogicalPath), + } + hit := model.StorageIndexRedundancyHit{ + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + ScanID: scanID, + TargetType: model.StorageIndexRedundancyTargetTypeDirectory, + TargetPath: workspace.LogicalPath + "/dup", + PublicPath: "/public/dup", + MatchKey: "dup|1", + VerificationStatus: model.StorageIndexVerificationStatusSuspected, + } + candidate := model.StorageIndexCandidate{ + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + ScanID: scanID, + CandidateType: "model_dir", + TargetPath: workspace.LogicalPath + "/candidate", + PublicPath: "/public/candidate", + CandidateScore: 80, + Status: model.StorageIndexCandidateStatusSuspected, + } + candidateFile := model.StorageIndexCandidateFile{ + WorkspaceType: workspace.WorkspaceType, + WorkspaceName: workspace.WorkspaceName, + ScanID: scanID, + CandidatePath: candidate.TargetPath, + FilePath: candidate.TargetPath + "/weights.bin", + FileName: "weights.bin", + RelativePath: "weights.bin", + VerificationStatus: model.StorageIndexVerificationStatusSuspected, + } + + for _, item := range []any{&entry, &metric, &hit, &candidate, &candidateFile} { + if err := db.Create(item).Error; err != nil { + t.Fatalf("seed storage index state failed: %v", err) + } + } +} + +func mustCreatePublicBaselineState(t *testing.T, db *gorm.DB, scanID string) { + t.Helper() + + root := model.StorageIndexPublicRootBaseline{ + ScanID: scanID, + ResourceName: "qwen", + LogicalPath: "/public/models/qwen", + Category: "model_dir", + } + file := model.StorageIndexPublicFileBaseline{ + ScanID: scanID, + PublicRootPath: root.LogicalPath, + PublicRootHash: hashString(root.LogicalPath), + FilePath: root.LogicalPath + "/weights.bin", + FileName: "weights.bin", + RelativePath: "weights.bin", + MatchKey: "weights.bin|1", + MatchKeyHash: hashString("weights.bin|1"), + } + + if err := db.Create(&root).Error; err != nil { + t.Fatalf("seed public root baseline failed: %v", err) + } + if err := db.Create(&file).Error; err != nil { + t.Fatalf("seed public file baseline failed: %v", err) + } +} + +func assertWorkspaceStateRowCount(t *testing.T, db *gorm.DB, workspace resolvedWorkspace, want int64) { + t.Helper() + + models := []any{ + &model.StorageIndexEntry{}, + &model.StorageIndexDirectoryMetric{}, + &model.StorageIndexRedundancyHit{}, + &model.StorageIndexCandidate{}, + &model.StorageIndexCandidateFile{}, + } + total := int64(0) + for _, table := range models { + var count int64 + if err := db.Model(table). + Where("workspace_type = ? AND workspace_name = ?", workspace.WorkspaceType, workspace.WorkspaceName). + Count(&count).Error; err != nil { + t.Fatalf("count workspace state rows failed: %v", err) + } + total += count + } + if total != want { + t.Fatalf("expected workspace %s/%s to have %d state rows, got %d", workspace.WorkspaceType, workspace.WorkspaceName, want, total) + } +} + +func assertTableRowCount(t *testing.T, db *gorm.DB, table any, want int64) { + t.Helper() + + var count int64 + if err := db.Model(table).Count(&count).Error; err != nil { + t.Fatalf("count table rows failed: %v", err) + } + if count != want { + t.Fatalf("expected %T to have %d rows, got %d", table, want, count) + } +} diff --git a/frontend/src/components/file/file-select-dialog.tsx b/frontend/src/components/file/file-select-dialog.tsx index 100d4cc2e..19009fae0 100644 --- a/frontend/src/components/file/file-select-dialog.tsx +++ b/frontend/src/components/file/file-select-dialog.tsx @@ -41,6 +41,7 @@ export const FileSelectDialog = ({ disabled, allowSelectFile = true, isrw = false, + isadmin = false, title, }: { value?: string @@ -48,6 +49,7 @@ export const FileSelectDialog = ({ disabled?: boolean allowSelectFile?: boolean isrw?: boolean + isadmin?: boolean title?: string }) => { const { t } = useTranslation() @@ -99,6 +101,7 @@ export const FileSelectDialog = ({ { setContent(item) }} diff --git a/frontend/src/components/file/folder-navigation.tsx b/frontend/src/components/file/folder-navigation.tsx index 9ca7c92e8..4b94a18b4 100644 --- a/frontend/src/components/file/folder-navigation.tsx +++ b/frontend/src/components/file/folder-navigation.tsx @@ -19,14 +19,20 @@ import { useLocation, useNavigate } from '@tanstack/react-router' import { useAtomValue } from 'jotai' import { ArrowRight, Folder, HardDrive, UserRound, UsersRound } from 'lucide-react' import { motion } from 'motion/react' -import { useMemo } from 'react' +import { useEffect, useMemo, useState } from 'react' import { useTranslation } from 'react-i18next' import { getFolderTitle } from '@/components/file/lazy-file-tree' import PageTitle from '@/components/layout/page-title' import { AccessMode, IUserContext } from '@/services/api/auth' -import { FileItem } from '@/services/api/file' +import { + DirectorySize, + FileItem, + MyQuota, + apiGetDirectorySize, + apiGetMyQuota, +} from '@/services/api/file' import { atomUserContext } from '@/utils/store' @@ -72,6 +78,68 @@ export default function FolderNavigation({ const { pathname } = useLocation() const navigate = useNavigate() const context = useAtomValue(atomUserContext) + const [userSpaceSize, setUserSpaceSize] = useState(null) + const [publicSpaceSize, setPublicSpaceSize] = useState(null) + const [accountSpaceSize, setAccountSpaceSize] = useState(null) + const [myQuota, setMyQuota] = useState(null) + const [isLoading, setIsLoading] = useState(false) + + // 获取空间大小和用户配额 + useEffect(() => { + const fetchSpaceSizes = async () => { + setIsLoading(true) + try { + const promises: Promise[] = [] + + if (context?.space) { + promises.push( + apiGetDirectorySize(`user/${context.space}`) + .then((r) => { + if (r.data) setUserSpaceSize(r.data) + }) + .catch(() => { + setUserSpaceSize(null) + }) + ) + } + + promises.push( + apiGetDirectorySize('public') + .then((r) => { + if (r.data) setPublicSpaceSize(r.data) + }) + .catch(() => { + setPublicSpaceSize(null) + }), + apiGetDirectorySize('account') + .then((r) => { + if (r.data) setAccountSpaceSize(r.data) + }) + .catch(() => { + setAccountSpaceSize(null) + }), + apiGetMyQuota() + .then((r) => { + if (r.data) setMyQuota(r.data) + }) + .catch(() => { + setMyQuota(null) + }) + ) + + await Promise.all(promises) + } catch { + setUserSpaceSize(null) + setPublicSpaceSize(null) + setAccountSpaceSize(null) + setMyQuota(null) + } finally { + setIsLoading(false) + } + } + + fetchSpaceSizes() + }, [context?.space]) // 对文件夹进行排序，公共 -> 账户 -> 用户 const sortFolders = (folders: FileItem[]) => { @@ -130,6 +198,55 @@ export default function FolderNavigation({ return t('folderNavigation.badge.noAccess', '无权限') } + // 格式化文件大小，自动选择合适的单位 + const formatFileSize = (bytes: number): { size: string; unit: string } => { + if (!Number.isFinite(bytes) || bytes <= 0) return { size: '0', unit: 'B' } + const units = ['B', 'KB', 'MB', 'GB', 'TB'] + const k = 1024 + const i = Math.max(0, Math.min(units.length - 1, Math.floor(Math.log(bytes) / Math.log(k)))) + return { + size: (bytes / Math.pow(k, i)).toFixed(2), + unit: units[i], + } + } + + // 根据 space 类型返回当前大小和配额（bytes）；quota=-1 表示无限制，null 表示共享空间无独立配额 + const getSpaceMetrics = ( + spaceType: string + ): { + size: number | null + quota: number | null + sizeUnit?: string + quotaUnit?: string + formattedSize?: string + } => { + if (spaceType === 'user') { + return { + size: userSpaceSize?.size ?? null, + quota: myQuota?.space_quota ?? null, + sizeUnit: userSpaceSize?.unit, + formattedSize: userSpaceSize?.formatted, + } + } + if (spaceType === 'public') { + return { + size: publicSpaceSize?.size ?? null, + quota: null, + sizeUnit: publicSpaceSize?.unit, + formattedSize: publicSpaceSize?.formatted, + } + } + if (spaceType === 'account') { + return { + size: accountSpaceSize?.size ?? null, + quota: null, + sizeUnit: accountSpaceSize?.unit, + formattedSize: accountSpaceSize?.formatted, + } + } + return { size: null, quota: null } + } + const handleTitleNavigation = (name: string) => { if (isPublicFolder(name)) { if (isadmin) { @@ -219,29 +336,95 @@ export default function FolderNavigation({

{/* Usage Metrics */} -
-
-
- ??? - GB -
- 总 ??? GB -
+ {(() => { + const { + size, + quota, + sizeUnit, + formattedSize: apiFormattedSize, + } = getSpaceMetrics(type) + const displaySize = + typeof size === 'number' && Number.isFinite(size) ? Math.max(0, size) : null + const hasQuota = quota !== null && quota > 0 + const isUnlimited = quota === -1 + const usageRatio = + hasQuota && displaySize !== null + ? Math.min(100, (displaySize / quota!) * 100) + : null -
-
-
+ // 使用API返回的格式化大小，如果没有则使用formatFileSize计算 + let formattedSize: { size: string; unit: string } | null = null + if (apiFormattedSize) { + // 从API返回的格式化字符串中提取大小和单位 + const match = apiFormattedSize.match(/([\d.]+)\s*(\w+)/) + if (match) { + formattedSize = { size: match[1], unit: match[2] } + } + } else if (displaySize !== null) { + if (sizeUnit) { + formattedSize = { size: displaySize.toFixed(2), unit: sizeUnit } + } else { + formattedSize = formatFileSize(displaySize) + } + } -
- ??% 已使用 - {r.size} 个文件 -
-
+ let formattedQuota: { size: string; unit: string } | null = null + if (hasQuota && quota !== null) { + formattedQuota = formatFileSize(quota) + } + + return ( +
+
+
+ {isLoading ? ( + 加载中... + ) : formattedSize ? ( + <> + + {formattedSize.size} + + + {formattedSize.unit} + + + ) : ( + — + )} +
+ + {isUnlimited + ? '无限制' + : formattedQuota + ? `总 ${formattedQuota.size} ${formattedQuota.unit}` + : '共享空间'} + +
+ +
+
+
+ +
+ + {usageRatio !== null + ? `${usageRatio.toFixed(1)}% 已使用` + : isUnlimited + ? formattedSize + ? formattedSize.size + ' ' + formattedSize.unit + ' 已使用' + : '—' + : '共享，无独立配额'} + + {r.size} 个文件 +
+
+ ) + })()} {/* Action Button */} + + + + + {t('systemConfig.storageDecision.resetConfirmTitle')} + + + {t('systemConfig.storageDecision.resetConfirmDesc')} + + + + {t('common.cancel')} + {t('common.confirm')} + + + + + +
+ + + + + ) +} diff --git a/frontend/src/routes/admin/more/index.tsx b/frontend/src/routes/admin/more/index.tsx index ad85054d6..0efabc045 100644 --- a/frontend/src/routes/admin/more/index.tsx +++ b/frontend/src/routes/admin/more/index.tsx @@ -18,15 +18,18 @@ import { apiAdminGetModelDownloadLimitConfig, apiAdminGetPodBandwidthConfig, apiAdminGetPrequeueConfig, + apiAdminGetStorageDecisionConfig, apiAdminGrantAllUsersExtraBalance, apiAdminResetAllBillingBalances, apiAdminResetLLMConfig, + apiAdminResetStorageDecisionConfig, apiAdminSetBillingStatus, apiAdminSetGpuAnalysisStatus, apiAdminUpdateLLMConfig, apiAdminUpdateModelDownloadLimitConfig, apiAdminUpdatePodBandwidthConfig, apiAdminUpdatePrequeueConfig, + apiAdminUpdateStorageDecisionConfig, } from '@/services/api/system-config' import { markApiErrorHandled } from '@/services/client' import { ERROR_RESOURCE_STATUS_ERROR } from '@/services/error_code' @@ -41,6 +44,13 @@ import { LlmFormSchema, LlmSettings, createLlmSettingsSchema } from './-componen import { ModelDownloadLimitSettings } from './-components/model-download-limit-settings' import { PodBandwidthSettings } from './-components/pod-bandwidth-settings' import { PrequeueSettings } from './-components/prequeue-settings' +import { + StorageDecisionFormSchema, + StorageDecisionSettings, + createStorageDecisionSettingsSchema, +} from './-components/storage-decision-settings' + +const DEFAULT_STORAGE_BASE_URL = 'http://192.168.5.68:30186/v1' export const Route = createFileRoute('/admin/more/')({ component: RouteComponent, @@ -60,6 +70,17 @@ function RouteComponent() { }, }) + const storageDecisionForm = useForm({ + resolver: zodResolver(createStorageDecisionSettingsSchema(t)), + defaultValues: { + decisionMode: 'agent', + configSource: 'platform', + baseUrl: DEFAULT_STORAGE_BASE_URL, + modelName: '', + apiKey: '', + }, + }) + const { data: llmConfigData } = useQuery({ queryKey: ['admin', 'system-config', 'llm'], queryFn: async () => { @@ -68,6 +89,14 @@ function RouteComponent() { }, }) + const { data: storageDecisionConfigData } = useQuery({ + queryKey: ['admin', 'system-config', 'storage-decision'], + queryFn: async () => { + const res = await apiAdminGetStorageDecisionConfig() + return res.data + }, + }) + const { data: gpuStatusData } = useQuery({ queryKey: ['admin', 'system-config', 'gpu-status'], queryFn: () => apiAdminGetGpuAnalysisStatus().then((res) => res.data), @@ -98,6 +127,11 @@ function RouteComponent() { queryFn: () => apiAdminGetPodBandwidthConfig().then((res) => res.data), }) + const { data: billingStatusData } = useQuery({ + queryKey: ['admin', 'system-config', 'billing-status'], + queryFn: () => apiAdminGetBillingStatus().then((res) => res.data), + }) + const [backfillEnabled, setBackfillEnabled] = useState(false) const [queueQuotaEnabled, setQueueQuotaEnabled] = useState(false) const [prequeueWaitingToleranceSeconds, setPrequeueWaitingToleranceSeconds] = useState('') @@ -105,45 +139,58 @@ function RouteComponent() { const [maxTotalActivationsPerRound, setMaxTotalActivationsPerRound] = useState('') const [prequeueCandidateSize, setPrequeueCandidateSize] = useState('') - const { data: billingStatusData } = useQuery({ - queryKey: ['admin', 'system-config', 'billing-status'], - queryFn: () => apiAdminGetBillingStatus().then((res) => res.data), - }) - useEffect(() => { - if (llmConfigData) { - llmForm.reset({ - baseUrl: llmConfigData.baseUrl, - modelName: llmConfigData.modelName, - apiKey: llmConfigData.apiKey || '', - }) + if (!llmConfigData) { + return } + + llmForm.reset({ + baseUrl: llmConfigData.baseUrl, + modelName: llmConfigData.modelName, + apiKey: llmConfigData.apiKey || '', + }) }, [llmConfigData, llmForm]) useEffect(() => { - if (prequeueConfigData) { - setBackfillEnabled(prequeueConfigData.backfillEnabled) - setQueueQuotaEnabled(prequeueConfigData.queueQuotaEnabled) - setPrequeueWaitingToleranceSeconds( - String(prequeueConfigData.normalJobWaitingToleranceSeconds ?? '') - ) - setActivateTickerIntervalSeconds( - String(prequeueConfigData.activateTickerIntervalSeconds ?? '') - ) - setMaxTotalActivationsPerRound(String(prequeueConfigData.maxTotalActivationsPerRound ?? '')) - setPrequeueCandidateSize(String(prequeueConfigData.prequeueCandidateSize ?? '')) + if (!storageDecisionConfigData) { + return } + + storageDecisionForm.reset({ + decisionMode: storageDecisionConfigData.decisionMode || 'agent', + configSource: storageDecisionConfigData.configSource || 'platform', + baseUrl: storageDecisionConfigData.baseUrl || DEFAULT_STORAGE_BASE_URL, + modelName: storageDecisionConfigData.modelName || '', + apiKey: storageDecisionConfigData.apiKey || '', + }) + }, [storageDecisionConfigData, storageDecisionForm]) + + useEffect(() => { + if (!prequeueConfigData) { + return + } + + setBackfillEnabled(prequeueConfigData.backfillEnabled) + setQueueQuotaEnabled(prequeueConfigData.queueQuotaEnabled) + setPrequeueWaitingToleranceSeconds( + String(prequeueConfigData.normalJobWaitingToleranceSeconds ?? '') + ) + setActivateTickerIntervalSeconds(String(prequeueConfigData.activateTickerIntervalSeconds ?? '')) + setMaxTotalActivationsPerRound(String(prequeueConfigData.maxTotalActivationsPerRound ?? '')) + setPrequeueCandidateSize(String(prequeueConfigData.prequeueCandidateSize ?? '')) }, [prequeueConfigData]) - const handleError = (error: unknown) => { - if (typeof error === 'object' && error !== null && 'data' in error) { - const errorData = (error as { data: IErrorResponse }).data - const errorCode = errorData?.code + const handleBusinessLogicError = (error: unknown, businessLogicMessage: string) => { + if (typeof error !== 'object' || error === null || !('data' in error)) { + return + } - if (errorCode === ERROR_RESOURCE_STATUS_ERROR) { - markApiErrorHandled(error) - toast.error(t('systemConfig.gpuAnalysis.error.llmCheckFailed')) - } + const errorData = (error as { data: IErrorResponse }).data + const errorCode = errorData?.code + + if (errorCode === ERROR_RESOURCE_STATUS_ERROR) { + markApiErrorHandled(error) + toast.error(businessLogicMessage) } } @@ -156,37 +203,74 @@ function RouteComponent() { }), onSuccess: (_, vars) => { queryClient.invalidateQueries({ queryKey: ['admin', 'system-config', 'llm'] }) - if (vars.validate) { - toast.success(t('systemConfig.llm.testAndSaveSuccess')) - } else { - toast.success(t('systemConfig.llm.saveSuccess')) - } + toast.success( + vars.validate ? t('systemConfig.llm.testAndSaveSuccess') : t('systemConfig.llm.saveSuccess') + ) }, - onError: handleError, + onError: (error) => + handleBusinessLogicError(error, t('systemConfig.gpuAnalysis.error.llmCheckFailed')), }) const resetLLMMutation = useMutation({ mutationFn: apiAdminResetLLMConfig, onSuccess: () => { queryClient.invalidateQueries({ queryKey: ['admin', 'system-config', 'llm'] }) - queryClient.invalidateQueries({ queryKey: ['admin', 'system-config', 'gpu-status'] }) toast.success(t('common.resetSuccess')) }, - onError: handleError, + onError: (error) => + handleBusinessLogicError(error, t('systemConfig.gpuAnalysis.error.llmCheckFailed')), + }) + + const updateStorageDecisionMutation = useMutation({ + mutationFn: (vars: { data: StorageDecisionFormSchema; validate: boolean }) => + apiAdminUpdateStorageDecisionConfig({ + ...vars.data, + baseUrl: vars.data.baseUrl ?? '', + modelName: vars.data.modelName ?? '', + apiKey: vars.data.apiKey ?? '', + validate: vars.validate, + }), + onSuccess: (_, vars) => { + queryClient.invalidateQueries({ queryKey: ['admin', 'system-config', 'storage-decision'] }) + toast.success( + vars.validate + ? t('systemConfig.storageDecision.testAndSaveSuccess') + : t('systemConfig.storageDecision.saveSuccess') + ) + }, + onError: (error) => + handleBusinessLogicError( + error, + t('systemConfig.storageDecision.error.connectionCheckFailed') + ), + }) + + const resetStorageDecisionMutation = useMutation({ + mutationFn: apiAdminResetStorageDecisionConfig, + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ['admin', 'system-config', 'storage-decision'] }) + toast.success(t('common.resetSuccess')) + }, + onError: (error) => + handleBusinessLogicError( + error, + t('systemConfig.storageDecision.error.connectionCheckFailed') + ), }) const toggleGpuMutation = useMutation({ mutationFn: apiAdminSetGpuAnalysisStatus, onSuccess: (_data, newStatus) => { queryClient.invalidateQueries({ queryKey: ['admin', 'system-config', 'gpu-status'] }) - - const message = newStatus - ? t('systemConfig.gpuAnalysis.enabledSuccess') - : t('systemConfig.gpuAnalysis.disabledSuccess') - toast.success(message) + toast.success( + newStatus + ? t('systemConfig.gpuAnalysis.enabledSuccess') + : t('systemConfig.gpuAnalysis.disabledSuccess') + ) }, - onError: handleError, + onError: (error) => + handleBusinessLogicError(error, t('systemConfig.gpuAnalysis.error.llmCheckFailed')), }) const updateBillingMutation = useMutation({ @@ -232,38 +316,6 @@ function RouteComponent() { onError: showErrorToast, }) - const handleLlmSubmit = (values: LlmFormSchema, validate: boolean) => { - updateLLMMutation.mutate({ data: values, validate }) - } - - const handleLlmReset = () => { - resetLLMMutation.mutate() - } - - const handleGpuToggle = async (checked: boolean) => { - if (checked) { - const isValid = await llmForm.trigger() - if (!isValid) { - toast.error(t('systemConfig.llm.validation.formInvalid')) - return - } - - toast.info(t('systemConfig.gpuAnalysis.verifyingLLM')) - const currentLlmValues = llmForm.getValues() - - updateLLMMutation.mutate( - { data: currentLlmValues, validate: true }, - { - onSuccess: () => { - toggleGpuMutation.mutate(true) - }, - } - ) - } else { - toggleGpuMutation.mutate(false) - } - } - const buildPrequeuePayload = () => ({ backfillEnabled, queueQuotaEnabled, @@ -302,7 +354,7 @@ function RouteComponent() { invalidatePrequeueConfig() toast.success(t('systemConfig.prequeue.saveSuccess')) }, - onError: handleError, + onError: showErrorToast, }) const updateModelDownloadLimitMutation = useMutation({ @@ -326,6 +378,39 @@ function RouteComponent() { onError: showErrorToast, }) + const handleLlmSubmit = (values: LlmFormSchema, validate: boolean) => { + updateLLMMutation.mutate({ data: values, validate }) + } + + const handleStorageDecisionSubmit = (values: StorageDecisionFormSchema, validate: boolean) => { + updateStorageDecisionMutation.mutate({ data: values, validate }) + } + + const handleGpuToggle = async (checked: boolean) => { + if (!checked) { + toggleGpuMutation.mutate(false) + return + } + + const isValid = await llmForm.trigger() + if (!isValid) { + toast.error(t('systemConfig.llm.validation.formInvalid')) + return + } + + toast.info(t('systemConfig.gpuAnalysis.verifyingLLM')) + const currentLlmValues = llmForm.getValues() + + updateLLMMutation.mutate( + { data: currentLlmValues, validate: true }, + { + onSuccess: () => { + toggleGpuMutation.mutate(true) + }, + } + ) + } + const handlePrequeueSubmit = () => { if (!validatePrequeuePositiveIntegers()) { return @@ -333,8 +418,6 @@ function RouteComponent() { updatePrequeueMutation.mutate() } - const isPrequeueConfigPending = updatePrequeueMutation.isPending - return (
resetLLMMutation.mutate()} /> grantAllUsersExtraMutation.mutate(payload)} /> + + + resetStorageDecisionMutation.mutate()} + /> +
) } diff --git a/frontend/src/routes/admin/route.tsx b/frontend/src/routes/admin/route.tsx index 58ca5047d..1cbd316b0 100644 --- a/frontend/src/routes/admin/route.tsx +++ b/frontend/src/routes/admin/route.tsx @@ -14,6 +14,7 @@ import { ScrollText, ServerIcon, SettingsIcon, + StoreIcon, UserRoundIcon, UsersRoundIcon, } from 'lucide-react' @@ -164,6 +165,11 @@ const useAdminSidebarGroups = (): NavGroupProps[] => { icon: FolderIcon, url: '/admin/files', }, + { + title: t('navigation.storageManagement'), + icon: StoreIcon, + url: '/admin/storage', + }, ], }, { diff --git a/frontend/src/routes/admin/storage/-components/storage-directory-compare-panel.tsx b/frontend/src/routes/admin/storage/-components/storage-directory-compare-panel.tsx new file mode 100644 index 000000000..eb1779273 --- /dev/null +++ b/frontend/src/routes/admin/storage/-components/storage-directory-compare-panel.tsx @@ -0,0 +1,374 @@ +import { useMutation, useQuery } from '@tanstack/react-query' +import { ChevronRight, RefreshCcw, Search } from 'lucide-react' +import { useMemo, useState } from 'react' +import { toast } from 'sonner' + +import { Badge } from '@/components/ui/badge' +import { Button } from '@/components/ui/button' +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card' +import { Collapsible, CollapsibleContent, CollapsibleTrigger } from '@/components/ui/collapsible' +import { Label } from '@/components/ui/label' +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/ui/select' +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from '@/components/ui/table' + +import { FileSelectDialog } from '@/components/file/file-select-dialog' + +import { + MetadataFolderCompareFile, + apiAdminCompareMetadataFolders, + apiAdminGetMetadataFolderCompareJob, +} from '@/services/api/storage' + +function getErrorMessage(error: unknown, fallback: string): string { + if (typeof error === 'object' && error !== null) { + const candidate = error as { + data?: { msg?: string } + response?: { data?: { msg?: string } } + } + return candidate.data?.msg ?? candidate.response?.data?.msg ?? fallback + } + return fallback +} + +function statusTone(status?: string): string { + switch (status) { + case 'done': + return 'border-green-300 bg-green-100 text-green-700' + case 'error': + return 'border-red-300 bg-red-100 text-red-700' + case 'running': + return 'border-sky-300 bg-sky-100 text-sky-700' + default: + return 'border-amber-300 bg-amber-100 text-amber-700' + } +} + +function statusLabel(status?: string): string { + switch (status) { + case '': + case undefined: + return '未开始' + case 'pending': + return '待处理' + case 'running': + return '运行中' + case 'done': + return '已完成' + case 'error': + return '失败' + default: + return status || '未开始' + } +} + +export default function StorageDirectoryComparePanel() { + const [isOpen, setIsOpen] = useState(false) + const [compareType, setCompareType] = useState<'auto' | 'model' | 'dataset'>('auto') + const [compareMode, setCompareMode] = useState<'optimized' | 'full_hash'>('optimized') + const [compareLeftPath, setCompareLeftPath] = useState('') + const [compareRightPath, setCompareRightPath] = useState('') + const [lastCompareJobId, setLastCompareJobId] = useState('') + + const compareMutation = useMutation({ + mutationFn: () => + apiAdminCompareMetadataFolders({ + left_path: compareLeftPath, + right_path: compareRightPath, + compare_type: compareType, + compare_mode: compareMode, + }), + onSuccess: (res) => { + setLastCompareJobId(res.data.job_id) + setIsOpen(true) + toast.success(`已启动目录比对任务 ${res.data.job_id}`) + }, + onError: (error: unknown) => { + toast.error(getErrorMessage(error, '目录比对失败')) + }, + }) + + const compareJobQuery = useQuery({ + queryKey: ['admin', 'storage-index', 'compare-job', lastCompareJobId], + queryFn: () => apiAdminGetMetadataFolderCompareJob(lastCompareJobId).then((res) => res.data), + enabled: !!lastCompareJobId, + refetchInterval: (query) => { + const status = query.state.data?.status + return status === 'done' || status === 'error' ? false : 3000 + }, + }) + + const compareResult = compareJobQuery.data?.result + const mismatchedFiles = useMemo( + () => compareResult?.files.filter((item) => !item.same) ?? [], + [compareResult] + ) + + return ( + + + +
+ + + 目录比对测试 + {lastCompareJobId && ( + + {statusLabel(compareJobQuery.data?.status)} + + )} + + + 独立选择两个目录，测试当前目录比对优化方案与完整 SHA256 基线的结果和耗时。 + 默认收起，用到时再展开即可。 + +
+
+ + + + +
+ + + + +
+
+ + +
+
+ + +
+
+ +
+
+ + setCompareLeftPath(`/${item.id.replace(/^\//, '')}`)} + /> +
+
+ + setCompareRightPath(`/${item.id.replace(/^\//, '')}`)} + /> +
+
+ +
+ +
+ + {lastCompareJobId && ( +
+
+ 最近比对任务： + {lastCompareJobId} + + {statusLabel(compareJobQuery.data?.status)} + +
+ {compareJobQuery.data?.error && ( +

{compareJobQuery.data.error}

+ )} +
+ )} + + {compareResult && ( +
+
+ + {compareResult.same ? '目录相同' : '目录不同'} + + + {compareResult.compare_mode === 'full_hash' ? '完整 SHA256 基线' : '优化方案'} + + + 总耗时 {compareResult.timing.total_ms} ms + +
+ +
+ + + + + + + + + + +
+ +
+
+
仅左侧存在
+
+ {compareResult.missing_left.length === 0 + ? '无' + : compareResult.missing_left.slice(0, 10).join('\n')} +
+
+
+
仅右侧存在
+
+ {compareResult.missing_right.length === 0 + ? '无' + : compareResult.missing_right.slice(0, 10).join('\n')} +
+
+
+ +
+
差异文件
+
+ + + + 文件 + 左侧路径 + 右侧路径 + 模式 + 原因 + + + + {mismatchedFiles.length === 0 ? ( + + + 无差异文件 + + + ) : ( + mismatchedFiles.slice(0, 20).map((item, index) => ( + + {item.file_name} + + {item.left_relative_path} + + + {item.right_relative_path} + + {item.verification_mode} + {item.reason || '-'} + + )) + )} + +
+
+
+
+ )} + + + + + ) +} + +function MetricCard({ label, value }: { label: string; value: string }) { + return ( +
+
{label}
+
{value}
+
+ ) +} diff --git a/frontend/src/routes/admin/storage/-components/storage-governance-panel.tsx b/frontend/src/routes/admin/storage/-components/storage-governance-panel.tsx new file mode 100644 index 000000000..dfdf8b4d7 --- /dev/null +++ b/frontend/src/routes/admin/storage/-components/storage-governance-panel.tsx @@ -0,0 +1,701 @@ +import { useMutation, useQuery } from '@tanstack/react-query' +import { Eye, History, RefreshCcw } from 'lucide-react' +import { useMemo, useState } from 'react' +import { toast } from 'sonner' + +import { Badge } from '@/components/ui/badge' +import { Button } from '@/components/ui/button' +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card' +import { + Dialog, + DialogContent, + DialogFooter, + DialogHeader, + DialogTitle, +} from '@/components/ui/dialog' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/ui/select' +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from '@/components/ui/table' +import { Textarea } from '@/components/ui/textarea' + +import { + ReplaySummary, + StorageDecisionRecordDetail, + apiAdminGetStorageDecision, + apiAdminGetStorageDecisions, + apiAdminReplayStorageDecisions, + apiAdminRunAutoShrink, +} from '@/services/api/storage' + +function formatBytes(bytes: number): string { + if (!Number.isFinite(bytes) || bytes <= 0) return '0 B' + const units = ['B', 'KB', 'MB', 'GB', 'TB'] + const exponent = Math.min(Math.floor(Math.log(bytes) / Math.log(1024)), units.length - 1) + return `${(bytes / 1024 ** exponent).toFixed(1)} ${units[exponent]}` +} + +function formatRatio(value?: number): string { + if (value === undefined || Number.isNaN(value)) return 'N/A' + return `${(value * 100).toFixed(1)}%` +} + +function safeStringify(value: unknown): string { + return JSON.stringify(value, null, 2) ?? '' +} + +function getErrorMessage(error: unknown, fallback: string): string { + if (typeof error === 'object' && error !== null) { + const candidate = error as { + data?: { msg?: string } + response?: { data?: { msg?: string } } + } + return candidate.data?.msg ?? candidate.response?.data?.msg ?? fallback + } + return fallback +} + +function decisionStatusTone(status: string): string { + switch (status) { + case 'done': + return 'bg-green-100 text-green-700 border-green-300' + case 'error': + return 'bg-red-100 text-red-700 border-red-300' + case 'running': + return 'bg-blue-100 text-blue-700 border-blue-300' + default: + return 'bg-yellow-100 text-yellow-700 border-yellow-300' + } +} + +function decisionStatusLabel(status: string): string { + switch (status) { + case 'pending': + return '待处理' + case 'running': + return '执行中' + case 'done': + return '已完成' + case 'error': + return '失败' + default: + return status + } +} + +function decisionSourceTone(source: string): string { + switch (source) { + case 'patrol': + return 'bg-sky-100 text-sky-700 border-sky-300' + case 'manual': + return 'bg-violet-100 text-violet-700 border-violet-300' + default: + return 'bg-slate-100 text-slate-700 border-slate-300' + } +} + +function decisionSourceLabel(source: string): string { + switch (source) { + case 'manual': + return '手动触发' + case 'patrol': + return '定时巡检' + case 'replay': + return '回放任务' + default: + return source + } +} + +function appliedActionLabel(action: string): string { + switch (action) { + case 'expand': + return '执行扩容' + case 'expand_failed': + return '扩容失败' + case 'freeze': + return '冻结新作业' + case 'freeze_failed': + return '冻结失败' + case 'observe': + return '仅观察' + case 'expand_and_freeze': + return '扩容并冻结' + case 'manual_expand': + return '手动执行扩容' + case 'manual_expand_and_freeze': + return '手动扩容并冻结' + case 'manual_freeze': + return '手动执行冻结' + case 'manual_freeze_failed': + return '手动冻结失败' + default: + return action || '待处理' + } +} + +function shrinkStageLabel(stage?: string): string { + switch (stage) { + case 'expanded': + return '扩容阶段' + case 'buffer_reduction': + return '缩容缓冲期' + default: + return stage || '无' + } +} + +function shrinkStageReason(stage?: string): string { + switch (stage) { + case 'expanded': + return '用户当前处于临时扩容后的观察阶段，系统暂不回收扩容额度。' + case 'buffer_reduction': + return '用户占用已回落到安全区间，系统先回收到中间缓冲配额，等待观察窗口结束后再完全恢复到原始配额。' + default: + return '当前没有处于自动缩容阶段。' + } +} + +export default function StorageGovernancePanel() { + const [filters, setFilters] = useState({ + user: '', + status: '', + source: '', + }) + const [isDetailOpen, setIsDetailOpen] = useState(false) + const [selectedDecisionJobId, setSelectedDecisionJobId] = useState(null) + const [isReplayOpen, setIsReplayOpen] = useState(false) + const [replaySummary, setReplaySummary] = useState(null) + const [replayConfig, setReplayConfig] = useState({ + limit: 50, + maxExpandRatio: 0.3, + maxExpandBytesGiB: 500, + minPlatformReservedRatio: 0.1, + minPlatformReservedBytesGiB: 200, + expansionCooldownHours: 6, + forceFreezeWhenOverQuota: true, + }) + + const decisionsQuery = useQuery({ + queryKey: ['admin', 'storage-decisions', filters], + queryFn: () => + apiAdminGetStorageDecisions(1, 100, { + user: filters.user || undefined, + status: filters.status || undefined, + source: filters.source || undefined, + }).then((res) => res.data.items), + staleTime: 30 * 1000, + }) + + const detailQuery = useQuery({ + queryKey: ['admin', 'storage-decision', selectedDecisionJobId], + queryFn: () => apiAdminGetStorageDecision(selectedDecisionJobId!).then((res) => res.data), + enabled: isDetailOpen && !!selectedDecisionJobId, + }) + + const replayMutation = useMutation({ + mutationFn: () => + apiAdminReplayStorageDecisions({ + limit: replayConfig.limit, + max_expand_ratio: replayConfig.maxExpandRatio, + max_expand_bytes: replayConfig.maxExpandBytesGiB * 1024 ** 3, + min_platform_reserved_ratio: replayConfig.minPlatformReservedRatio, + min_platform_reserved_bytes: replayConfig.minPlatformReservedBytesGiB * 1024 ** 3, + expansion_cooldown_hours: replayConfig.expansionCooldownHours, + force_freeze_when_over_quota: replayConfig.forceFreezeWhenOverQuota, + }), + onSuccess: (res) => { + setReplaySummary(res.data) + toast.success('回放评估已完成') + }, + onError: (error: unknown) => { + toast.error(getErrorMessage(error, '回放评估失败')) + }, + }) + + const autoShrinkMutation = useMutation({ + mutationFn: () => apiAdminRunAutoShrink(), + onSuccess: (res) => { + toast.success(res.data.message || '自动缩容扫描已完成') + void decisionsQuery.refetch() + }, + onError: (error: unknown) => { + toast.error(getErrorMessage(error, '自动缩容扫描失败')) + }, + }) + + const latestRecords = useMemo( + () => (decisionsQuery.data ?? []).slice(0, 20), + [decisionsQuery.data] + ) + const detail = detailQuery.data + + return ( + <> + + +
+ + + 决策记录与回放评估 + + + 查看已持久化的 LLM 决策记录、约束调整结果，并对历史决策执行新的策略回放。 + +
+
+ + + +
+ + +
+
+ + + setFilters((current) => ({ ...current, user: event.target.value })) + } + placeholder="按用户名筛选" + /> +
+
+ + +
+
+ + +
+
+ +
+ + + + 创建时间 + 用户 + 来源 + 状态 + 最终动作 + 最终扩容量 + 约束结果 + 耗时 + 操作 + + + + {latestRecords.length === 0 ? ( + + + {decisionsQuery.isLoading ? '正在加载决策记录...' : '暂无决策记录'} + + + ) : ( + latestRecords.map((record) => ( + + {record.created_at} + {record.username} + + + {decisionSourceLabel(record.source)} + + + + + {decisionStatusLabel(record.status)} + + + {appliedActionLabel(record.applied_action)} + + {record.final_allow_expand + ? formatBytes(record.final_expand_bytes) + : '不扩容'} + + +
+ {record.constraint_adjusted && 已调整} + {record.constraint_blocked && 已阻断} + {!record.constraint_adjusted && !record.constraint_blocked && ( + 直接通过 + )} +
+ + {record.latency_ms} ms + + + + + )) + )} + +
+
+ + + + + + + 决策详情 + + {detailQuery.isLoading ? ( +
正在加载决策详情...
+ ) : !detail ? ( +
暂无可用的决策详情。
+ ) : ( + + )} + + + + + + + 回放评估 + +
+
+ + + setReplayConfig((current) => ({ + ...current, + limit: Number(event.target.value) || 1, + })) + } + /> +
+
+ + + setReplayConfig((current) => ({ + ...current, + maxExpandRatio: Number(event.target.value) || 0, + })) + } + /> +
+
+ + + setReplayConfig((current) => ({ + ...current, + maxExpandBytesGiB: Number(event.target.value) || 1, + })) + } + /> +
+
+ + + setReplayConfig((current) => ({ + ...current, + minPlatformReservedRatio: Number(event.target.value) || 0, + })) + } + /> +
+
+ + + setReplayConfig((current) => ({ + ...current, + minPlatformReservedBytesGiB: Number(event.target.value) || 0, + })) + } + /> +
+
+ + + setReplayConfig((current) => ({ + ...current, + expansionCooldownHours: Number(event.target.value) || 0, + })) + } + /> +
+
+ +
+
+
超额后强制冻结
+
+ 如果回放后的策略拒绝扩容，且用户已经超过理论配额，则强制给出冻结建议。 +
+
+ +
+ + {replaySummary && } + + + + + + + + + ) +} + +function DecisionDetailContent({ detail }: { detail: StorageDecisionRecordDetail }) { + const snapshot = detail.snapshot + const currentShrinkStage = detail.current_shrink_stage || snapshot?.shrink_stage + + return ( +
+
+ + + + + + + + + + +
+ + {snapshot && ( + + + 决策快照 + 记录决策时的用户存储状态与平台上下文。 + + + + + + + + + + + + + )} + + + + 缩容阶段说明 + + 当前记录对应用户在自动缩容状态机中的阶段，以及系统为什么处于这个阶段。 + + + + +

+ {shrinkStageReason(currentShrinkStage)} +

+ + + +
+ + + +
+
+ ) +} + +function ReplaySummaryView({ summary }: { summary: ReplaySummary }) { + const records = summary.records ?? [] + + return ( +
+
+ + + + + +
+ +
+ + + + 用户 + 原始扩容 + 回放扩容 + 原始冻结 + 回放冻结 + 回放说明 + + + + {records.length === 0 ? ( + + + 暂无回放结果。 + + + ) : ( + records.slice(0, 10).map((record) => ( + + {record.username} + + {record.stored_allow_expand ? formatBytes(record.stored_expand_bytes) : '否'} + + + {record.replay_allow_expand ? formatBytes(record.replay_expand_bytes) : '否'} + + {record.stored_freeze ? '是' : '否'} + {record.replay_freeze ? '是' : '否'} + + {record.evaluation.adjustments.join('；') || + record.evaluation.violations.join('；') || + '无变化'} + + + )) + )} + +
+
+
+ ) +} + +function MetricCard({ label, value }: { label: string; value: string }) { + return ( +
+
{label}
+
{value}
+
+ ) +} + +function JsonPanel({ title, value }: { title: string; value: unknown }) { + return ( +
+
{title}
+