Fix/pr 420 storage governance

.gitattributes

@@ -0,0 +1,9 @@
+* text=auto eol=lf
+*.png -text
+*.jpg -text
+*.jpeg -text
+*.gif -text
+*.webp -text
+*.pdf -text
+*.zip -text
+backend/crater -text

.gitignore

@@ -79,3 +79,47 @@ kubeconfig*
 
 # Claude Code
 .claude/
+storage/etc/config.yaml
+.gocache/
+.gomodcache/
+
+# Local build/test scratch
+.gocache*/
+.gotmp*/
+backend/crater
+
+# Python cache
+__pycache__/
+*.pyc
+*.pyo
+
+# Temporary experiment artifacts
+tmp_*
+rewrite_section3*.py
+storage-governance-datasets.zip
+storage-governance-qwen25-7b.zip
+
+# Editor/tool backup files with random numeric suffixes
+*.go.[0-9]*
+
+# Local workspace-only directories and assets
+/.claude/
+/.gocache-test/
+/.gopath/
+/_external/
+/datasets/
+/models/
+/models_compare/
+/offline-tools/
+/offline-tools.zip
+/graduation-thesis-latex/
+
+# Local document/build logs
+/texput.log
+
+# Local documentation drafts
+/docs/zh-CN/framework-architecture-and-workflow.md
+/docs/zh-CN/storage-governance-requirements/
+/docs/zh-CN/storage-management.md
+/docs/zh-CN/storage-metadata-index-policy-engine.md
+/docs/zh-CN/storage-tiering-policy-satisfier.md

.vscode/launch.json

@@ -12,9 +12,11 @@
             "program": "${workspaceFolder}/backend/cmd/crater/main.go",
             "cwd": "${workspaceFolder}/backend",
             "env": {
+                "GOCACHE": "${workspaceFolder}/backend/.gocache",
+                "GOTOOLCHAIN": "go1.25.4",
                 "KUBECONFIG": "${workspaceFolder}/backend/kubeconfig",
                 "NO_PROXY": "k8s.cluster.master"
             }
         }
     ]
-}
+}

backend/.vscode/launch.json

@@ -16,6 +16,8 @@
                 "${workspaceFolder}/etc/debug-config.yaml"
             ],
             "env": {
+                "GOCACHE": "${workspaceFolder}/.gocache",
+                "GOTOOLCHAIN": "go1.25.4",
                 "KUBECONFIG": "${env:HOME}/.kube/config",
                 "NO_PROXY": "k8s.cluster.master"
             }
@@ -32,6 +34,8 @@
                 "${workspaceFolder}/etc/debug-config-actgpu.yaml"
             ],
             "env": {
+                "GOCACHE": "${workspaceFolder}/.gocache",
+                "GOTOOLCHAIN": "go1.25.4",
                 "KUBECONFIG": "${workspaceFolder}/kubeconfig_act"
             }
         },
@@ -47,6 +51,8 @@
                 "${workspaceFolder}/etc/debug-config-little.yaml"
             ],
             "env": {
+                "GOCACHE": "${workspaceFolder}/.gocache",
+                "GOTOOLCHAIN": "go1.25.4",
                 "KUBECONFIG": "${workspaceFolder}/kubeconfig_little"
             }
         },
@@ -62,8 +68,10 @@
                 "${workspaceFolder}/etc/debug-config-ali.yaml"
             ],
             "env": {
+                "GOCACHE": "${workspaceFolder}/.gocache",
+                "GOTOOLCHAIN": "go1.25.4",
                 "KUBECONFIG": "${workspaceFolder}/kubeconfig_ali"
             }
         }
     ]
-}
+}

backend/cmd/crater/helper/config.go

@@ -106,6 +106,7 @@ func (ci *ConfigInitializer) SetupManagerDependencies(registerConfig *handler.Re
 	registerConfig.CronJobManager = cronjob.NewCronJobManager(
 		registerConfig.Client,
 		registerConfig.KubeClient,
+		registerConfig.KubeConfig,
 		registerConfig.PrometheusClient,
 		registerConfig.GpuAnalysisService,
 		registerConfig.BillingService,

backend/cmd/gorm-gen/models/migrate.go

@@ -1573,6 +1573,9 @@ func main() {
 			},
 		},
 		modelDownloadSubmissionMigration(),
+		storageGovernanceMigration(),
+		storageGovernanceAutomationCleanupMigration(),
+		storageUsageManualRefreshMigration(),
 	})
 
 	m.InitSchema(func(tx *gorm.DB) error {
@@ -1609,6 +1612,9 @@ func main() {
 			&model.PrequeueConfig{},
 			&model.QueueQuotaLimit{},
 			&model.UserBanRecord{},
+			&model.UserSpaceSize{},
+			&model.TenantUsageHistory{},
+			&model.StorageDecisionRecord{},
 		)
 		if err != nil {
 			return err
@@ -1735,6 +1741,169 @@ func main() {
 	}
 }
 
+func storageGovernanceMigration() *gormigrate.Migration {
+	type userStorageColumns struct {
+		SpaceQuota           int64      `gorm:"type:bigint;default:-1"`
+		OriginalSpaceQuota   *int64     `gorm:"type:bigint;default:null"`
+		JobsFrozen           bool       `gorm:"type:boolean;default:false"`
+		ShrinkStage          *string    `gorm:"type:varchar(64);default:null"`
+		ShrinkStageUpdatedAt *time.Time `gorm:"default:null"`
+	}
+
+	return &gormigrate.Migration{
+		ID: "202608020001",
+		Migrate: func(tx *gorm.DB) error {
+			if err := tx.AutoMigrate(
+				&model.UserSpaceSize{},
+				&model.TenantUsageHistory{},
+				&model.StorageDecisionRecord{},
+			); err != nil {
+				return err
+			}
+
+			migrator := tx.Table("users").Migrator()
+			for _, field := range []string{
+				"SpaceQuota",
+				"OriginalSpaceQuota",
+				"JobsFrozen",
+				"ShrinkStage",
+				"ShrinkStageUpdatedAt",
+			} {
+				if migrator.HasColumn(&userStorageColumns{}, field) {
+					continue
+				}
+				if err := migrator.AddColumn(&userStorageColumns{}, field); err != nil {
+					return err
+				}
+			}
+
+			jobs := []model.CronJobConfig{
+				{
+					Name:    "update-user-space-size",
+					Type:    model.CronJobTypePatrolFunc,
+					Spec:    "*/30 * * * *",
+					Status:  model.CronJobConfigStatusIdle,
+					Config:  datatypes.JSON(`{}`),
+					EntryID: -1,
+				},
+			}
+			for i := range jobs {
+				if err := tx.Where("name = ?", jobs[i].Name).FirstOrCreate(&jobs[i]).Error; err != nil {
+					return err
+				}
+			}
+
+			return tx.Table("cron_job_configs").
+				Where("name IN ?", []string{
+					"refresh-public-storage-index-baseline",
+					"refresh-user-storage-index-daily",
+					"analyze-storage-alerts",
+					"auto-shrink-storage-expansions",
+				}).
+				Delete(nil).Error
+		},
+		Rollback: func(tx *gorm.DB) error {
+			if err := tx.Table("cron_job_configs").
+				Where("name IN ?", []string{
+					"update-user-space-size",
+				}).
+				Delete(nil).Error; err != nil {
+				return err
+			}
+
+			migrator := tx.Table("users").Migrator()
+			for _, field := range []string{
+				"ShrinkStageUpdatedAt",
+				"ShrinkStage",
+				"JobsFrozen",
+				"OriginalSpaceQuota",
+				"SpaceQuota",
+			} {
+				if migrator.HasColumn(&userStorageColumns{}, field) {
+					if err := migrator.DropColumn(&userStorageColumns{}, field); err != nil {
+						return err
+					}
+				}
+			}
+
+			return tx.Migrator().DropTable(
+				&model.StorageDecisionRecord{},
+				&model.TenantUsageHistory{},
+				&model.UserSpaceSize{},
+			)
+		},
+	}
+}
+
+func storageGovernanceAutomationCleanupMigration() *gormigrate.Migration {
+	jobNames := []string{
+		"update-user-space-size",
+		"analyze-storage-alerts",
+		"auto-shrink-storage-expansions",
+	}
+
+	return &gormigrate.Migration{
+		ID: "202608080001",
+		Migrate: func(tx *gorm.DB) error {
+			return tx.Table("cron_job_configs").Where("name IN ?", jobNames).Delete(nil).Error
+		},
+		Rollback: func(tx *gorm.DB) error {
+			jobs := []model.CronJobConfig{
+				{
+					Name:    "update-user-space-size",
+					Type:    model.CronJobTypePatrolFunc,
+					Spec:    "*/30 * * * *",
+					Status:  model.CronJobConfigStatusIdle,
+					Config:  datatypes.JSON(`{}`),
+					EntryID: -1,
+				},
+				{
+					Name:    "analyze-storage-alerts",
+					Type:    model.CronJobTypePatrolFunc,
+					Spec:    "*/30 * * * *",
+					Status:  model.CronJobConfigStatusSuspended,
+					Config:  datatypes.JSON(`{}`),
+					EntryID: -1,
+				},
+				{
+					Name:    "auto-shrink-storage-expansions",
+					Type:    model.CronJobTypePatrolFunc,
+					Spec:    "0 * * * *",
+					Status:  model.CronJobConfigStatusSuspended,
+					Config:  datatypes.JSON(`{}`),
+					EntryID: -1,
+				},
+			}
+			for i := range jobs {
+				if err := tx.Where("name = ?", jobs[i].Name).FirstOrCreate(&jobs[i]).Error; err != nil {
+					return err
+				}
+			}
+			return nil
+		},
+	}
+}
+
+func storageUsageManualRefreshMigration() *gormigrate.Migration {
+	return &gormigrate.Migration{
+		ID: "202608090001",
+		Migrate: func(tx *gorm.DB) error {
+			return tx.Table("cron_job_configs").Where("name = ?", "update-user-space-size").Delete(nil).Error
+		},
+		Rollback: func(tx *gorm.DB) error {
+			job := model.CronJobConfig{
+				Name:    "update-user-space-size",
+				Type:    model.CronJobTypePatrolFunc,
+				Spec:    "*/30 * * * *",
+				Status:  model.CronJobConfigStatusIdle,
+				Config:  datatypes.JSON(`{}`),
+				EntryID: -1,
+			}
+			return tx.Where("name = ?", job.Name).FirstOrCreate(&job).Error
+		},
+	}
+}
+
 type jobListIndex struct {
 	create string
 	drop   string

backend/cmd/storage-server/main.go

@@ -11,6 +11,7 @@ import (
 	"github.com/raids-lab/crater/dao/query"
 	"github.com/raids-lab/crater/internal/storage"
 	"github.com/raids-lab/crater/pkg/config"
+	"github.com/raids-lab/crater/pkg/storagequota"
 )
 
 var (
@@ -63,8 +64,21 @@ func main() {
 		}
 	}
 
-	_ = config.GetConfig()
-	query.SetDefault(query.GetDB())
+	mode := strings.ToLower(firstNonEmptyEnv("CRATER_STORAGE_MODE"))
+	if mode == "" {
+		mode = "full"
+	}
+	if mode != "full" && mode != "quota-agent" {
+		klog.Fatalf("unsupported storage-server mode %q; expected full or quota-agent", mode)
+	}
+	hasInternalCredential := strings.TrimSpace(os.Getenv(storagequota.InternalTokenEnv)) != "" ||
+		strings.TrimSpace(os.Getenv(storagequota.InternalSecretEnv)) != ""
+	if mode == "full" || !hasInternalCredential {
+		_ = config.GetConfig()
+	}
+	if mode == "full" {
+		query.SetDefault(query.GetDB())
+	}
 
 	port := firstNonEmptyEnv("CRATER_STORAGE_PORT", "PORT")
 	if port == "" {
@@ -79,14 +93,18 @@ func main() {
 		klog.Fatalf("failed to create storage root directory %s: %v", rootDir, err)
 	}
 	storage.SetRootDir(rootDir)
-	go storage.StartCheckSpace()
 
 	r := gin.Default()
-	storage.RegisterRoutes(r)
+	if mode == "quota-agent" {
+		storage.RegisterQuotaRoutes(r)
+	} else {
+		go storage.StartCheckSpace()
+		storage.RegisterRoutes(r)
+	}
 
 	addr := normalizePort(port)
-	klog.Infof("storage-server starting on %s (version=%s, commit=%s, buildType=%s, buildTime=%s)",
-		addr, AppVersion, CommitSHA, BuildType, BuildTime)
+	klog.Infof("storage-server starting on %s (mode=%s, version=%s, commit=%s, buildType=%s, buildTime=%s)",
+		addr, mode, AppVersion, CommitSHA, BuildType, BuildTime)
 	if err := r.Run(addr); err != nil {
 		klog.Fatalf("failed to run storage-server: %v", err)
 	}