diff --git a/backend/go.mod b/backend/go.mod index 09d21c0c8..54b5eb437 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -26,6 +26,7 @@ require ( golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 golang.org/x/net v0.41.0 golang.org/x/sync v0.16.0 + golang.org/x/sys v0.34.0 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 @@ -143,7 +144,6 @@ require ( golang.org/x/arch v0.18.0 // indirect golang.org/x/mod v0.25.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/sys v0.34.0 // indirect golang.org/x/term v0.33.0 // indirect golang.org/x/text v0.27.0 // indirect golang.org/x/time v0.9.0 // indirect diff --git a/backend/internal/bizerr/groups.go b/backend/internal/bizerr/groups.go index 08781d9bf..18f36fbac 100644 --- a/backend/internal/bizerr/groups.go +++ b/backend/internal/bizerr/groups.go @@ -57,6 +57,8 @@ type notFoundGroup struct { ServiceSshdNotFound BizCode `code:"40402"` // K8sResourceNotFound: 集群中找不到指定的 Pod, Deployment 或 Namespace K8sResourceNotFound BizCode `code:"40403"` + // StorageResourceNotFound: 存储空间中找不到指定的文件或目录 + StorageResourceNotFound BizCode `code:"40404"` } // methodNotAllowedGroup 405xx - 方法不允许 diff --git a/backend/internal/storage/dataset.go b/backend/internal/storage/dataset.go index 62e19e90b..33902cc7b 100644 --- a/backend/internal/storage/dataset.go +++ b/backend/internal/storage/dataset.go @@ -2,6 +2,7 @@ package storage import ( "context" + "errors" "fmt" "net/http" "os" @@ -11,7 +12,9 @@ import ( "github.com/raids-lab/crater/dao/model" "github.com/raids-lab/crater/dao/query" + "github.com/raids-lab/crater/internal/bizerr" "github.com/raids-lab/crater/internal/resputil" + "github.com/raids-lab/crater/internal/util" "github.com/gin-gonic/gin" ) @@ -20,45 +23,180 @@ type MoveFileReq struct { Dst string `json:"dst" binding:"required"` } +var ( + errMoveSourceNotFound = errors.New("move source does not exist") + errMoveTargetExists = errors.New("move destination exists") + errMoveNoReplaceUnsupported = errors.New("atomic no-replace move is unsupported") +) + +type moveFileHandlerDeps struct { + authenticate func(*gin.Context) (util.JWTMessage, error) + permission func(string, util.JWTMessage, *gin.Context) model.FilePermission + redirect func(*gin.Context, string, util.JWTMessage) (string, error) + openTarget func(string, string, string) (*os.Root, string, error) + move func(*os.Root, string, *os.Root, string) error + storageRoot string +} + +func defaultMoveFileHandlerDeps() moveFileHandlerDeps { + return moveFileHandlerDeps{ + authenticate: CheckJWTToken, + permission: GetPermission, + redirect: Redirect, + openTarget: openUploadTarget, + move: moveStorageEntry, + storageRoot: storageRootDir, + } +} + func MoveFile(c *gin.Context) { AlloweOption(c) - checkfs() - jwttoken, err := CheckJWTToken(c) + moveFileWithDeps(c, defaultMoveFileHandlerDeps()) +} + +//nolint:gocyclo // Keep each authorization and filesystem failure mapped to its specific public error contract. +func moveFileWithDeps(c *gin.Context, deps moveFileHandlerDeps) { + jwttoken, err := deps.authenticate(c) if err != nil { - resputil.Error(c, err.Error(), resputil.NotSpecified) + resputil.HandleError(c, bizerr.Auth.TokenInvalid.New("invalid token")) return } var moveFileReq MoveFileReq - err = c.ShouldBind(&moveFileReq) + if err := c.ShouldBindJSON(&moveFileReq); err != nil { + resputil.HandleError(c, bizerr.BadRequest.InvalidRequest.Wrap(err, "invalid move request")) + return + } + + sourcePath, err := normalizeWebDAVMutationLogicalPath(c.Param("path")) if err != nil { - resputil.BadRequestError(c, err.Error()) + resputil.HandleError(c, bizerr.BadRequest.ParameterError.New("invalid source path")) return } - param := strings.TrimPrefix(c.Request.URL.Path, "/api/ss/move") - sourcePermission := GetPermission(param, jwttoken, c) - dstPermission := GetPermission(moveFileReq.Dst, jwttoken, c) + destinationPath, err := normalizeWebDAVMutationLogicalPath(moveFileReq.Dst) + if err != nil { + resputil.HandleError(c, bizerr.BadRequest.ParameterError.New("invalid destination path")) + return + } + if sourcePath == destinationPath || strings.HasPrefix(destinationPath, sourcePath+"/") { + resputil.HandleError(c, bizerr.BadRequest.ParameterError.New("destination must be outside the source path")) + return + } + + sourcePermission := deps.permission(sourcePath, jwttoken, c) + dstPermission := deps.permission(destinationPath, jwttoken, c) if sourcePermission != model.ReadWrite || dstPermission != model.ReadWrite { - resputil.HTTPError(c, http.StatusUnauthorized, "You have no permission to move files or move files to this location ", - resputil.NotSpecified) + resputil.HandleError(c, bizerr.Forbidden.PermissionDenied.New("write permission is required for source and destination")) return } - realPath, err := Redirect(c, param, jwttoken) + + realSource, err := deps.redirect(c, sourcePath, jwttoken) if err != nil { - resputil.Error(c, err.Error(), resputil.NotSpecified) + resputil.HandleError(c, bizerr.Internal.FileSystemError.Wrap(err, "failed to resolve move source")) return } - realDst, err := Redirect(c, moveFileReq.Dst, jwttoken) + realDestination, err := deps.redirect(c, destinationPath, jwttoken) if err != nil { - resputil.Error(c, err.Error(), resputil.NotSpecified) + resputil.HandleError(c, bizerr.Internal.FileSystemError.Wrap(err, "failed to resolve move destination")) + return } - err = moveFiles(c.Request.Context(), realPath, realDst, false) + + sourceRoot, err := deps.redirect(c, strings.Split(sourcePath, "/")[0], jwttoken) if err != nil { - resputil.Error(c, err.Error(), resputil.NotSpecified) + resputil.HandleError(c, bizerr.Internal.FileSystemError.Wrap(err, "failed to resolve move source")) + return + } + destinationRoot, err := deps.redirect(c, strings.Split(destinationPath, "/")[0], jwttoken) + if err != nil { + resputil.HandleError(c, bizerr.Internal.FileSystemError.Wrap(err, "failed to resolve move destination")) + return + } + + sourceParent, sourceName, err := deps.openTarget(deps.storageRoot, sourceRoot, realSource) + if err != nil { + handleMoveTargetOpenError(c, err, true, "source parent directory is unavailable") + return + } + defer sourceParent.Close() + destinationParent, destinationName, err := deps.openTarget( + deps.storageRoot, + destinationRoot, + realDestination, + ) + if err != nil { + handleMoveTargetOpenError(c, err, false, "destination parent directory is unavailable") return } + defer destinationParent.Close() + + if err := deps.move(sourceParent, sourceName, destinationParent, destinationName); err != nil { + switch { + case errors.Is(err, errMoveSourceNotFound): + resputil.HandleError(c, bizerr.NotFound.StorageResourceNotFound.New("source path does not exist")) + case errors.Is(err, errMoveTargetExists): + resputil.HandleError(c, bizerr.Conflict.ResourceAlreadyExists.New("destination path already exists")) + default: + resputil.HandleError(c, bizerr.Internal.FileSystemError.Wrap(err, "failed to move storage entry")) + } + return + } + resputil.Success(c, "move files successfully") } +func handleMoveTargetOpenError(c *gin.Context, err error, source bool, message string) { + if errors.Is(err, errUploadParentInvalid) { + if source && isUploadParentMissing(err) { + resputil.HandleError(c, bizerr.NotFound.StorageResourceNotFound.New("source path does not exist")) + return + } + if isUploadParentInfrastructureFailure(err) { + resputil.HandleError(c, bizerr.Internal.FileSystemError.Wrap(err, "failed to access storage")) + return + } + resputil.HandleError(c, bizerr.Conflict.ResourceStatusError.New(message)) + return + } + resputil.HandleError(c, bizerr.Internal.FileSystemError.Wrap(err, "failed to access storage")) +} + +//nolint:gocyclo // The explicit checks preserve no-clobber and source-not-found semantics around one rename. +func moveStorageEntry( + sourceParent *os.Root, + sourceName string, + destinationParent *os.Root, + destinationName string, +) error { + if sourceParent == nil || destinationParent == nil || + sourceName == "" || sourceName == "." || sourceName == parentPathSegment || + destinationName == "" || destinationName == "." || destinationName == parentPathSegment || + filepath.Base(sourceName) != sourceName || filepath.Base(destinationName) != destinationName { + return errUploadParentInvalid + } + if _, err := sourceParent.Lstat(sourceName); err != nil { + if os.IsNotExist(err) { + return errMoveSourceNotFound + } + return err + } + if _, err := destinationParent.Lstat(destinationName); err == nil { + return errMoveTargetExists + } else if !os.IsNotExist(err) { + return err + } + if err := renameStorageNoReplace(sourceParent, sourceName, destinationParent, destinationName); err != nil { + if os.IsExist(err) { + return errMoveTargetExists + } + if os.IsNotExist(err) { + if _, sourceErr := sourceParent.Lstat(sourceName); os.IsNotExist(sourceErr) { + return errMoveSourceNotFound + } + } + return err + } + return nil +} + func MoveDatasetOrModel(c *gin.Context) { AlloweOption(c) checkfs() diff --git a/backend/internal/storage/file.go b/backend/internal/storage/file.go index eed53600c..cd1203b6f 100644 --- a/backend/internal/storage/file.go +++ b/backend/internal/storage/file.go @@ -874,6 +874,7 @@ func RegisterFile(webdavGroup *gin.RouterGroup) { webdavGroup.GET("/admin/files", GetAllFiles) webdavGroup.GET("/admin/files/*path", GetAllFiles) webdavGroup.GET("/download/*path", Download) + webdavGroup.POST("/upload/*path", UploadFile) webdavGroup.DELETE("/delete/*path", DeleteFile) webdavGroup.GET("/userspace", GetUserSpace) webdavGroup.GET("/queuespace", GetAccountSpace) diff --git a/backend/internal/storage/mkdir.go b/backend/internal/storage/mkdir.go new file mode 100644 index 000000000..c9ce34137 --- /dev/null +++ b/backend/internal/storage/mkdir.go @@ -0,0 +1,111 @@ +package storage + +import ( + "errors" + "net/http" + "os" + "strings" + + "github.com/gin-gonic/gin" + + "github.com/raids-lab/crater/dao/model" + "github.com/raids-lab/crater/internal/bizerr" + "github.com/raids-lab/crater/internal/resputil" + "github.com/raids-lab/crater/internal/util" +) + +type createDirectoryHandlerDeps struct { + authenticate func(*gin.Context) (util.JWTMessage, error) + permission func(string, util.JWTMessage, *gin.Context) model.FilePermission + redirect func(*gin.Context, string, util.JWTMessage) (string, error) + openTarget func(string, string, string) (*os.Root, string, error) + mkdir func(*os.Root, string, os.FileMode) error + storageRoot string +} + +func defaultCreateDirectoryHandlerDeps() createDirectoryHandlerDeps { + return createDirectoryHandlerDeps{ + authenticate: CheckJWTToken, + permission: GetPermission, + redirect: Redirect, + openTarget: openUploadTarget, + mkdir: createStorageDirectory, + storageRoot: storageRootDir, + } +} + +// CreateDirectory creates exactly one directory through the existing WebDAV +// MKCOL route while returning Crater's stable error envelope on failure. +func CreateDirectory(c *gin.Context) { + AlloweOption(c) + createDirectoryWithDeps(c, defaultCreateDirectoryHandlerDeps()) +} + +func createDirectoryWithDeps(c *gin.Context, deps createDirectoryHandlerDeps) { + token, err := deps.authenticate(c) + if err != nil { + resputil.HandleError(c, bizerr.Auth.TokenInvalid.New("invalid token")) + return + } + + logicalPath, err := normalizeWebDAVMutationLogicalPath(c.Param("path")) + if err != nil { + resputil.HandleError(c, bizerr.BadRequest.ParameterError.New("invalid directory path")) + return + } + if permission := deps.permission(logicalPath, token, c); permission != model.ReadWrite { + resputil.HandleError(c, bizerr.Forbidden.PermissionDenied.New("write permission is required")) + return + } + + realPath, err := deps.redirect(c, logicalPath, token) + if err != nil { + resputil.HandleError(c, bizerr.Internal.FileSystemError.Wrap(err, "failed to resolve directory path")) + return + } + logicalRoot := strings.Split(logicalPath, "/")[0] + realRoot, err := deps.redirect(c, logicalRoot, token) + if err != nil { + resputil.HandleError(c, bizerr.Internal.FileSystemError.Wrap(err, "failed to resolve directory path")) + return + } + + parent, targetName, err := deps.openTarget(deps.storageRoot, realRoot, realPath) + if err != nil { + if errors.Is(err, errUploadParentInvalid) { + if isUploadParentInfrastructureFailure(err) { + resputil.HandleError(c, bizerr.Internal.FileSystemError.Wrap(err, "failed to access storage")) + return + } + resputil.HandleError(c, bizerr.Conflict.ResourceStatusError.New("directory parent is unavailable")) + return + } + resputil.HandleError(c, bizerr.Internal.FileSystemError.Wrap(err, "failed to access storage")) + return + } + defer parent.Close() + + if err := deps.mkdir(parent, targetName, model.RWXFolderPerm); err != nil { + switch { + case os.IsExist(err): + resputil.HandleError(c, bizerr.Conflict.ResourceAlreadyExists.New("directory path already exists")) + case os.IsNotExist(err): + resputil.HandleError(c, bizerr.Conflict.ResourceStatusError.New("directory parent is unavailable")) + default: + resputil.HandleError(c, bizerr.Internal.FileSystemError.Wrap(err, "failed to create directory")) + } + return + } + c.Status(http.StatusCreated) +} + +func createStorageDirectory(parent *os.Root, name string, mode os.FileMode) error { + if parent == nil || name == "" || name == "." || name == parentPathSegment || + mode.Perm() != mode { + return errUploadParentInvalid + } + if err := parent.Mkdir(name, mode); err != nil { + return err + } + return chmodCreatedStorageDirectory(parent, name, mode) +} diff --git a/backend/internal/storage/mkdir_chmod_other.go b/backend/internal/storage/mkdir_chmod_other.go new file mode 100644 index 000000000..29f03e574 --- /dev/null +++ b/backend/internal/storage/mkdir_chmod_other.go @@ -0,0 +1,24 @@ +//go:build !darwin && !dragonfly && !freebsd && !linux && !netbsd && !openbsd + +package storage + +import ( + "errors" + "os" +) + +func chmodCreatedStorageDirectory(parent *os.Root, name string, mode os.FileMode) error { + created, err := parent.Open(name) + if err != nil { + return err + } + defer created.Close() + info, err := created.Stat() + if err != nil { + return err + } + if !info.IsDir() { + return errors.New("created storage entry is no longer a directory") + } + return created.Chmod(mode) +} diff --git a/backend/internal/storage/mkdir_chmod_unix.go b/backend/internal/storage/mkdir_chmod_unix.go new file mode 100644 index 000000000..8f433c277 --- /dev/null +++ b/backend/internal/storage/mkdir_chmod_unix.go @@ -0,0 +1,42 @@ +//go:build darwin || dragonfly || freebsd || linux || netbsd || openbsd + +package storage + +import ( + "errors" + "os" + + "golang.org/x/sys/unix" +) + +func chmodCreatedStorageDirectory(parent *os.Root, name string, mode os.FileMode) error { + parentDirectory, err := parent.Open(".") + if err != nil { + return err + } + defer parentDirectory.Close() + + var directory int + for { + directory, err = unix.Openat( + int(parentDirectory.Fd()), + name, + unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, + 0, + ) + if !errors.Is(err, unix.EINTR) { + break + } + } + if err != nil { + return err + } + defer unix.Close(directory) + + for { + err = unix.Fchmod(directory, uint32(mode.Perm())) + if !errors.Is(err, unix.EINTR) { + return err + } + } +} diff --git a/backend/internal/storage/mkdir_chmod_unix_test.go b/backend/internal/storage/mkdir_chmod_unix_test.go new file mode 100644 index 000000000..0cf552c23 --- /dev/null +++ b/backend/internal/storage/mkdir_chmod_unix_test.go @@ -0,0 +1,36 @@ +//go:build darwin || dragonfly || freebsd || linux || netbsd || openbsd + +package storage + +import ( + "os" + "path/filepath" + "testing" +) + +func TestChmodCreatedStorageDirectoryDoesNotFollowSymlink(t *testing.T) { + storage := t.TempDir() + target := filepath.Join(storage, "target") + if err := os.Mkdir(target, 0o700); err != nil { + t.Fatal(err) + } + if err := os.Symlink("target", filepath.Join(storage, "created")); err != nil { + t.Fatal(err) + } + root, err := os.OpenRoot(storage) + if err != nil { + t.Fatal(err) + } + defer root.Close() + + if err := chmodCreatedStorageDirectory(root, "created", 0o777); err == nil { + t.Fatal("chmod through a symlink unexpectedly succeeded") + } + info, err := os.Stat(target) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o700 { + t.Fatalf("symlink target mode = %v, want 0700", info.Mode().Perm()) + } +} diff --git a/backend/internal/storage/mkdir_test.go b/backend/internal/storage/mkdir_test.go new file mode 100644 index 000000000..34da883b7 --- /dev/null +++ b/backend/internal/storage/mkdir_test.go @@ -0,0 +1,248 @@ +package storage + +import ( + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/gin-gonic/gin" + + "github.com/raids-lab/crater/dao/model" + "github.com/raids-lab/crater/internal/util" +) + +const ( + testLogicalUserRoot = "user" + testRealUserRoot = "users/alice" +) + +func TestCreateStorageDirectoryCreatesOneDirectoryWithRequestedMode(t *testing.T) { + storage := t.TempDir() + root, err := os.OpenRoot(storage) + if err != nil { + t.Fatal(err) + } + defer root.Close() + + if err := createStorageDirectory(root, "new-directory", 0o777); err != nil { + t.Fatal(err) + } + info, err := os.Stat(filepath.Join(storage, "new-directory")) + if err != nil { + t.Fatal(err) + } + if !info.IsDir() || info.Mode().Perm() != 0o777 { + t.Fatalf("directory mode = %v", info.Mode()) + } + if err := createStorageDirectory(root, "new-directory", 0o777); !os.IsExist(err) { + t.Fatalf("existing directory error = %v, want exists", err) + } +} + +func TestCreateStorageDirectoryRaceHasOneWinner(t *testing.T) { + storage := t.TempDir() + const contenders = 8 + roots := make([]*os.Root, contenders) + t.Cleanup(func() { + for _, root := range roots { + if root != nil { + _ = root.Close() + } + } + }) + for index := range roots { + root, err := os.OpenRoot(storage) + if err != nil { + t.Fatal(err) + } + roots[index] = root + } + + var wait sync.WaitGroup + wait.Add(contenders) + start := make(chan struct{}) + errorsSeen := make(chan error, contenders) + for index := range roots { + go func() { + defer wait.Done() + <-start + errorsSeen <- createStorageDirectory(roots[index], "new-directory", 0o777) + }() + } + close(start) + wait.Wait() + close(errorsSeen) + + successes := 0 + conflicts := 0 + for err := range errorsSeen { + switch { + case err == nil: + successes++ + case os.IsExist(err): + conflicts++ + default: + t.Fatalf("unexpected mkdir error: %v", err) + } + } + if successes != 1 || conflicts != contenders-1 { + t.Fatalf("successes=%d conflicts=%d", successes, conflicts) + } + info, err := os.Stat(filepath.Join(storage, "new-directory")) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o777 { + t.Fatalf("directory mode = %v, want 0777", info.Mode().Perm()) + } +} + +func TestCreateDirectoryHandlerHTTPContract(t *testing.T) { + gin.SetMode(gin.TestMode) + + t.Run("created", func(t *testing.T) { + storage := newCreateDirectoryHandlerStorage(t) + recorder := serveCreateDirectory(t, testCreateDirectoryHandlerDeps(storage), "user/parent/new-directory") + if recorder.Code != http.StatusCreated { + t.Fatalf("status = %d, want 201; body=%s", recorder.Code, recorder.Body.String()) + } + info, err := os.Stat(filepath.Join(storage, "users", "alice", "parent", "new-directory")) + if err != nil { + t.Fatal(err) + } + if !info.IsDir() || info.Mode().Perm() != 0o777 { + t.Fatalf("directory mode = %v", info.Mode()) + } + }) + + tests := []struct { + name string + path string + wantStatus int + wantCode int + mutate func(*testing.T, string, *createDirectoryHandlerDeps) + }{ + { + name: "invalid root path", path: testLogicalUserRoot, + wantStatus: http.StatusBadRequest, wantCode: 40004, + }, + { + name: "invalid traversal", path: "user/../public/new-directory", + wantStatus: http.StatusBadRequest, wantCode: 40004, + }, + { + name: "unauthorized", path: "user/parent/new-directory", + wantStatus: http.StatusUnauthorized, wantCode: 40102, + mutate: func(_ *testing.T, _ string, deps *createDirectoryHandlerDeps) { + deps.authenticate = func(*gin.Context) (util.JWTMessage, error) { + return util.JWTMessage{}, errors.New("invalid token") + } + }, + }, + { + name: "forbidden", path: "public/new-directory", + wantStatus: http.StatusForbidden, wantCode: 40301, + mutate: func(_ *testing.T, _ string, deps *createDirectoryHandlerDeps) { + deps.permission = func(string, util.JWTMessage, *gin.Context) model.FilePermission { + return model.ReadOnly + } + }, + }, + { + name: "already exists", path: "user/parent/existing", + wantStatus: http.StatusConflict, wantCode: 40901, + mutate: func(t *testing.T, storage string, _ *createDirectoryHandlerDeps) { + if err := os.Mkdir(filepath.Join(storage, "users", "alice", "parent", "existing"), 0o700); err != nil { + t.Fatal(err) + } + }, + }, + { + name: "missing parent", path: "user/missing/new-directory", + wantStatus: http.StatusConflict, wantCode: 40902, + }, + { + name: "redirect failure", path: "user/parent/new-directory", + wantStatus: http.StatusInternalServerError, wantCode: 50005, + mutate: func(_ *testing.T, storage string, deps *createDirectoryHandlerDeps) { + deps.redirect = func(*gin.Context, string, util.JWTMessage) (string, error) { + return "", errors.New(filepath.Join(storage, "private-path")) + } + }, + }, + { + name: "filesystem failure", path: "user/parent/new-directory", + wantStatus: http.StatusInternalServerError, wantCode: 50005, + mutate: func(_ *testing.T, _ string, deps *createDirectoryHandlerDeps) { + deps.mkdir = func(*os.Root, string, os.FileMode) error { + return errors.New("disk unavailable") + } + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + storage := newCreateDirectoryHandlerStorage(t) + deps := testCreateDirectoryHandlerDeps(storage) + if test.mutate != nil { + test.mutate(t, storage, &deps) + } + recorder := serveCreateDirectory(t, deps, test.path) + assertMoveEnvelope(t, recorder, test.wantStatus, test.wantCode) + if strings.Contains(recorder.Body.String(), storage) { + t.Fatalf("response leaked physical storage path: %s", recorder.Body.String()) + } + }) + } +} + +func newCreateDirectoryHandlerStorage(t *testing.T) string { + t.Helper() + storage := t.TempDir() + if err := os.MkdirAll(filepath.Join(storage, "users", "alice", "parent"), 0o755); err != nil { + t.Fatal(err) + } + return storage +} + +func testCreateDirectoryHandlerDeps(storage string) createDirectoryHandlerDeps { + return createDirectoryHandlerDeps{ + authenticate: func(*gin.Context) (util.JWTMessage, error) { + return util.JWTMessage{}, nil + }, + permission: func(string, util.JWTMessage, *gin.Context) model.FilePermission { + return model.ReadWrite + }, + redirect: func(_ *gin.Context, logicalPath string, _ util.JWTMessage) (string, error) { + if logicalPath == testLogicalUserRoot { + return testRealUserRoot, nil + } + return testRealUserRoot + "/" + strings.TrimPrefix(logicalPath, testLogicalUserRoot+"/"), nil + }, + openTarget: openUploadTarget, + mkdir: createStorageDirectory, + storageRoot: storage, + } +} + +func serveCreateDirectory( + t *testing.T, + deps createDirectoryHandlerDeps, + logicalPath string, +) *httptest.ResponseRecorder { + t.Helper() + router := gin.New() + router.Handle("MKCOL", "/mkdir/*path", func(c *gin.Context) { + createDirectoryWithDeps(c, deps) + }) + request := httptest.NewRequest("MKCOL", "/mkdir/"+logicalPath, http.NoBody) + recorder := httptest.NewRecorder() + router.ServeHTTP(recorder, request) + return recorder +} diff --git a/backend/internal/storage/move_noreplace_bsd.go b/backend/internal/storage/move_noreplace_bsd.go new file mode 100644 index 000000000..fbf0b380c --- /dev/null +++ b/backend/internal/storage/move_noreplace_bsd.go @@ -0,0 +1,14 @@ +//go:build dragonfly || freebsd || netbsd || openbsd + +package storage + +import "os" + +func renameStorageNoReplace( + sourceParent *os.Root, + sourceName string, + destinationParent *os.Root, + destinationName string, +) error { + return errMoveNoReplaceUnsupported +} diff --git a/backend/internal/storage/move_noreplace_darwin.go b/backend/internal/storage/move_noreplace_darwin.go new file mode 100644 index 000000000..361302e0d --- /dev/null +++ b/backend/internal/storage/move_noreplace_darwin.go @@ -0,0 +1,37 @@ +//go:build darwin + +package storage + +import ( + "errors" + "os" + + "golang.org/x/sys/unix" +) + +func renameStorageNoReplace( + sourceParent *os.Root, + sourceName string, + destinationParent *os.Root, + destinationName string, +) error { + sourceDirectory, destinationDirectory, err := openUploadDirectoryHandles(sourceParent, destinationParent) + if err != nil { + return err + } + defer sourceDirectory.Close() + defer destinationDirectory.Close() + err = unix.RenameatxNp( + int(sourceDirectory.Fd()), + sourceName, + int(destinationDirectory.Fd()), + destinationName, + unix.RENAME_EXCL, + ) + if errors.Is(err, unix.EINVAL) || + errors.Is(err, unix.ENOSYS) || + errors.Is(err, unix.EOPNOTSUPP) { + return errMoveNoReplaceUnsupported + } + return err +} diff --git a/backend/internal/storage/move_noreplace_linux.go b/backend/internal/storage/move_noreplace_linux.go new file mode 100644 index 000000000..f965ad739 --- /dev/null +++ b/backend/internal/storage/move_noreplace_linux.go @@ -0,0 +1,37 @@ +//go:build linux + +package storage + +import ( + "errors" + "os" + + "golang.org/x/sys/unix" +) + +func renameStorageNoReplace( + sourceParent *os.Root, + sourceName string, + destinationParent *os.Root, + destinationName string, +) error { + sourceDirectory, destinationDirectory, err := openUploadDirectoryHandles(sourceParent, destinationParent) + if err != nil { + return err + } + defer sourceDirectory.Close() + defer destinationDirectory.Close() + err = unix.Renameat2( + int(sourceDirectory.Fd()), + sourceName, + int(destinationDirectory.Fd()), + destinationName, + unix.RENAME_NOREPLACE, + ) + if errors.Is(err, unix.EINVAL) || + errors.Is(err, unix.ENOSYS) || + errors.Is(err, unix.EOPNOTSUPP) { + return errMoveNoReplaceUnsupported + } + return err +} diff --git a/backend/internal/storage/move_noreplace_other.go b/backend/internal/storage/move_noreplace_other.go new file mode 100644 index 000000000..364b419ac --- /dev/null +++ b/backend/internal/storage/move_noreplace_other.go @@ -0,0 +1,11 @@ +//go:build !darwin && !dragonfly && !freebsd && !linux && !netbsd && !openbsd + +package storage + +import ( + "os" +) + +func renameStorageNoReplace(*os.Root, string, *os.Root, string) error { + return errMoveNoReplaceUnsupported +} diff --git a/backend/internal/storage/move_test.go b/backend/internal/storage/move_test.go new file mode 100644 index 000000000..3cc7cf7af --- /dev/null +++ b/backend/internal/storage/move_test.go @@ -0,0 +1,505 @@ +package storage + +import ( + "bytes" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/gin-gonic/gin" + + "github.com/raids-lab/crater/dao/model" + "github.com/raids-lab/crater/internal/util" +) + +func TestMoveStorageEntryMovesFileAndDirectory(t *testing.T) { + for _, test := range []struct { + name string + makeEntry func(*testing.T, string) + assert func(*testing.T, string) + }{ + { + name: "file", + makeEntry: func(t *testing.T, path string) { + t.Helper() + if err := os.WriteFile(path, []byte("payload"), 0o600); err != nil { + t.Fatal(err) + } + }, + assert: func(t *testing.T, path string) { + t.Helper() + assertStoredFile(t, path, []byte("payload")) + }, + }, + { + name: "directory", + makeEntry: func(t *testing.T, path string) { + t.Helper() + if err := os.Mkdir(path, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(path, "nested.txt"), []byte("nested"), 0o600); err != nil { + t.Fatal(err) + } + }, + assert: func(t *testing.T, path string) { + t.Helper() + assertStoredFile(t, filepath.Join(path, "nested.txt"), []byte("nested")) + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + storage := t.TempDir() + sourceDirectory := filepath.Join(storage, "source") + destinationDirectory := filepath.Join(storage, "destination") + if err := os.Mkdir(sourceDirectory, 0o700); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(destinationDirectory, 0o700); err != nil { + t.Fatal(err) + } + test.makeEntry(t, filepath.Join(sourceDirectory, "entry")) + + sourceRoot, err := os.OpenRoot(sourceDirectory) + if err != nil { + t.Fatal(err) + } + defer sourceRoot.Close() + destinationRoot, err := os.OpenRoot(destinationDirectory) + if err != nil { + t.Fatal(err) + } + defer destinationRoot.Close() + + if err := moveStorageEntry(sourceRoot, "entry", destinationRoot, "renamed"); err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(filepath.Join(sourceDirectory, "entry")); !os.IsNotExist(err) { + t.Fatalf("source still exists: %v", err) + } + test.assert(t, filepath.Join(destinationDirectory, "renamed")) + }) + } +} + +func TestMoveStorageEntryDoesNotOverwrite(t *testing.T) { + storage := t.TempDir() + if err := os.WriteFile(filepath.Join(storage, "source"), []byte("source"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(storage, "destination"), []byte("destination"), 0o600); err != nil { + t.Fatal(err) + } + root, err := os.OpenRoot(storage) + if err != nil { + t.Fatal(err) + } + defer root.Close() + + if err := moveStorageEntry(root, "source", root, "destination"); !errors.Is(err, errMoveTargetExists) { + t.Fatalf("error = %v, want errMoveTargetExists", err) + } + assertStoredFile(t, filepath.Join(storage, "source"), []byte("source")) + assertStoredFile(t, filepath.Join(storage, "destination"), []byte("destination")) +} + +func TestMoveStorageEntryReportsMissingSource(t *testing.T) { + root, err := os.OpenRoot(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer root.Close() + + if err := moveStorageEntry(root, "missing", root, "destination"); !errors.Is(err, errMoveSourceNotFound) { + t.Fatalf("error = %v, want errMoveSourceNotFound", err) + } +} + +//nolint:gocyclo // This concurrency test counts winners, conflicts, and remaining sources in one assertion. +func TestMoveStorageEntryRaceHasOneWinner(t *testing.T) { + storage := t.TempDir() + const contenders = 8 + for index := 0; index < contenders; index++ { + name := "source-" + string(rune('a'+index)) + if err := os.WriteFile(filepath.Join(storage, name), []byte(name), 0o600); err != nil { + t.Fatal(err) + } + } + + roots := make([]*os.Root, contenders) + t.Cleanup(func() { + for _, root := range roots { + if root != nil { + _ = root.Close() + } + } + }) + for index := range roots { + root, err := os.OpenRoot(storage) + if err != nil { + t.Fatal(err) + } + roots[index] = root + } + + var wait sync.WaitGroup + wait.Add(contenders) + start := make(chan struct{}) + errorsSeen := make(chan error, contenders) + for index := 0; index < contenders; index++ { + go func() { + defer wait.Done() + <-start + name := "source-" + string(rune('a'+index)) + errorsSeen <- moveStorageEntry(roots[index], name, roots[index], "destination") + }() + } + close(start) + wait.Wait() + close(errorsSeen) + + successes := 0 + conflicts := 0 + for err := range errorsSeen { + switch { + case err == nil: + successes++ + case errors.Is(err, errMoveTargetExists): + conflicts++ + default: + t.Fatalf("unexpected move error: %v", err) + } + } + if successes != 1 || conflicts != contenders-1 { + t.Fatalf("successes=%d conflicts=%d", successes, conflicts) + } + if _, err := os.Stat(filepath.Join(storage, "destination")); err != nil { + t.Fatal(err) + } + + remaining := 0 + for index := 0; index < contenders; index++ { + name := "source-" + string(rune('a'+index)) + if _, err := os.Stat(filepath.Join(storage, name)); err == nil { + remaining++ + } else if !os.IsNotExist(err) { + t.Fatal(err) + } + } + if remaining != contenders-1 { + t.Fatalf("remaining sources = %d, want %d", remaining, contenders-1) + } +} + +func TestMoveFileHandlerHTTPContract(t *testing.T) { + gin.SetMode(gin.TestMode) + + t.Run("moves file", func(t *testing.T) { + storage := newMoveHandlerStorage(t) + recorder := serveMove(t, testMoveHandlerDeps(storage), "user/source.txt", "user/archive/result.txt") + assertMoveEnvelope(t, recorder, http.StatusOK, 0) + var envelope struct { + Data string `json:"data"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &envelope); err != nil { + t.Fatal(err) + } + if envelope.Data != "move files successfully" { + t.Fatalf("success data = %q", envelope.Data) + } + assertStoredFile(t, filepath.Join(storage, "users", "alice", "archive", "result.txt"), []byte("source")) + if _, err := os.Stat(filepath.Join(storage, "users", "alice", "source.txt")); !os.IsNotExist(err) { + t.Fatalf("source still exists: %v", err) + } + }) + + t.Run("keeps admin path compatibility", func(t *testing.T) { + storage := t.TempDir() + if err := os.MkdirAll(filepath.Join(storage, "users", "alice"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(storage, "users", "bob", "archive"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(storage, "users", "alice", "source.txt"), []byte("admin"), 0o600); err != nil { + t.Fatal(err) + } + deps := testMoveHandlerDeps(storage) + deps.redirect = func(_ *gin.Context, logicalPath string, _ util.JWTMessage) (string, error) { + if logicalPath == "admin-user" { + return "users", nil + } + return "users/" + strings.TrimPrefix(logicalPath, "admin-user/"), nil + } + recorder := serveMove( + t, + deps, + "admin-user/alice/source.txt", + "admin-user/bob/archive/result.txt", + ) + assertMoveEnvelope(t, recorder, http.StatusOK, 0) + assertStoredFile(t, filepath.Join(storage, "users", "bob", "archive", "result.txt"), []byte("admin")) + }) + + tests := []struct { + name string + source string + destination string + wantStatus int + wantCode int + mutate func(*testing.T, string, *moveFileHandlerDeps) + }{ + { + name: "invalid source", source: testLogicalUserRoot, destination: "user/archive/result.txt", + wantStatus: http.StatusBadRequest, wantCode: 40004, + }, + { + name: "invalid destination", source: "user/source.txt", destination: "../public/result.txt", + wantStatus: http.StatusBadRequest, wantCode: 40004, + }, + { + name: "same path", source: "user/source.txt", destination: "/user//source.txt", + wantStatus: http.StatusBadRequest, wantCode: 40004, + }, + { + name: "destination below source", source: "user/source.txt", destination: "user/source.txt/nested", + wantStatus: http.StatusBadRequest, wantCode: 40004, + }, + { + name: "unauthorized", source: "user/source.txt", destination: "user/archive/result.txt", + wantStatus: http.StatusUnauthorized, wantCode: 40102, + mutate: func(_ *testing.T, _ string, deps *moveFileHandlerDeps) { + deps.authenticate = func(*gin.Context) (util.JWTMessage, error) { + return util.JWTMessage{}, errors.New("invalid token") + } + }, + }, + { + name: "source forbidden", source: "user/source.txt", destination: "user/archive/result.txt", + wantStatus: http.StatusForbidden, wantCode: 40301, + mutate: func(_ *testing.T, _ string, deps *moveFileHandlerDeps) { + deps.permission = func(path string, _ util.JWTMessage, _ *gin.Context) model.FilePermission { + if path == "user/source.txt" { + return model.ReadOnly + } + return model.ReadWrite + } + }, + }, + { + name: "destination forbidden", source: "user/source.txt", destination: "public/result.txt", + wantStatus: http.StatusForbidden, wantCode: 40301, + mutate: func(_ *testing.T, _ string, deps *moveFileHandlerDeps) { + deps.permission = func(path string, _ util.JWTMessage, _ *gin.Context) model.FilePermission { + if strings.HasPrefix(path, "public/") { + return model.ReadOnly + } + return model.ReadWrite + } + }, + }, + { + name: "missing source", source: "user/missing.txt", destination: "user/archive/result.txt", + wantStatus: http.StatusNotFound, wantCode: 40404, + }, + { + name: "missing source parent", source: "user/missing/source.txt", destination: "user/archive/result.txt", + wantStatus: http.StatusNotFound, wantCode: 40404, + }, + { + name: "destination exists", source: "user/source.txt", destination: "user/archive/existing.txt", + wantStatus: http.StatusConflict, wantCode: 40901, + mutate: func(t *testing.T, storage string, _ *moveFileHandlerDeps) { + target := filepath.Join(storage, "users", "alice", "archive", "existing.txt") + if err := os.WriteFile(target, []byte("existing"), 0o600); err != nil { + t.Fatal(err) + } + }, + }, + { + name: "missing destination parent", source: "user/source.txt", destination: "user/missing/result.txt", + wantStatus: http.StatusConflict, wantCode: 40902, + }, + { + name: "destination redirect failure", source: "user/source.txt", destination: "user/archive/result.txt", + wantStatus: http.StatusInternalServerError, wantCode: 50005, + mutate: func(_ *testing.T, storage string, deps *moveFileHandlerDeps) { + redirect := deps.redirect + deps.redirect = func(c *gin.Context, logicalPath string, token util.JWTMessage) (string, error) { + if logicalPath == "user/archive/result.txt" { + return "", errors.New(filepath.Join(storage, "private-path")) + } + return redirect(c, logicalPath, token) + } + deps.openTarget = func(string, string, string) (*os.Root, string, error) { + t.Fatal("openTarget called after destination redirect failed") + return nil, "", nil + } + deps.move = func(*os.Root, string, *os.Root, string) error { + t.Fatal("move called after destination redirect failed") + return nil + } + }, + }, + { + name: "filesystem failure", source: "user/source.txt", destination: "user/archive/result.txt", + wantStatus: http.StatusInternalServerError, wantCode: 50005, + mutate: func(_ *testing.T, _ string, deps *moveFileHandlerDeps) { + deps.move = func(*os.Root, string, *os.Root, string) error { + return errors.New("disk unavailable") + } + }, + }, + { + name: "source parent access failure", source: "user/source.txt", destination: "user/archive/result.txt", + wantStatus: http.StatusInternalServerError, wantCode: 50005, + mutate: func(_ *testing.T, _ string, deps *moveFileHandlerDeps) { + deps.openTarget = func(string, string, string) (*os.Root, string, error) { + return nil, "", &uploadParentAccessError{cause: os.ErrPermission} + } + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + storage := newMoveHandlerStorage(t) + deps := testMoveHandlerDeps(storage) + if test.mutate != nil { + test.mutate(t, storage, &deps) + } + recorder := serveMove(t, deps, test.source, test.destination) + assertMoveEnvelope(t, recorder, test.wantStatus, test.wantCode) + if strings.Contains(recorder.Body.String(), storage) { + t.Fatalf("response leaked physical storage path: %s", recorder.Body.String()) + } + }) + } +} + +func TestMoveFileHandlerRejectsInvalidRequestBody(t *testing.T) { + gin.SetMode(gin.TestMode) + for _, body := range []string{`{`, `{}`} { + storage := newMoveHandlerStorage(t) + deps := testMoveHandlerDeps(storage) + router := gin.New() + router.POST("/move/*path", func(c *gin.Context) { + moveFileWithDeps(c, deps) + }) + request := httptest.NewRequest(http.MethodPost, "/move/user/source.txt", strings.NewReader(body)) + request.Header.Set("Content-Type", "application/json") + recorder := httptest.NewRecorder() + router.ServeHTTP(recorder, request) + assertMoveEnvelope(t, recorder, http.StatusBadRequest, 40001) + } +} + +func TestRegisterRoutesServesMoveEndpoint(t *testing.T) { + gin.SetMode(gin.TestMode) + router := gin.New() + RegisterRoutes(router) + + request := httptest.NewRequest( + http.MethodPost, + "/api/ss/move/user/source.txt", + bytes.NewBufferString(`{"dst":"user/result.txt"}`), + ) + request.Header.Set("Content-Type", "application/json") + recorder := httptest.NewRecorder() + router.ServeHTTP(recorder, request) + assertMoveEnvelope(t, recorder, http.StatusUnauthorized, 40102) +} + +func TestRegisterRoutesServesMKCOLEndpoint(t *testing.T) { + gin.SetMode(gin.TestMode) + router := gin.New() + RegisterRoutes(router) + + request := httptest.NewRequest("MKCOL", "/api/ss/user/new-directory", http.NoBody) + recorder := httptest.NewRecorder() + router.ServeHTTP(recorder, request) + assertMoveEnvelope(t, recorder, http.StatusUnauthorized, 40102) +} + +func newMoveHandlerStorage(t *testing.T) string { + t.Helper() + storage := t.TempDir() + userRoot := filepath.Join(storage, "users", "alice") + if err := os.MkdirAll(filepath.Join(userRoot, "archive"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(userRoot, "source.txt"), []byte("source"), 0o600); err != nil { + t.Fatal(err) + } + return storage +} + +func testMoveHandlerDeps(storage string) moveFileHandlerDeps { + return moveFileHandlerDeps{ + authenticate: func(*gin.Context) (util.JWTMessage, error) { + return util.JWTMessage{}, nil + }, + permission: func(string, util.JWTMessage, *gin.Context) model.FilePermission { + return model.ReadWrite + }, + redirect: func(_ *gin.Context, logicalPath string, _ util.JWTMessage) (string, error) { + if logicalPath == testLogicalUserRoot { + return testRealUserRoot, nil + } + if strings.HasPrefix(logicalPath, testLogicalUserRoot+"/") { + return testRealUserRoot + "/" + strings.TrimPrefix(logicalPath, testLogicalUserRoot+"/"), nil + } + if logicalPath == "public" { + return "public", nil + } + return logicalPath, nil + }, + openTarget: openUploadTarget, + move: moveStorageEntry, + storageRoot: storage, + } +} + +func serveMove( + t *testing.T, + deps moveFileHandlerDeps, + sourcePath string, + destinationPath string, +) *httptest.ResponseRecorder { + t.Helper() + body, err := json.Marshal(MoveFileReq{Dst: destinationPath}) + if err != nil { + t.Fatal(err) + } + router := gin.New() + router.POST("/move/*path", func(c *gin.Context) { + moveFileWithDeps(c, deps) + }) + request := httptest.NewRequest(http.MethodPost, "/move/"+sourcePath, bytes.NewReader(body)) + request.Header.Set("Content-Type", "application/json") + recorder := httptest.NewRecorder() + router.ServeHTTP(recorder, request) + return recorder +} + +func assertMoveEnvelope(t *testing.T, recorder *httptest.ResponseRecorder, wantStatus, wantCode int) { + t.Helper() + if recorder.Code != wantStatus { + t.Fatalf("status = %d, want %d; body=%s", recorder.Code, wantStatus, recorder.Body.String()) + } + var envelope struct { + Code int `json:"code"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &envelope); err != nil { + t.Fatalf("decode response: %v; body=%s", err, recorder.Body.String()) + } + if envelope.Code != wantCode { + t.Fatalf("code = %d, want %d; body=%s", envelope.Code, wantCode, recorder.Body.String()) + } +} diff --git a/backend/internal/storage/remove.go b/backend/internal/storage/remove.go new file mode 100644 index 000000000..2f6decd6f --- /dev/null +++ b/backend/internal/storage/remove.go @@ -0,0 +1,260 @@ +package storage + +import ( + "errors" + "net/http" + "os" + "path/filepath" + "strings" + "syscall" + "unicode" + + "github.com/gin-gonic/gin" + + "github.com/raids-lab/crater/dao/model" + "github.com/raids-lab/crater/internal/bizerr" + "github.com/raids-lab/crater/internal/resputil" + "github.com/raids-lab/crater/internal/util" +) + +var ( + errRemoveTargetNotFound = errors.New("remove target does not exist") + errRemoveRecursiveRequired = errors.New("recursive removal is required for a directory") + errRemoveTargetChanged = errors.New("remove target changed during the operation") + errRemoveCrossDevice = errors.New("recursive removal cannot cross a filesystem boundary") + errRemoveOperationUnsupported = errors.New("safe removal is unsupported on this platform") +) + +type removeFileResponse struct { + RemotePath string `json:"remote_path"` + Recursive bool `json:"recursive"` +} + +type removeFileHandlerDeps struct { + authenticate func(*gin.Context) (util.JWTMessage, error) + permission func(string, util.JWTMessage, *gin.Context) model.FilePermission + redirect func(*gin.Context, string, util.JWTMessage) (string, error) + openTarget func(string, string, string) (*os.Root, string, error) + remove func(*os.Root, string, bool) error + storageRoot string +} + +func defaultRemoveFileHandlerDeps() removeFileHandlerDeps { + return removeFileHandlerDeps{ + authenticate: CheckJWTToken, + permission: GetPermission, + redirect: Redirect, + openTarget: openUploadTarget, + remove: removeStorageEntry, + storageRoot: storageRootDir, + } +} + +// RemoveFile deletes exactly one ordinary-user storage path. It is separate +// from the legacy /delete route so callers must explicitly opt in before a +// directory tree can be removed. +func RemoveFile(c *gin.Context) { + removeFileWithDeps(c, defaultRemoveFileHandlerDeps()) +} + +func removeFileWithDeps(c *gin.Context, deps removeFileHandlerDeps) { + token, err := deps.authenticate(c) + if err != nil { + resputil.HandleError(c, bizerr.Auth.TokenInvalid.New("invalid token")) + return + } + + recursiveValues, present := c.Request.URL.Query()["recursive"] + if !present { + resputil.HandleError(c, bizerr.BadRequest.MissingParameter.New("recursive is required")) + return + } + if len(recursiveValues) != 1 { + resputil.HandleError(c, bizerr.BadRequest.ParameterError.New("recursive must be true or false")) + return + } + recursive, err := parseRemoveRecursive(recursiveValues[0]) + if err != nil { + resputil.HandleError(c, bizerr.BadRequest.ParameterError.New("recursive must be true or false")) + return + } + + logicalPath, err := normalizeRemoveLogicalPath(c.Param("path")) + if err != nil { + resputil.HandleError(c, bizerr.BadRequest.ParameterError.New("invalid remote path")) + return + } + if permission := deps.permission(logicalPath, token, c); permission != model.ReadWrite { + resputil.HandleError(c, bizerr.Forbidden.PermissionDenied.New("write permission is required")) + return + } + + realPath, err := deps.redirect(c, logicalPath, token) + if err != nil { + resputil.HandleError(c, bizerr.Internal.FileSystemError.Wrap(err, "failed to resolve remove target")) + return + } + logicalRoot := strings.SplitN(logicalPath, "/", 2)[0] + realRoot, err := deps.redirect(c, logicalRoot, token) + if err != nil { + resputil.HandleError(c, bizerr.Internal.FileSystemError.Wrap(err, "failed to resolve remove target")) + return + } + + parent, targetName, err := deps.openTarget(deps.storageRoot, realRoot, realPath) + if err != nil { + handleRemoveTargetOpenError(c, err) + return + } + defer parent.Close() + + if err := deps.remove(parent, targetName, recursive); err != nil { + switch { + case errors.Is(err, errRemoveTargetNotFound): + resputil.HandleError(c, bizerr.NotFound.StorageResourceNotFound.New("remote path does not exist")) + case errors.Is(err, errRemoveRecursiveRequired): + resputil.HandleError(c, bizerr.Conflict.ResourceStatusError.New("directory removal requires recursive=true")) + case errors.Is(err, errRemoveTargetChanged): + resputil.HandleError(c, bizerr.Conflict.ResourceStatusError.New("remote path changed during removal")) + case errors.Is(err, errRemoveCrossDevice): + resputil.HandleError(c, bizerr.Conflict.ResourceStatusError.New("remote path crosses a filesystem boundary")) + default: + resputil.HandleError(c, bizerr.Internal.FileSystemError.Wrap(err, "failed to remove storage entry")) + } + return + } + + c.JSON(http.StatusOK, resputil.Response[removeFileResponse]{ + Code: resputil.OK, + Data: removeFileResponse{ + RemotePath: logicalPath, + Recursive: recursive, + }, + Message: "", + }) +} + +func handleRemoveTargetOpenError(c *gin.Context, err error) { + if errors.Is(err, errUploadParentInvalid) { + if isUploadParentMissing(err) { + resputil.HandleError(c, bizerr.NotFound.StorageResourceNotFound.New("remote path does not exist")) + return + } + if isUploadParentInfrastructureFailure(err) { + resputil.HandleError(c, bizerr.Internal.FileSystemError.Wrap(err, "failed to access storage")) + return + } + resputil.HandleError(c, bizerr.Conflict.ResourceStatusError.New("remote path parent is unavailable")) + return + } + resputil.HandleError(c, bizerr.Internal.FileSystemError.Wrap(err, "failed to access storage")) +} + +func parseRemoveRecursive(raw string) (bool, error) { + switch raw { + case "false": + return false, nil + case "true": + return true, nil + default: + return false, errors.New("invalid recursive value") + } +} + +func normalizeRemoveLogicalPath(raw string) (string, error) { + if strings.ContainsRune(raw, '\\') { + return "", errors.New("backslashes are not allowed") + } + for _, character := range raw { + if unicode.IsControl(character) { + return "", errors.New("control characters are not allowed") + } + } + + raw = strings.TrimPrefix(raw, "/") + if raw == "" || strings.HasPrefix(raw, "/") || strings.HasSuffix(raw, "/") { + return "", errors.New("empty path segments are not allowed") + } + segments := strings.Split(raw, "/") + if len(segments) < 2 { + return "", errors.New("a path below a logical root is required") + } + for _, segment := range segments { + if segment == "" || segment == "." || segment == parentPathSegment { + return "", errors.New("ambiguous path segments are not allowed") + } + } + switch segments[0] { + case model.UserPath, model.PublicPath, model.AccountPath: + default: + return "", errors.New("invalid logical root") + } + return strings.Join(segments, "/"), nil +} + +type removeStorageEntryDeps struct { + lstat func(*os.Root, string) (os.FileInfo, error) + unlinkNonDirectory func(*os.Root, string) error + removeDirectory func(*os.Root, string) error +} + +func defaultRemoveStorageEntryDeps() removeStorageEntryDeps { + return removeStorageEntryDeps{ + lstat: func(parent *os.Root, name string) (os.FileInfo, error) { + return parent.Lstat(name) + }, + unlinkNonDirectory: removeStorageNonDirectory, + removeDirectory: removeStorageDirectoryRecursive, + } +} + +func removeStorageEntry(parent *os.Root, name string, recursive bool) error { + return removeStorageEntryWithDeps(parent, name, recursive, defaultRemoveStorageEntryDeps()) +} + +//nolint:gocyclo // Type-race classification must never upgrade a non-directory unlink into recursive removal. +func removeStorageEntryWithDeps( + parent *os.Root, + name string, + recursive bool, + deps removeStorageEntryDeps, +) error { + if parent == nil || name == "" || name == "." || name == parentPathSegment || + filepath.Base(name) != name { + return errUploadParentInvalid + } + + info, err := deps.lstat(parent, name) + if err != nil { + if os.IsNotExist(err) || errors.Is(err, syscall.ENOTDIR) { + return errRemoveTargetNotFound + } + return err + } + if info.IsDir() { + if !recursive { + return errRemoveRecursiveRequired + } + return deps.removeDirectory(parent, name) + } + + err = deps.unlinkNonDirectory(parent, name) + if err == nil { + return nil + } + if os.IsNotExist(err) || errors.Is(err, syscall.ENOTDIR) { + return errRemoveTargetNotFound + } + if errors.Is(err, syscall.EISDIR) || errors.Is(err, syscall.EPERM) { + current, statErr := deps.lstat(parent, name) + switch { + case statErr == nil && current.IsDir(): + return errRemoveTargetChanged + case os.IsNotExist(statErr), errors.Is(statErr, syscall.ENOTDIR): + return errRemoveTargetNotFound + case statErr != nil: + return statErr + } + } + return err +} diff --git a/backend/internal/storage/remove_other.go b/backend/internal/storage/remove_other.go new file mode 100644 index 000000000..4062e96db --- /dev/null +++ b/backend/internal/storage/remove_other.go @@ -0,0 +1,13 @@ +//go:build !darwin && !dragonfly && !freebsd && !linux && !netbsd && !openbsd + +package storage + +import "os" + +func removeStorageNonDirectory(*os.Root, string) error { + return errRemoveOperationUnsupported +} + +func removeStorageDirectoryRecursive(*os.Root, string) error { + return errRemoveOperationUnsupported +} diff --git a/backend/internal/storage/remove_other_test.go b/backend/internal/storage/remove_other_test.go new file mode 100644 index 000000000..9c89ab3b2 --- /dev/null +++ b/backend/internal/storage/remove_other_test.go @@ -0,0 +1,30 @@ +//go:build !darwin && !dragonfly && !freebsd && !linux && !netbsd && !openbsd + +package storage + +import ( + "errors" + "os" + "path/filepath" + "testing" +) + +func TestRemoveStorageEntryFailsClosedOnUnsupportedPlatform(t *testing.T) { + storage := t.TempDir() + path := filepath.Join(storage, "file") + if err := os.WriteFile(path, []byte("keep"), 0o600); err != nil { + t.Fatal(err) + } + root, err := os.OpenRoot(storage) + if err != nil { + t.Fatal(err) + } + defer root.Close() + + if err := removeStorageEntry(root, "file", false); !errors.Is(err, errRemoveOperationUnsupported) { + t.Fatalf("error = %v, want errRemoveOperationUnsupported", err) + } + if _, err := os.Stat(path); err != nil { + t.Fatalf("file changed on unsupported platform: %v", err) + } +} diff --git a/backend/internal/storage/remove_test.go b/backend/internal/storage/remove_test.go new file mode 100644 index 000000000..fc12dcb40 --- /dev/null +++ b/backend/internal/storage/remove_test.go @@ -0,0 +1,499 @@ +package storage + +import ( + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "syscall" + "testing" + + "github.com/gin-gonic/gin" + + "github.com/raids-lab/crater/dao/model" + "github.com/raids-lab/crater/internal/util" +) + +func TestParseRemoveRecursiveRequiresExplicitBoolean(t *testing.T) { + for _, test := range []struct { + raw string + want bool + ok bool + }{ + {raw: "false", want: false, ok: true}, + {raw: "true", want: true, ok: true}, + {raw: "", ok: false}, + {raw: "TRUE", ok: false}, + {raw: "False", ok: false}, + {raw: "0", ok: false}, + {raw: "1", ok: false}, + } { + t.Run(test.raw, func(t *testing.T) { + got, err := parseRemoveRecursive(test.raw) + if test.ok { + if err != nil || got != test.want { + t.Fatalf("parseRemoveRecursive(%q) = %v, %v", test.raw, got, err) + } + return + } + if err == nil { + t.Fatalf("parseRemoveRecursive(%q) unexpectedly succeeded", test.raw) + } + }) + } +} + +func TestNormalizeRemoveLogicalPathIsStrictAndUserScoped(t *testing.T) { + for _, test := range []struct { + raw string + want string + }{ + {raw: "user/file.txt", want: "user/file.txt"}, + {raw: "/user/folder/file.txt", want: "user/folder/file.txt"}, + {raw: "public/shared", want: "public/shared"}, + {raw: "account/project", want: "account/project"}, + } { + t.Run("accepts "+test.raw, func(t *testing.T) { + got, err := normalizeRemoveLogicalPath(test.raw) + if err != nil || got != test.want { + t.Fatalf("normalizeRemoveLogicalPath(%q) = %q, %v", test.raw, got, err) + } + }) + } + + for _, raw := range []string{ + "", + "/", + "user", + "/user", + "user/", + "user//file.txt", + "/user//file.txt", + "//user/file.txt", + "user/./file.txt", + "user/../file.txt", + "user/file.txt/..", + "admin-user/alice/file.txt", + "admin-public/file.txt", + "admin-account/team/file.txt", + "dataset/file.txt", + "model/file.txt", + "unknown/file.txt", + `user\file.txt`, + "user/control\ncharacter", + } { + t.Run("rejects "+raw, func(t *testing.T) { + if got, err := normalizeRemoveLogicalPath(raw); err == nil { + t.Fatalf("normalizeRemoveLogicalPath(%q) = %q, want error", raw, got) + } + }) + } +} + +func TestRemoveStorageEntryRequiresRecursiveForDirectory(t *testing.T) { + storage := t.TempDir() + target := filepath.Join(storage, "directory") + if err := os.Mkdir(target, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(target, "nested.txt"), []byte("nested"), 0o600); err != nil { + t.Fatal(err) + } + root, err := os.OpenRoot(storage) + if err != nil { + t.Fatal(err) + } + defer root.Close() + + if err := removeStorageEntry(root, "directory", false); !errors.Is(err, errRemoveRecursiveRequired) { + t.Fatalf("error = %v, want errRemoveRecursiveRequired", err) + } + if _, err := os.Stat(filepath.Join(target, "nested.txt")); err != nil { + t.Fatalf("directory changed after non-recursive refusal: %v", err) + } + + if err := removeStorageEntry(root, "directory", true); err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(target); !os.IsNotExist(err) { + t.Fatalf("directory still exists: %v", err) + } +} + +func TestRemoveStorageEntryReportsMissingAndRejectsInvalidNames(t *testing.T) { + root, err := os.OpenRoot(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer root.Close() + + if err := removeStorageEntry(root, "missing", false); !errors.Is(err, errRemoveTargetNotFound) { + t.Fatalf("missing target error = %v", err) + } + for _, name := range []string{"", ".", "..", "nested/file"} { + if err := removeStorageEntry(root, name, false); !errors.Is(err, errUploadParentInvalid) { + t.Fatalf("name %q error = %v, want errUploadParentInvalid", name, err) + } + } +} + +func TestRemoveStorageEntryDoesNotUpgradeFileRaceToRecursiveDelete(t *testing.T) { + storage := t.TempDir() + filePath := filepath.Join(storage, "file") + directoryPath := filepath.Join(storage, "directory") + if err := os.WriteFile(filePath, []byte("file"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(directoryPath, 0o700); err != nil { + t.Fatal(err) + } + fileInfo, err := os.Lstat(filePath) + if err != nil { + t.Fatal(err) + } + directoryInfo, err := os.Lstat(directoryPath) + if err != nil { + t.Fatal(err) + } + root, err := os.OpenRoot(storage) + if err != nil { + t.Fatal(err) + } + defer root.Close() + + lstatCalls := 0 + recursiveCalled := false + deps := removeStorageEntryDeps{ + lstat: func(*os.Root, string) (os.FileInfo, error) { + lstatCalls++ + if lstatCalls == 1 { + return fileInfo, nil + } + return directoryInfo, nil + }, + unlinkNonDirectory: func(*os.Root, string) error { + return syscall.EISDIR + }, + removeDirectory: func(*os.Root, string) error { + recursiveCalled = true + return nil + }, + } + err = removeStorageEntryWithDeps(root, "target", true, deps) + if !errors.Is(err, errRemoveTargetChanged) { + t.Fatalf("error = %v, want errRemoveTargetChanged", err) + } + if recursiveCalled { + t.Fatal("file-to-directory race was upgraded to recursive removal") + } +} + +func TestRemoveStorageEntryPreservesRegularFilePermissionError(t *testing.T) { + storage := t.TempDir() + path := filepath.Join(storage, "file") + if err := os.WriteFile(path, []byte("file"), 0o600); err != nil { + t.Fatal(err) + } + fileInfo, err := os.Lstat(path) + if err != nil { + t.Fatal(err) + } + root, err := os.OpenRoot(storage) + if err != nil { + t.Fatal(err) + } + defer root.Close() + + deps := removeStorageEntryDeps{ + lstat: func(*os.Root, string) (os.FileInfo, error) { + return fileInfo, nil + }, + unlinkNonDirectory: func(*os.Root, string) error { + return syscall.EPERM + }, + removeDirectory: func(*os.Root, string) error { + t.Fatal("recursive removal called for a regular file") + return nil + }, + } + if err := removeStorageEntryWithDeps(root, "file", true, deps); !errors.Is(err, syscall.EPERM) { + t.Fatalf("error = %v, want EPERM", err) + } +} + +func TestRemoveFileHandlerHTTPContract(t *testing.T) { + gin.SetMode(gin.TestMode) + + t.Run("removes file and returns metadata", func(t *testing.T) { + storage := newRemoveHandlerStorage(t) + recorder := serveRemove(t, testRemoveHandlerDeps(storage), "user/file.txt", "recursive=false") + assertRemoveEnvelope(t, recorder, http.StatusOK, 0) + var envelope struct { + Data removeFileResponse `json:"data"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &envelope); err != nil { + t.Fatal(err) + } + if envelope.Data.RemotePath != "user/file.txt" || envelope.Data.Recursive { + t.Fatalf("success data = %+v", envelope.Data) + } + if _, err := os.Lstat(filepath.Join(storage, "users", "alice", "file.txt")); !os.IsNotExist(err) { + t.Fatalf("file still exists: %v", err) + } + }) + + t.Run("removes directory only with recursive opt in", func(t *testing.T) { + storage := newRemoveHandlerStorage(t) + recorder := serveRemove(t, testRemoveHandlerDeps(storage), "user/tree", "recursive=true") + assertRemoveEnvelope(t, recorder, http.StatusOK, 0) + if _, err := os.Lstat(filepath.Join(storage, "users", "alice", "tree")); !os.IsNotExist(err) { + t.Fatalf("directory still exists: %v", err) + } + }) + + tests := []struct { + name string + path string + query string + wantStatus int + wantCode int + mutate func(*testing.T, string, *removeFileHandlerDeps) + assert func(*testing.T, string) + }{ + { + name: "unauthorized", path: "user/file.txt", query: "recursive=false", + wantStatus: http.StatusUnauthorized, wantCode: 40102, + mutate: func(_ *testing.T, _ string, deps *removeFileHandlerDeps) { + deps.authenticate = func(*gin.Context) (util.JWTMessage, error) { + return util.JWTMessage{}, errors.New("invalid token") + } + }, + }, + { + name: "missing recursive", path: "user/file.txt", query: "", + wantStatus: http.StatusBadRequest, wantCode: 40003, + }, + { + name: "ambiguous recursive", path: "user/file.txt", query: "recursive=false&recursive=true", + wantStatus: http.StatusBadRequest, wantCode: 40004, + }, + { + name: "invalid recursive", path: "user/file.txt", query: "recursive=TRUE", + wantStatus: http.StatusBadRequest, wantCode: 40004, + }, + { + name: "empty recursive", path: "user/file.txt", query: "recursive=", + wantStatus: http.StatusBadRequest, wantCode: 40004, + }, + { + name: "logical root target", path: "user", query: "recursive=true", + wantStatus: http.StatusBadRequest, wantCode: 40004, + }, + { + name: "reserved admin path", path: "admin-user/alice/file.txt", query: "recursive=false", + wantStatus: http.StatusBadRequest, wantCode: 40004, + }, + { + name: "forbidden", path: "user/file.txt", query: "recursive=false", + wantStatus: http.StatusForbidden, wantCode: 40301, + mutate: func(_ *testing.T, _ string, deps *removeFileHandlerDeps) { + deps.permission = func(string, util.JWTMessage, *gin.Context) model.FilePermission { + return model.ReadOnly + } + }, + }, + { + name: "missing target", path: "user/missing.txt", query: "recursive=false", + wantStatus: http.StatusNotFound, wantCode: 40404, + }, + { + name: "missing parent", path: "user/missing/file.txt", query: "recursive=false", + wantStatus: http.StatusNotFound, wantCode: 40404, + }, + { + name: "directory without recursive", path: "user/tree", query: "recursive=false", + wantStatus: http.StatusConflict, wantCode: 40902, + assert: func(t *testing.T, storage string) { + t.Helper() + if _, err := os.Stat(filepath.Join(storage, "users", "alice", "tree", "nested.txt")); err != nil { + t.Fatalf("directory changed after refusal: %v", err) + } + }, + }, + { + name: "redirect failure", path: "user/file.txt", query: "recursive=false", + wantStatus: http.StatusInternalServerError, wantCode: 50005, + mutate: func(_ *testing.T, storage string, deps *removeFileHandlerDeps) { + deps.redirect = func(*gin.Context, string, util.JWTMessage) (string, error) { + return "", errors.New(filepath.Join(storage, "private-path")) + } + }, + }, + { + name: "parent access failure", path: "user/file.txt", query: "recursive=false", + wantStatus: http.StatusInternalServerError, wantCode: 50005, + mutate: func(_ *testing.T, _ string, deps *removeFileHandlerDeps) { + deps.openTarget = func(string, string, string) (*os.Root, string, error) { + return nil, "", &uploadParentAccessError{cause: os.ErrPermission} + } + }, + }, + { + name: "target changed", path: "user/file.txt", query: "recursive=false", + wantStatus: http.StatusConflict, wantCode: 40902, + mutate: func(_ *testing.T, _ string, deps *removeFileHandlerDeps) { + deps.remove = func(*os.Root, string, bool) error { + return errRemoveTargetChanged + } + }, + }, + { + name: "cross device", path: "user/tree", query: "recursive=true", + wantStatus: http.StatusConflict, wantCode: 40902, + mutate: func(_ *testing.T, _ string, deps *removeFileHandlerDeps) { + deps.remove = func(*os.Root, string, bool) error { + return errRemoveCrossDevice + } + }, + }, + { + name: "safe operation unsupported", path: "user/file.txt", query: "recursive=false", + wantStatus: http.StatusInternalServerError, wantCode: 50005, + mutate: func(_ *testing.T, _ string, deps *removeFileHandlerDeps) { + deps.remove = func(*os.Root, string, bool) error { + return errRemoveOperationUnsupported + } + }, + }, + { + name: "filesystem failure", path: "user/file.txt", query: "recursive=false", + wantStatus: http.StatusInternalServerError, wantCode: 50005, + mutate: func(_ *testing.T, _ string, deps *removeFileHandlerDeps) { + deps.remove = func(*os.Root, string, bool) error { + return errors.New("disk unavailable") + } + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + storage := newRemoveHandlerStorage(t) + deps := testRemoveHandlerDeps(storage) + if test.mutate != nil { + test.mutate(t, storage, &deps) + } + recorder := serveRemove(t, deps, test.path, test.query) + assertRemoveEnvelope(t, recorder, test.wantStatus, test.wantCode) + if strings.Contains(recorder.Body.String(), storage) { + t.Fatalf("response leaked physical storage path: %s", recorder.Body.String()) + } + if test.assert != nil { + test.assert(t, storage) + } + }) + } +} + +func TestRegisterRoutesServesSafeRemoveAndPreservesLegacyDelete(t *testing.T) { + gin.SetMode(gin.TestMode) + router := gin.New() + RegisterRoutes(router) + + for _, path := range []string{ + "/api/ss/files/user/file.txt?recursive=false", + "/api/ss/delete/user/file.txt", + } { + request := httptest.NewRequest(http.MethodDelete, path, http.NoBody) + recorder := httptest.NewRecorder() + router.ServeHTTP(recorder, request) + if recorder.Code == http.StatusNotFound { + t.Fatalf("DELETE %s was not registered", path) + } + } + + request := httptest.NewRequest( + http.MethodDelete, + "/api/ss/files/user/file.txt?recursive=false", + http.NoBody, + ) + recorder := httptest.NewRecorder() + router.ServeHTTP(recorder, request) + assertRemoveEnvelope(t, recorder, http.StatusUnauthorized, 40102) +} + +func newRemoveHandlerStorage(t *testing.T) string { + t.Helper() + storage := t.TempDir() + userRoot := filepath.Join(storage, "users", "alice") + if err := os.MkdirAll(filepath.Join(userRoot, "tree"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(userRoot, "file.txt"), []byte("file"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(userRoot, "tree", "nested.txt"), []byte("nested"), 0o600); err != nil { + t.Fatal(err) + } + return storage +} + +func testRemoveHandlerDeps(storage string) removeFileHandlerDeps { + return removeFileHandlerDeps{ + authenticate: func(*gin.Context) (util.JWTMessage, error) { + return util.JWTMessage{}, nil + }, + permission: func(string, util.JWTMessage, *gin.Context) model.FilePermission { + return model.ReadWrite + }, + redirect: func(_ *gin.Context, logicalPath string, _ util.JWTMessage) (string, error) { + if logicalPath == "user" { + return "users/alice", nil + } + return "users/alice/" + strings.TrimPrefix(logicalPath, "user/"), nil + }, + openTarget: openUploadTarget, + remove: removeStorageEntry, + storageRoot: storage, + } +} + +func serveRemove( + t *testing.T, + deps removeFileHandlerDeps, + logicalPath string, + rawQuery string, +) *httptest.ResponseRecorder { + t.Helper() + router := gin.New() + router.DELETE("/remove/*path", func(c *gin.Context) { + removeFileWithDeps(c, deps) + }) + url := "/remove/" + logicalPath + if rawQuery != "" { + url += "?" + rawQuery + } + request := httptest.NewRequest(http.MethodDelete, url, http.NoBody) + recorder := httptest.NewRecorder() + router.ServeHTTP(recorder, request) + return recorder +} + +func assertRemoveEnvelope(t *testing.T, recorder *httptest.ResponseRecorder, wantStatus, wantCode int) { + t.Helper() + if recorder.Code != wantStatus { + t.Fatalf("status = %d, want %d; body=%s", recorder.Code, wantStatus, recorder.Body.String()) + } + var envelope struct { + Code int `json:"code"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &envelope); err != nil { + t.Fatalf("decode response: %v; body=%s", err, recorder.Body.String()) + } + if envelope.Code != wantCode { + t.Fatalf("code = %d, want %d; body=%s", envelope.Code, wantCode, recorder.Body.String()) + } +} diff --git a/backend/internal/storage/remove_unix.go b/backend/internal/storage/remove_unix.go new file mode 100644 index 000000000..06673139a --- /dev/null +++ b/backend/internal/storage/remove_unix.go @@ -0,0 +1,242 @@ +//go:build darwin || dragonfly || freebsd || linux || netbsd || openbsd + +package storage + +import ( + "errors" + "os" + "syscall" + + "golang.org/x/sys/unix" +) + +func removeStorageNonDirectory(parent *os.Root, name string) error { + parentDirectory, err := parent.Open(".") + if err != nil { + return err + } + defer parentDirectory.Close() + + return unlinkatRetry(int(parentDirectory.Fd()), name, 0) +} + +func removeStorageDirectoryRecursive(parent *os.Root, name string) error { + parentDirectory, err := parent.Open(".") + if err != nil { + return err + } + defer parentDirectory.Close() + + var parentStat unix.Stat_t + if err := fstatRetry(int(parentDirectory.Fd()), &parentStat); err != nil { + return err + } + + targetDirectory, err := openDirectoryAt(int(parentDirectory.Fd()), name) + if err != nil { + return classifyRecursiveOpenError(err) + } + target := os.NewFile(uintptr(targetDirectory), name) + if target == nil { + _ = unix.Close(targetDirectory) + return errors.New("failed to create directory handle") + } + + var targetStat unix.Stat_t + if err := fstatRetry(targetDirectory, &targetStat); err != nil { + _ = target.Close() + return err + } + if err := requireRecursiveStorageDevice(&parentStat, &targetStat); err != nil { + _ = target.Close() + return err + } + + if err := removeDirectoryContents(target, &parentStat); err != nil { + _ = target.Close() + return err + } + if err := verifyDirectoryIdentityAt(int(parentDirectory.Fd()), name, &targetStat); err != nil { + _ = target.Close() + return err + } + if err := target.Close(); err != nil { + return err + } + if err := unlinkatRetry(int(parentDirectory.Fd()), name, unix.AT_REMOVEDIR); err != nil { + return classifyRecursiveUnlinkError(err) + } + return nil +} + +func removeDirectoryContents(directory *os.File, rootDevice *unix.Stat_t) error { + names, err := directory.Readdirnames(-1) + if err != nil { + return err + } + directoryFD := int(directory.Fd()) + for _, name := range names { + if name == "" || name == "." || name == parentPathSegment { + return errRemoveTargetChanged + } + + var entryStat unix.Stat_t + err := fstatatRetry(directoryFD, name, &entryStat, unix.AT_SYMLINK_NOFOLLOW) + if errors.Is(err, unix.ENOENT) { + continue + } + if err != nil { + return err + } + if err := requireRecursiveStorageDevice(rootDevice, &entryStat); err != nil { + return err + } + + if entryStat.Mode&unix.S_IFMT == unix.S_IFDIR { + if err := removeDirectoryEntry(directoryFD, name, rootDevice); errors.Is(err, errRemoveTargetNotFound) { + continue + } else if err != nil { + return err + } + continue + } + if err := unlinkatRetry(directoryFD, name, 0); err != nil { + switch { + case errors.Is(err, unix.ENOENT): + continue + case errors.Is(err, unix.EISDIR), errors.Is(err, unix.EPERM): + return errRemoveTargetChanged + default: + return err + } + } + } + return nil +} + +func removeDirectoryEntry(parentFD int, name string, rootDevice *unix.Stat_t) error { + childFD, err := openDirectoryAt(parentFD, name) + if err != nil { + return classifyRecursiveOpenError(err) + } + child := os.NewFile(uintptr(childFD), name) + if child == nil { + _ = unix.Close(childFD) + return errors.New("failed to create directory handle") + } + + var childStat unix.Stat_t + if err := fstatRetry(childFD, &childStat); err != nil { + _ = child.Close() + return err + } + if err := requireRecursiveStorageDevice(rootDevice, &childStat); err != nil { + _ = child.Close() + return err + } + if err := removeDirectoryContents(child, rootDevice); err != nil { + _ = child.Close() + return err + } + if err := verifyDirectoryIdentityAt(parentFD, name, &childStat); err != nil { + _ = child.Close() + return err + } + if err := child.Close(); err != nil { + return err + } + if err := unlinkatRetry(parentFD, name, unix.AT_REMOVEDIR); err != nil { + return classifyRecursiveUnlinkError(err) + } + return nil +} + +func openDirectoryAt(parentFD int, name string) (int, error) { + for { + directory, err := unix.Openat( + parentFD, + name, + unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, + 0, + ) + if !errors.Is(err, unix.EINTR) { + return directory, err + } + } +} + +func fstatRetry(fd int, stat *unix.Stat_t) error { + for { + err := unix.Fstat(fd, stat) + if !errors.Is(err, unix.EINTR) { + return err + } + } +} + +func fstatatRetry(fd int, name string, stat *unix.Stat_t, flags int) error { + for { + err := unix.Fstatat(fd, name, stat, flags) + if !errors.Is(err, unix.EINTR) { + return err + } + } +} + +func unlinkatRetry(fd int, name string, flags int) error { + for { + err := unix.Unlinkat(fd, name, flags) + if !errors.Is(err, unix.EINTR) { + return err + } + } +} + +func verifyDirectoryIdentityAt(parentFD int, name string, opened *unix.Stat_t) error { + var current unix.Stat_t + if err := fstatatRetry(parentFD, name, ¤t, unix.AT_SYMLINK_NOFOLLOW); err != nil { + if errors.Is(err, unix.ENOENT) { + return errRemoveTargetNotFound + } + return err + } + if current.Mode&unix.S_IFMT != unix.S_IFDIR || + current.Dev != opened.Dev || + current.Ino != opened.Ino { + return errRemoveTargetChanged + } + return nil +} + +func requireRecursiveStorageDevice(rootDevice, entryDevice *unix.Stat_t) error { + if rootDevice == nil || entryDevice == nil || entryDevice.Dev != rootDevice.Dev { + return errRemoveCrossDevice + } + return nil +} + +func classifyRecursiveOpenError(err error) error { + switch { + case errors.Is(err, unix.ENOENT): + return errRemoveTargetNotFound + case errors.Is(err, unix.ENOTDIR), errors.Is(err, unix.ELOOP): + return errRemoveTargetChanged + default: + return err + } +} + +func classifyRecursiveUnlinkError(err error) error { + switch { + case errors.Is(err, unix.ENOENT): + return errRemoveTargetNotFound + case errors.Is(err, unix.ENOTEMPTY), + errors.Is(err, unix.EEXIST), + errors.Is(err, unix.ENOTDIR): + return errRemoveTargetChanged + case errors.Is(err, unix.EBUSY), errors.Is(err, syscall.EXDEV): + return errRemoveCrossDevice + default: + return err + } +} diff --git a/backend/internal/storage/remove_unix_test.go b/backend/internal/storage/remove_unix_test.go new file mode 100644 index 000000000..1f1e02d34 --- /dev/null +++ b/backend/internal/storage/remove_unix_test.go @@ -0,0 +1,129 @@ +//go:build darwin || dragonfly || freebsd || linux || netbsd || openbsd + +package storage + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "golang.org/x/sys/unix" +) + +func TestRemoveStorageEntryUnlinksFinalSymlinkWithoutFollowingIt(t *testing.T) { + storage := t.TempDir() + outside := t.TempDir() + outsideFile := filepath.Join(outside, "keep.txt") + if err := os.WriteFile(outsideFile, []byte("keep"), 0o600); err != nil { + t.Fatal(err) + } + link := filepath.Join(storage, "link") + if err := os.Symlink(outsideFile, link); err != nil { + t.Fatal(err) + } + root, err := os.OpenRoot(storage) + if err != nil { + t.Fatal(err) + } + defer root.Close() + + if err := removeStorageEntry(root, "link", false); err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(link); !os.IsNotExist(err) { + t.Fatalf("symlink still exists: %v", err) + } + content, err := os.ReadFile(outsideFile) + if err != nil { + t.Fatal(err) + } + if string(content) != "keep" { + t.Fatalf("outside target changed: %q", content) + } +} + +func TestRecursiveRemoveDoesNotFollowNestedSymlink(t *testing.T) { + storage := t.TempDir() + outside := t.TempDir() + outsideFile := filepath.Join(outside, "keep.txt") + if err := os.WriteFile(outsideFile, []byte("keep"), 0o600); err != nil { + t.Fatal(err) + } + tree := filepath.Join(storage, "tree") + if err := os.Mkdir(tree, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tree, "inside.txt"), []byte("inside"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(tree, "outside-link")); err != nil { + t.Fatal(err) + } + root, err := os.OpenRoot(storage) + if err != nil { + t.Fatal(err) + } + defer root.Close() + + if err := removeStorageEntry(root, "tree", true); err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(tree); !os.IsNotExist(err) { + t.Fatalf("tree still exists: %v", err) + } + content, err := os.ReadFile(outsideFile) + if err != nil { + t.Fatal(err) + } + if string(content) != "keep" { + t.Fatalf("outside target changed: %q", content) + } +} + +func TestRecursiveRemoveDeviceBoundaryCheckFailsClosed(t *testing.T) { + first := unix.Stat_t{Dev: 1} + same := unix.Stat_t{Dev: 1} + different := unix.Stat_t{Dev: 2} + if err := requireRecursiveStorageDevice(&first, &same); err != nil { + t.Fatalf("same device rejected: %v", err) + } + if err := requireRecursiveStorageDevice(&first, &different); !errors.Is(err, errRemoveCrossDevice) { + t.Fatalf("cross-device error = %v, want errRemoveCrossDevice", err) + } + if err := requireRecursiveStorageDevice(nil, &same); !errors.Is(err, errRemoveCrossDevice) { + t.Fatalf("missing device error = %v, want errRemoveCrossDevice", err) + } +} + +func TestRecursiveRemoveDetectsDirectoryNameSubstitution(t *testing.T) { + storage := t.TempDir() + original := filepath.Join(storage, "target") + if err := os.Mkdir(original, 0o700); err != nil { + t.Fatal(err) + } + parent, err := os.Open(storage) + if err != nil { + t.Fatal(err) + } + defer parent.Close() + childFD, err := openDirectoryAt(int(parent.Fd()), "target") + if err != nil { + t.Fatal(err) + } + defer unix.Close(childFD) + var opened unix.Stat_t + if err := fstatRetry(childFD, &opened); err != nil { + t.Fatal(err) + } + + if err := os.Rename(original, filepath.Join(storage, "renamed")); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(original, 0o700); err != nil { + t.Fatal(err) + } + if err := verifyDirectoryIdentityAt(int(parent.Fd()), "target", &opened); !errors.Is(err, errRemoveTargetChanged) { + t.Fatalf("substitution error = %v, want errRemoveTargetChanged", err) + } +} diff --git a/backend/internal/storage/router.go b/backend/internal/storage/router.go index 3f2879044..2f6f79019 100644 --- a/backend/internal/storage/router.go +++ b/backend/internal/storage/router.go @@ -5,7 +5,6 @@ import "github.com/gin-gonic/gin" func RegisterRoutes(r *gin.Engine) { methods := []string{ "PUT", - "MKCOL", "PROPFIND", "PROPPATCH", } @@ -14,8 +13,12 @@ func RegisterRoutes(r *gin.Engine) { r.Handle(m, "/api/ss", WebDav) r.Handle(m, "/api/ss/*path", WebDav) } + r.Handle("MKCOL", "/api/ss", CreateDirectory) + r.Handle("MKCOL", "/api/ss/*path", CreateDirectory) webdavGroup := r.Group("api/ss", WebDAVMiddleware()) RegisterDataset(webdavGroup) RegisterFile(webdavGroup) + webdavGroup.DELETE("/files", RemoveFile) + webdavGroup.DELETE("/files/*path", RemoveFile) } diff --git a/backend/internal/storage/upload.go b/backend/internal/storage/upload.go new file mode 100644 index 000000000..ce5d757f6 --- /dev/null +++ b/backend/internal/storage/upload.go @@ -0,0 +1,555 @@ +package storage + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "syscall" + "unicode" + + "github.com/gin-gonic/gin" + "k8s.io/klog/v2" + + "github.com/raids-lab/crater/dao/model" + "github.com/raids-lab/crater/internal/bizerr" + "github.com/raids-lab/crater/internal/resputil" + "github.com/raids-lab/crater/internal/util" +) + +const ( + uploadFileMode os.FileMode = 0o644 + uploadStageDirMode os.FileMode = 0o700 + uploadStageFileMode os.FileMode = 0o600 + uploadStageAttempts = 16 + uploadStageRandomBytes = 16 + uploadStagePayload = "payload" + parentPathSegment = ".." +) + +var ( + errUploadTargetExists = errors.New("upload target exists") + errUploadTargetNotRegular = errors.New("upload target is not a regular file") + errUploadParentInvalid = errors.New("upload parent is missing, invalid, or outside the authorized storage root") +) + +// uploadParentAccessError preserves the operating-system cause of a failed +// parent-directory lookup while continuing to classify it as an invalid +// upload parent for existing callers. +type uploadParentAccessError struct { + cause error +} + +func (e *uploadParentAccessError) Error() string { + return errUploadParentInvalid.Error() +} + +func (e *uploadParentAccessError) Unwrap() error { + if e == nil { + return nil + } + return e.cause +} + +func (e *uploadParentAccessError) Is(target error) bool { + return target == errUploadParentInvalid +} + +func isUploadParentMissing(err error) bool { + var accessErr *uploadParentAccessError + return errors.As(err, &accessErr) && + (errors.Is(accessErr, os.ErrNotExist) || errors.Is(accessErr, syscall.ENOTDIR)) +} + +func isUploadParentInfrastructureFailure(err error) bool { + var accessErr *uploadParentAccessError + return errors.As(err, &accessErr) && + (errors.Is(accessErr, os.ErrPermission) || errors.Is(accessErr, syscall.EIO)) +} + +type uploadSourceError struct { + cause error +} + +func (e *uploadSourceError) Error() string { + return "read upload source: " + e.cause.Error() +} + +func (e *uploadSourceError) Unwrap() error { + if e == nil { + return nil + } + return e.cause +} + +type uploadOutcome struct { + Bytes int64 + Overwritten bool +} + +type uploadResponse struct { + RemotePath string `json:"remote_path"` + Bytes int64 `json:"bytes"` + Overwritten bool `json:"overwritten"` +} + +type uploadHandlerDeps struct { + authenticate func(*gin.Context) (util.JWTMessage, error) + permission func(string, util.JWTMessage, *gin.Context) model.FilePermission + redirect func(*gin.Context, string, util.JWTMessage) (string, error) + openTarget func(string, string, string) (*os.Root, string, error) + stagePublish func(io.Reader, *os.Root, string, bool, os.FileMode) (uploadOutcome, error) + storageRoot string +} + +func defaultUploadHandlerDeps() uploadHandlerDeps { + return uploadHandlerDeps{ + authenticate: CheckJWTToken, + permission: GetPermission, + redirect: Redirect, + openTarget: openUploadTarget, + stagePublish: stageAndPublishFile, + storageRoot: storageRootDir, + } +} + +// UploadFile atomically publishes one raw request body into an ordinary-user +// storage path. It deliberately uses a dedicated endpoint because the bundled +// WebDAV PUT handler truncates an existing target before the request completes. +func UploadFile(c *gin.Context) { + uploadFileWithDeps(c, defaultUploadHandlerDeps()) +} + +//nolint:gocyclo // Authentication, path authorization, and staged-publish errors have distinct stable API mappings. +func uploadFileWithDeps(c *gin.Context, deps uploadHandlerDeps) { + token, err := deps.authenticate(c) + if err != nil { + resputil.HandleError(c, bizerr.Auth.TokenInvalid.New("invalid token")) + return + } + + overwrite, err := parseUploadOverwrite(c.Query("overwrite")) + if err != nil { + resputil.HandleError(c, bizerr.BadRequest.ParameterError.New("overwrite must be true or false")) + return + } + + logicalPath, err := normalizeUploadLogicalPath(c.Param("path")) + if err != nil { + resputil.HandleError(c, bizerr.BadRequest.ParameterError.New("invalid remote file path")) + return + } + + if permission := deps.permission(logicalPath, token, c); permission != model.ReadWrite { + resputil.HandleError(c, bizerr.Forbidden.PermissionDenied.New("write permission is required")) + return + } + + realPath, err := deps.redirect(c, logicalPath, token) + if err != nil { + klog.Errorf("resolve upload target: %v", err) + resputil.HandleError(c, bizerr.Internal.FileSystemError.Wrap(err, "failed to resolve upload target")) + return + } + logicalRoot := strings.Split(logicalPath, "/")[0] + realRoot, err := deps.redirect(c, logicalRoot, token) + if err != nil { + klog.Errorf("resolve upload storage root: %v", err) + resputil.HandleError(c, bizerr.Internal.FileSystemError.Wrap(err, "failed to resolve upload target")) + return + } + + parentRoot, targetName, err := deps.openTarget(deps.storageRoot, realRoot, realPath) + if err != nil { + if errors.Is(err, errUploadParentInvalid) { + if isUploadParentInfrastructureFailure(err) { + klog.Errorf("open upload target: %v", err) + resputil.HandleError(c, bizerr.Internal.FileSystemError.Wrap(err, "failed to access upload storage")) + return + } + resputil.HandleError(c, bizerr.Conflict.ResourceStatusError.New("upload parent directory is unavailable")) + return + } + klog.Errorf("open upload target: %v", err) + resputil.HandleError(c, bizerr.Internal.FileSystemError.Wrap(err, "failed to access upload storage")) + return + } + defer parentRoot.Close() + + outcome, err := deps.stagePublish(c.Request.Body, parentRoot, targetName, overwrite, uploadFileMode) + if err != nil { + switch { + case errors.Is(err, errUploadTargetExists): + resputil.HandleError(c, bizerr.Conflict.ResourceAlreadyExists.New("target file already exists")) + case errors.Is(err, errUploadTargetNotRegular): + resputil.HandleError(c, bizerr.Conflict.ResourceStatusError.New("target path is not a regular file")) + case errors.Is(err, errUploadParentInvalid): + resputil.HandleError(c, bizerr.Conflict.ResourceStatusError.New("upload parent directory is unavailable")) + default: + var sourceErr *uploadSourceError + if errors.As(err, &sourceErr) { + resputil.HandleError(c, bizerr.BadRequest.InvalidRequest.Wrap(sourceErr, "failed to read upload body")) + return + } + klog.Errorf("publish uploaded file: %v", err) + resputil.HandleError(c, bizerr.Internal.FileSystemError.Wrap(err, "failed to store uploaded file")) + } + return + } + + status := http.StatusCreated + if outcome.Overwritten { + status = http.StatusOK + } + c.JSON(status, resputil.Response[uploadResponse]{ + Code: resputil.OK, + Data: uploadResponse{ + RemotePath: logicalPath, + Bytes: outcome.Bytes, + Overwritten: outcome.Overwritten, + }, + Message: "", + }) +} + +func parseUploadOverwrite(raw string) (bool, error) { + switch raw { + case "", "false": + return false, nil + case "true": + return true, nil + default: + return false, errors.New("invalid overwrite value") + } +} + +func normalizeUploadLogicalPath(raw string) (string, error) { + return normalizeStorageLogicalPath(raw, func(root string) bool { + switch root { + case model.UserPath, model.PublicPath, model.AccountPath: + return true + default: + return false + } + }) +} + +func normalizeWebDAVMutationLogicalPath(raw string) (string, error) { + return normalizeStorageLogicalPath(raw, func(root string) bool { + switch root { + case model.UserPath, + model.PublicPath, + model.AccountPath, + model.AdminUserPath, + model.AdminPublicPath, + model.AdminAccountPath: + return true + default: + return false + } + }) +} + +func normalizeStorageLogicalPath(raw string, allowedRoot func(string) bool) (string, error) { + if strings.ContainsRune(raw, '\\') { + return "", errors.New("backslashes are not allowed") + } + for _, character := range raw { + if unicode.IsControl(character) { + return "", errors.New("control characters are not allowed") + } + } + + trimmed := strings.Trim(raw, "/") + rawSegments := strings.Split(trimmed, "/") + segments := make([]string, 0, len(rawSegments)) + for _, segment := range rawSegments { + if segment == parentPathSegment { + return "", errors.New("parent traversal is not allowed") + } + if segment == "" || segment == "." { + continue + } + segments = append(segments, segment) + } + if len(segments) < 2 { + return "", errors.New("a file below a logical root is required") + } + if allowedRoot == nil || !allowedRoot(segments[0]) { + return "", errors.New("invalid logical root") + } + return strings.Join(segments, "/"), nil +} + +// openUploadTarget returns a directory handle anchored to the resolved target +// parent. os.Root rejects symlink traversal outside both the configured storage +// root and the caller's authorized real root, and remains anchored if a parent +// directory is renamed concurrently. +func openUploadTarget(storageRoot, authorizedRealRoot, targetRealPath string) (*os.Root, string, error) { + storage, err := os.OpenRoot(storageRoot) + if err != nil { + return nil, "", err + } + defer storage.Close() + + if !strings.HasPrefix(targetRealPath, strings.TrimSuffix(authorizedRealRoot, "/")+"/") { + return nil, "", errUploadParentInvalid + } + authorizedPath, err := cleanStorageRelativePath(authorizedRealRoot) + if err != nil { + return nil, "", errUploadParentInvalid + } + targetPath, err := cleanStorageRelativePath(targetRealPath) + if err != nil { + return nil, "", errUploadParentInvalid + } + targetRelative, err := filepath.Rel(authorizedPath, targetPath) + if err != nil || targetRelative == "." || pathEscapesRoot(targetRelative) { + return nil, "", errUploadParentInvalid + } + + authorized, err := storage.OpenRoot(authorizedPath) + if err != nil { + return nil, "", &uploadParentAccessError{cause: err} + } + defer authorized.Close() + + parentRelative := filepath.Dir(targetRelative) + targetName := filepath.Base(targetRelative) + if targetName == "." || targetName == string(filepath.Separator) { + return nil, "", errUploadParentInvalid + } + parent, err := authorized.OpenRoot(parentRelative) + if err != nil { + return nil, "", &uploadParentAccessError{cause: err} + } + return parent, targetName, nil +} + +func cleanStorageRelativePath(raw string) (string, error) { + if raw == "" || strings.ContainsRune(raw, '\\') { + return "", errUploadParentInvalid + } + for _, character := range raw { + if unicode.IsControl(character) { + return "", errUploadParentInvalid + } + } + + raw, err := trimLegacyStorageRootSlash(raw) + if err != nil { + return "", err + } + canonical, err := canonicalStorageSegments(raw) + if err != nil { + return "", err + } + normalized := filepath.FromSlash(canonical) + if filepath.Clean(normalized) != normalized || filepath.IsAbs(normalized) || pathEscapesRoot(normalized) { + return "", errUploadParentInvalid + } + return normalized, nil +} + +func trimLegacyStorageRootSlash(raw string) (string, error) { + if !strings.HasPrefix(raw, "/") { + return raw, nil + } + raw = strings.TrimPrefix(raw, "/") + if raw == "" || strings.HasPrefix(raw, "/") { + return "", errUploadParentInvalid + } + return raw, nil +} + +func canonicalStorageSegments(raw string) (string, error) { + // Historical User.Space and Account.Space records may start with "/". + // Redirect joins them after the configured prefix and produces one empty + // separator segment (for example "users//space/alice"). Accept exactly one + // such legacy marker, while rejecting every other ambiguous empty segment. + segments := strings.Split(raw, "/") + canonical := make([]string, 0, len(segments)) + legacyEmptySeen := false + for index, segment := range segments { + if segment == "" { + if legacyEmptySeen || index == 0 || index == len(segments)-1 { + return "", errUploadParentInvalid + } + legacyEmptySeen = true + continue + } + if segment == "." || segment == parentPathSegment { + return "", errUploadParentInvalid + } + canonical = append(canonical, segment) + } + return strings.Join(canonical, "/"), nil +} + +func pathEscapesRoot(path string) bool { + return path == parentPathSegment || + strings.HasPrefix(path, parentPathSegment+string(filepath.Separator)) +} + +func stageAndPublishFile( + source io.Reader, + parent *os.Root, + targetName string, + overwrite bool, + mode os.FileMode, +) (uploadOutcome, error) { + if err := validateUploadTarget(parent, targetName, overwrite); err != nil { + return uploadOutcome{}, err + } + + stageName, stageRoot, staged, err := createUploadStage(parent) + if err != nil { + return uploadOutcome{}, err + } + defer cleanupUploadStage(parent, stageRoot, stageName) + + written, err := writeUploadStage(source, staged, mode) + if err != nil { + return uploadOutcome{}, err + } + return publishStagedUpload(stageRoot, parent, targetName, overwrite, written) +} + +func validateUploadTarget(parent *os.Root, targetName string, overwrite bool) error { + if parent == nil || targetName == "" || targetName == "." || + targetName == parentPathSegment || filepath.Base(targetName) != targetName { + return errUploadParentInvalid + } + targetInfo, err := parent.Lstat(targetName) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + if !targetInfo.Mode().IsRegular() { + return errUploadTargetNotRegular + } + if !overwrite { + return errUploadTargetExists + } + return nil +} + +func writeUploadStage(source io.Reader, staged *os.File, mode os.FileMode) (int64, error) { + tracked := &trackedUploadSource{source: source} + written, copyErr := io.Copy(staged, tracked) + if copyErr != nil { + _ = staged.Close() + if tracked.readErr != nil { + return 0, &uploadSourceError{cause: tracked.readErr} + } + return 0, copyErr + } + if err := staged.Chmod(mode); err != nil { + _ = staged.Close() + return 0, err + } + if err := staged.Sync(); err != nil { + _ = staged.Close() + return 0, err + } + if err := staged.Close(); err != nil { + return 0, err + } + return written, nil +} + +func publishStagedUpload( + stageRoot, parent *os.Root, + targetName string, + overwrite bool, + written int64, +) (uploadOutcome, error) { + for retry := 0; retry < 2; retry++ { + if err := publishUploadNoClobber(stageRoot, parent, targetName); err == nil { + return uploadOutcome{Bytes: written}, nil + } else if !os.IsExist(err) { + return uploadOutcome{}, err + } + + targetInfo, statErr := parent.Lstat(targetName) + if os.IsNotExist(statErr) { + continue + } + if statErr != nil { + return uploadOutcome{}, statErr + } + if !targetInfo.Mode().IsRegular() { + return uploadOutcome{}, errUploadTargetNotRegular + } + if !overwrite { + return uploadOutcome{}, errUploadTargetExists + } + if err := publishUploadOverwrite(stageRoot, parent, targetName); err != nil { + return uploadOutcome{}, err + } + return uploadOutcome{Bytes: written, Overwritten: true}, nil + } + return uploadOutcome{}, errUploadTargetExists +} + +func createUploadStage(parent *os.Root) (string, *os.Root, *os.File, error) { + for attempt := 0; attempt < uploadStageAttempts; attempt++ { + random := make([]byte, uploadStageRandomBytes) + if _, err := rand.Read(random); err != nil { + return "", nil, nil, err + } + stageName := ".crater-upload-" + hex.EncodeToString(random) + if err := parent.Mkdir(stageName, uploadStageDirMode); err != nil { + if os.IsExist(err) { + continue + } + return "", nil, nil, err + } + stageRoot, err := parent.OpenRoot(stageName) + if err != nil { + _ = parent.Remove(stageName) + return "", nil, nil, err + } + staged, err := stageRoot.OpenFile( + uploadStagePayload, + os.O_WRONLY|os.O_CREATE|os.O_EXCL, + uploadStageFileMode, + ) + if err != nil { + _ = stageRoot.Close() + _ = parent.Remove(stageName) + return "", nil, nil, err + } + return stageName, stageRoot, staged, nil + } + return "", nil, nil, errors.New("could not allocate a private upload staging directory") +} + +func cleanupUploadStage(parent, stageRoot *os.Root, stageName string) { + if stageRoot != nil { + _ = stageRoot.Remove(uploadStagePayload) + _ = stageRoot.Close() + } + if parent != nil && stageName != "" { + _ = parent.Remove(stageName) + } +} + +type trackedUploadSource struct { + source io.Reader + readErr error +} + +func (reader *trackedUploadSource) Read(data []byte) (int, error) { + read, err := reader.source.Read(data) + if err != nil && !errors.Is(err, io.EOF) { + reader.readErr = err + } + return read, err +} diff --git a/backend/internal/storage/upload_publish_other.go b/backend/internal/storage/upload_publish_other.go new file mode 100644 index 000000000..2d2f3367a --- /dev/null +++ b/backend/internal/storage/upload_publish_other.go @@ -0,0 +1,18 @@ +//go:build !darwin && !dragonfly && !freebsd && !linux && !netbsd && !openbsd + +package storage + +import ( + "errors" + "os" +) + +var errUploadPublishUnsupported = errors.New("atomic upload publishing is unsupported on this platform") + +func publishUploadNoClobber(*os.Root, *os.Root, string) error { + return errUploadPublishUnsupported +} + +func publishUploadOverwrite(*os.Root, *os.Root, string) error { + return errUploadPublishUnsupported +} diff --git a/backend/internal/storage/upload_publish_unix.go b/backend/internal/storage/upload_publish_unix.go new file mode 100644 index 000000000..f438ec015 --- /dev/null +++ b/backend/internal/storage/upload_publish_unix.go @@ -0,0 +1,60 @@ +//go:build darwin || dragonfly || freebsd || linux || netbsd || openbsd + +package storage + +import ( + "os" + + "golang.org/x/sys/unix" +) + +// Publish from the already-open staging directory. Resolving the source +// relative to its directory descriptor prevents a writable-parent rename race +// from substituting another staging directory after the upload completes. +func publishUploadNoClobber(stage, parent *os.Root, targetName string) error { + stageDirectory, parentDirectory, err := openUploadDirectoryHandles(stage, parent) + if err != nil { + return err + } + defer stageDirectory.Close() + defer parentDirectory.Close() + return unix.Linkat( + int(stageDirectory.Fd()), + uploadStagePayload, + int(parentDirectory.Fd()), + targetName, + 0, + ) +} + +func publishUploadOverwrite(stage, parent *os.Root, targetName string) error { + stageDirectory, parentDirectory, err := openUploadDirectoryHandles(stage, parent) + if err != nil { + return err + } + defer stageDirectory.Close() + defer parentDirectory.Close() + return unix.Renameat( + int(stageDirectory.Fd()), + uploadStagePayload, + int(parentDirectory.Fd()), + targetName, + ) +} + +func openUploadDirectoryHandles(stage, parent *os.Root) ( + stageDirectory *os.File, + parentDirectory *os.File, + err error, +) { + stageDirectory, err = stage.Open(".") + if err != nil { + return nil, nil, err + } + parentDirectory, err = parent.Open(".") + if err != nil { + _ = stageDirectory.Close() + return nil, nil, err + } + return stageDirectory, parentDirectory, nil +} diff --git a/backend/internal/storage/upload_test.go b/backend/internal/storage/upload_test.go new file mode 100644 index 000000000..8586013ed --- /dev/null +++ b/backend/internal/storage/upload_test.go @@ -0,0 +1,694 @@ +package storage + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/gin-gonic/gin" + + "github.com/raids-lab/crater/dao/model" + "github.com/raids-lab/crater/internal/util" +) + +const testUploadMode os.FileMode = 0o640 + +func TestStageAndPublishFileStreamsBinaryAtomically(t *testing.T) { + directory := t.TempDir() + payload := []byte{0x00, 0xff, 'c', 'r', 'a', 't', 'e', 'r'} + + outcome, err := stageInDirectory(t, directory, "data.bin", bytes.NewReader(payload), false) + if err != nil { + t.Fatalf("stageAndPublishFile: %v", err) + } + if outcome.Bytes != int64(len(payload)) || outcome.Overwritten { + t.Fatalf("outcome = %#v", outcome) + } + target := filepath.Join(directory, "data.bin") + assertStoredFile(t, target, payload) + info, err := os.Stat(target) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o640 { + t.Fatalf("mode = %o, want 640", info.Mode().Perm()) + } + assertNoUploadTemps(t, directory) +} + +func TestStageAndPublishFilePublishesEmptyFile(t *testing.T) { + directory := t.TempDir() + outcome, err := stageInDirectory(t, directory, "empty.bin", bytes.NewReader(nil), false) + if err != nil { + t.Fatalf("stageAndPublishFile: %v", err) + } + if outcome.Bytes != 0 || outcome.Overwritten { + t.Fatalf("outcome = %#v", outcome) + } + assertStoredFile(t, filepath.Join(directory, "empty.bin"), nil) + assertNoUploadTemps(t, directory) +} + +func TestStageAndPublishFileNeedsExplicitOverwrite(t *testing.T) { + directory := t.TempDir() + target := filepath.Join(directory, "data.bin") + if err := os.WriteFile(target, []byte("original"), 0o600); err != nil { + t.Fatal(err) + } + + _, err := stageInDirectory(t, directory, "data.bin", bytes.NewBufferString("replacement"), false) + if !errors.Is(err, errUploadTargetExists) { + t.Fatalf("error = %v, want errUploadTargetExists", err) + } + assertStoredFile(t, target, []byte("original")) + assertNoUploadTemps(t, directory) + + outcome, err := stageInDirectory(t, directory, "data.bin", bytes.NewBufferString("replacement"), true) + if err != nil { + t.Fatalf("overwrite: %v", err) + } + if !outcome.Overwritten { + t.Fatalf("outcome = %#v, want overwritten", outcome) + } + assertStoredFile(t, target, []byte("replacement")) + assertNoUploadTemps(t, directory) +} + +func TestStageAndPublishFileOverwriteCreatesWhenAbsent(t *testing.T) { + directory := t.TempDir() + outcome, err := stageInDirectory(t, directory, "new.bin", bytes.NewBufferString("new"), true) + if err != nil { + t.Fatal(err) + } + if outcome.Overwritten { + t.Fatalf("outcome = %#v, want new file", outcome) + } + assertStoredFile(t, filepath.Join(directory, "new.bin"), []byte("new")) +} + +func TestStageAndPublishFileKeepsOldTargetDuringTransfer(t *testing.T) { + directory := t.TempDir() + target := filepath.Join(directory, "data.bin") + if err := os.WriteFile(target, []byte("old-complete"), 0o600); err != nil { + t.Fatal(err) + } + root, err := os.OpenRoot(directory) + if err != nil { + t.Fatal(err) + } + defer root.Close() + + source, writer := io.Pipe() + type result struct { + outcome uploadOutcome + err error + } + done := make(chan result, 1) + go func() { + outcome, err := stageAndPublishFile(source, root, "data.bin", true, 0o640) + done <- result{outcome: outcome, err: err} + }() + if _, err := writer.Write([]byte("new-part-1")); err != nil { + t.Fatal(err) + } + assertStoredFile(t, target, []byte("old-complete")) + if _, err := writer.Write([]byte("-part-2")); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + got := <-done + if got.err != nil { + t.Fatal(got.err) + } + if !got.outcome.Overwritten { + t.Fatalf("outcome = %#v", got.outcome) + } + assertStoredFile(t, target, []byte("new-part-1-part-2")) + assertNoUploadTemps(t, directory) +} + +func TestStageAndPublishFileCleansSourceFailureWithoutChangingTarget(t *testing.T) { + directory := t.TempDir() + target := filepath.Join(directory, "data.bin") + if err := os.WriteFile(target, []byte("old"), 0o600); err != nil { + t.Fatal(err) + } + sentinel := errors.New("source failed") + source := io.MultiReader(bytes.NewBufferString("partial"), uploadErrorReader{err: sentinel}) + + _, err := stageInDirectory(t, directory, "data.bin", source, true) + var sourceErr *uploadSourceError + if !errors.As(err, &sourceErr) || !errors.Is(sourceErr, sentinel) { + t.Fatalf("error = %T %v, want uploadSourceError", err, err) + } + assertStoredFile(t, target, []byte("old")) + assertNoUploadTemps(t, directory) +} + +func TestStageAndPublishFilePublishesOpenedStageAfterNameReplacement(t *testing.T) { + directory := t.TempDir() + var renamedStage string + var replacementStage string + source := &callbackEOFReader{ + data: []byte("safe"), + onEOF: func() { + matches, err := filepath.Glob(filepath.Join(directory, ".crater-upload-*")) + if err != nil || len(matches) != 1 { + t.Fatalf("staging entries = %#v, err=%v", matches, err) + } + replacementStage = matches[0] + renamedStage = replacementStage + "-renamed" + if err := os.Rename(replacementStage, renamedStage); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(replacementStage, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(replacementStage, uploadStagePayload), []byte("attacker"), 0o600); err != nil { + t.Fatal(err) + } + }, + } + + if _, err := stageInDirectory(t, directory, "result.bin", source, false); err != nil { + t.Fatal(err) + } + assertStoredFile(t, filepath.Join(directory, "result.bin"), []byte("safe")) + if renamedStage == "" || replacementStage == "" { + t.Fatal("replacement callback did not run") + } +} + +func TestStageAndPublishFileRejectsNonRegularTargets(t *testing.T) { + directory := t.TempDir() + targetDirectory := filepath.Join(directory, "target") + if err := os.Mkdir(targetDirectory, 0o700); err != nil { + t.Fatal(err) + } + if _, err := stageInDirectory(t, directory, "target", bytes.NewBufferString("x"), true); !errors.Is(err, errUploadTargetNotRegular) { + t.Fatalf("directory error = %v", err) + } + + if err := os.Symlink(targetDirectory, filepath.Join(directory, "link")); err != nil { + t.Fatal(err) + } + if _, err := stageInDirectory(t, directory, "link", bytes.NewBufferString("x"), true); !errors.Is(err, errUploadTargetNotRegular) { + t.Fatalf("symlink error = %v", err) + } +} + +func TestStageAndPublishFileRaceHasOneWinner(t *testing.T) { + directory := t.TempDir() + root, err := os.OpenRoot(directory) + if err != nil { + t.Fatal(err) + } + defer root.Close() + + const contenders = 8 + var wait sync.WaitGroup + wait.Add(contenders) + start := make(chan struct{}) + errorsSeen := make(chan error, contenders) + + for index := 0; index < contenders; index++ { + go func() { + defer wait.Done() + <-start + _, err := stageAndPublishFile( + bytes.NewBufferString(string(rune('a'+index))), + root, + "race.bin", + false, + 0o640, + ) + errorsSeen <- err + }() + } + close(start) + wait.Wait() + close(errorsSeen) + + successes := 0 + exists := 0 + for err := range errorsSeen { + switch { + case err == nil: + successes++ + case errors.Is(err, errUploadTargetExists): + exists++ + default: + t.Fatalf("unexpected error: %v", err) + } + } + if successes != 1 || exists != contenders-1 { + t.Fatalf("successes=%d exists=%d", successes, exists) + } + assertNoUploadTemps(t, directory) +} + +func TestOpenUploadTargetRejectsEscapingParentSymlink(t *testing.T) { + storage := t.TempDir() + authorized := filepath.Join(storage, "users", "alice") + outside := filepath.Join(storage, "users", "bob") + if err := os.MkdirAll(authorized, 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(outside, 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(authorized, "escape")); err != nil { + t.Fatal(err) + } + + _, _, err := openUploadTarget(storage, testRealUserRoot, "users/alice/escape/secret.bin") + if !errors.Is(err, errUploadParentInvalid) { + t.Fatalf("error = %v, want errUploadParentInvalid", err) + } +} + +func TestOpenUploadTargetRejectsAuthorizedRootEscapingStorage(t *testing.T) { + storage := t.TempDir() + outside := t.TempDir() + if err := os.MkdirAll(filepath.Join(storage, "users"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(storage, "users", "alice")); err != nil { + t.Fatal(err) + } + + _, _, err := openUploadTarget(storage, testRealUserRoot, "users/alice/secret.bin") + if !errors.Is(err, errUploadParentInvalid) { + t.Fatalf("error = %v, want errUploadParentInvalid", err) + } +} + +func TestOpenUploadTargetRejectsRawInternalTraversal(t *testing.T) { + for _, candidate := range []string{ + "users/../users/alice", + "users/alice/../alice/result.bin", + `users\..\users\alice`, + "users/.", + "users/alice/", + "//users/alice", + "users///alice", + "users/alice//runs//result.bin", + "users/alice\n", + } { + if _, err := cleanStorageRelativePath(candidate); !errors.Is(err, errUploadParentInvalid) { + t.Fatalf("cleanStorageRelativePath(%q) error = %v, want errUploadParentInvalid", candidate, err) + } + } +} + +func TestCleanStorageRelativePathSupportsOneLegacyAbsoluteSpaceMarker(t *testing.T) { + for input, want := range map[string]string{ + "/public/models": filepath.Join("public", "models"), + "users//space/zhouyh25": filepath.Join("users", "space", "zhouyh25"), + "users//space/zhouyh25/file": filepath.Join("users", "space", "zhouyh25", "file"), + } { + got, err := cleanStorageRelativePath(input) + if err != nil { + t.Fatalf("cleanStorageRelativePath(%q): %v", input, err) + } + if got != want { + t.Fatalf("cleanStorageRelativePath(%q) = %q, want %q", input, got, want) + } + } +} + +func TestOpenUploadTargetSupportsLegacyLeadingSlashSpace(t *testing.T) { + storage := t.TempDir() + if err := os.MkdirAll(filepath.Join(storage, "users", "space", "alice", "jobs"), 0o755); err != nil { + t.Fatal(err) + } + parent, targetName, err := openUploadTarget( + storage, + "users//space/alice", + "users//space/alice/jobs/result.bin", + ) + if err != nil { + t.Fatal(err) + } + defer parent.Close() + if targetName != "result.bin" { + t.Fatalf("targetName = %q", targetName) + } +} + +func TestOpenUploadTargetDoesNotCreateMissingParent(t *testing.T) { + storage := t.TempDir() + if err := os.MkdirAll(filepath.Join(storage, "users", "alice"), 0o755); err != nil { + t.Fatal(err) + } + + _, _, err := openUploadTarget(storage, testRealUserRoot, "users/alice/missing/secret.bin") + if !errors.Is(err, errUploadParentInvalid) { + t.Fatalf("error = %v, want errUploadParentInvalid", err) + } + if _, err := os.Stat(filepath.Join(storage, "users", "alice", "missing")); !os.IsNotExist(err) { + t.Fatalf("missing parent was created: %v", err) + } +} + +func TestOpenedUploadTargetRemainsAnchoredAcrossParentRename(t *testing.T) { + storage := t.TempDir() + jobs := filepath.Join(storage, "users", "alice", "jobs") + original := filepath.Join(storage, "users", "alice", "jobs-original") + outside := t.TempDir() + if err := os.MkdirAll(jobs, 0o755); err != nil { + t.Fatal(err) + } + + parent, targetName, err := openUploadTarget(storage, testRealUserRoot, "users/alice/jobs/result.bin") + if err != nil { + t.Fatal(err) + } + defer parent.Close() + if err := os.Rename(jobs, original); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, jobs); err != nil { + t.Fatal(err) + } + + if _, err := stageAndPublishFile(bytes.NewBufferString("safe"), parent, targetName, false, 0o640); err != nil { + t.Fatal(err) + } + assertStoredFile(t, filepath.Join(original, "result.bin"), []byte("safe")) + if _, err := os.Stat(filepath.Join(outside, "result.bin")); !os.IsNotExist(err) { + t.Fatalf("upload escaped through replacement symlink: %v", err) + } +} + +func TestUploadFileHandlerHTTPContract(t *testing.T) { + gin.SetMode(gin.TestMode) + t.Run("created", func(t *testing.T) { + storage := newUploadHandlerStorage(t) + recorder := serveUpload(t, testUploadHandlerDeps(storage), "user/result.bin", "overwrite=false", bytes.NewBufferString("data")) + assertUploadEnvelope(t, recorder, http.StatusCreated, 0) + assertStoredFile(t, filepath.Join(storage, "users", "alice", "result.bin"), []byte("data")) + info, err := os.Stat(filepath.Join(storage, "users", "alice", "result.bin")) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != uploadFileMode { + t.Fatalf("uploaded mode = %o, want %o", info.Mode().Perm(), uploadFileMode) + } + }) + + t.Run("overwritten", func(t *testing.T) { + storage := newUploadHandlerStorage(t) + target := filepath.Join(storage, "users", "alice", "result.bin") + if err := os.WriteFile(target, []byte("old"), 0o600); err != nil { + t.Fatal(err) + } + recorder := serveUpload(t, testUploadHandlerDeps(storage), "user/result.bin", "overwrite=true", bytes.NewBufferString("new")) + assertUploadEnvelope(t, recorder, http.StatusOK, 0) + assertStoredFile(t, target, []byte("new")) + }) + + tests := []struct { + name string + path string + query string + wantStatus int + wantCode int + mutate func(*testing.T, string, *uploadHandlerDeps) + }{ + {name: "invalid overwrite", path: "user/result.bin", query: "overwrite=yes", wantStatus: http.StatusBadRequest, wantCode: 40004}, + {name: "invalid path", path: testLogicalUserRoot, query: "overwrite=false", wantStatus: http.StatusBadRequest, wantCode: 40004}, + { + name: "unauthorized", path: "user/result.bin", query: "overwrite=false", + wantStatus: http.StatusUnauthorized, wantCode: 40102, + mutate: func(_ *testing.T, _ string, deps *uploadHandlerDeps) { + deps.authenticate = func(*gin.Context) (util.JWTMessage, error) { + return util.JWTMessage{}, errors.New("invalid token") + } + }, + }, + { + name: "forbidden", path: "user/result.bin", query: "overwrite=false", + wantStatus: http.StatusForbidden, wantCode: 40301, + mutate: func(_ *testing.T, _ string, deps *uploadHandlerDeps) { + deps.permission = func(string, util.JWTMessage, *gin.Context) model.FilePermission { + return model.ReadOnly + } + }, + }, + { + name: "existing target", path: "user/result.bin", query: "overwrite=false", + wantStatus: http.StatusConflict, wantCode: 40901, + mutate: func(t *testing.T, storage string, _ *uploadHandlerDeps) { + if err := os.WriteFile(filepath.Join(storage, "users", "alice", "result.bin"), []byte("old"), 0o600); err != nil { + t.Fatal(err) + } + }, + }, + { + name: "nonregular target", path: "user/result.bin", query: "overwrite=true", + wantStatus: http.StatusConflict, wantCode: 40902, + mutate: func(t *testing.T, storage string, _ *uploadHandlerDeps) { + if err := os.Mkdir(filepath.Join(storage, "users", "alice", "result.bin"), 0o700); err != nil { + t.Fatal(err) + } + }, + }, + { + name: "missing parent", path: "user/missing/result.bin", query: "overwrite=false", + wantStatus: http.StatusConflict, wantCode: 40902, + }, + { + name: "source read failure", path: "user/result.bin", query: "overwrite=false", + wantStatus: http.StatusBadRequest, wantCode: 40001, + mutate: func(_ *testing.T, _ string, deps *uploadHandlerDeps) { + deps.stagePublish = func(io.Reader, *os.Root, string, bool, os.FileMode) (uploadOutcome, error) { + return uploadOutcome{}, &uploadSourceError{cause: errors.New("broken body")} + } + }, + }, + { + name: "filesystem failure", path: "user/result.bin", query: "overwrite=false", + wantStatus: http.StatusInternalServerError, wantCode: 50005, + mutate: func(_ *testing.T, _ string, deps *uploadHandlerDeps) { + deps.openTarget = func(string, string, string) (*os.Root, string, error) { + return nil, "", errors.New("disk unavailable") + } + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + storage := newUploadHandlerStorage(t) + deps := testUploadHandlerDeps(storage) + if test.mutate != nil { + test.mutate(t, storage, &deps) + } + recorder := serveUpload(t, deps, test.path, test.query, bytes.NewBufferString("data")) + assertUploadEnvelope(t, recorder, test.wantStatus, test.wantCode) + }) + } +} + +func TestNormalizeUploadLogicalPath(t *testing.T) { + if got, err := normalizeUploadLogicalPath("/user//runs/./a.bin"); err != nil || got != "user/runs/a.bin" { + t.Fatalf("normalize = %q, %v", got, err) + } + for _, invalid := range []string{testLogicalUserRoot, "admin/file", "user/../public/file", `user\file`, "user/a\nb"} { + if _, err := normalizeUploadLogicalPath(invalid); err == nil { + t.Errorf("normalizeUploadLogicalPath(%q) accepted invalid path", invalid) + } + } +} + +func TestNormalizeWebDAVMutationLogicalPathKeepsAdminCompatibility(t *testing.T) { + for _, valid := range []string{ + "user/new-directory", + "public/new-directory", + "account/new-directory", + "admin-user/alice/new-directory", + "admin-public/new-directory", + "admin-account/team/new-directory", + } { + if _, err := normalizeWebDAVMutationLogicalPath(valid); err != nil { + t.Errorf("normalizeWebDAVMutationLogicalPath(%q): %v", valid, err) + } + } + for _, invalid := range []string{ + testLogicalUserRoot, + "admin-user", + "crater-model/new-directory", + "user/../public/new-directory", + } { + if _, err := normalizeWebDAVMutationLogicalPath(invalid); err == nil { + t.Errorf("normalizeWebDAVMutationLogicalPath(%q) accepted invalid path", invalid) + } + } +} + +func TestParseUploadOverwrite(t *testing.T) { + for _, valid := range []string{"", "false", "true"} { + if _, err := parseUploadOverwrite(valid); err != nil { + t.Errorf("parseUploadOverwrite(%q): %v", valid, err) + } + } + for _, invalid := range []string{"1", "TRUE", "yes"} { + if _, err := parseUploadOverwrite(invalid); err == nil { + t.Errorf("parseUploadOverwrite(%q) accepted invalid value", invalid) + } + } +} + +func TestRegisterRoutesServesUploadEndpoint(t *testing.T) { + gin.SetMode(gin.TestMode) + router := gin.New() + RegisterRoutes(router) + + request := httptest.NewRequest( + http.MethodPost, + "/api/ss/upload/user/test.bin?overwrite=false", + bytes.NewBufferString("data"), + ) + recorder := httptest.NewRecorder() + router.ServeHTTP(recorder, request) + assertUploadEnvelope(t, recorder, http.StatusUnauthorized, 40102) +} + +type uploadErrorReader struct { + err error +} + +func (reader uploadErrorReader) Read([]byte) (int, error) { + return 0, reader.err +} + +type callbackEOFReader struct { + data []byte + onEOF func() + read bool +} + +func (reader *callbackEOFReader) Read(buffer []byte) (int, error) { + if !reader.read { + reader.read = true + return copy(buffer, reader.data), nil + } + reader.onEOF() + return 0, io.EOF +} + +func stageInDirectory( + t *testing.T, + directory string, + targetName string, + source io.Reader, + overwrite bool, +) (uploadOutcome, error) { + t.Helper() + root, err := os.OpenRoot(directory) + if err != nil { + t.Fatal(err) + } + defer root.Close() + return stageAndPublishFile(source, root, targetName, overwrite, testUploadMode) +} + +func newUploadHandlerStorage(t *testing.T) string { + t.Helper() + storage := t.TempDir() + if err := os.MkdirAll(filepath.Join(storage, "users", "alice"), 0o755); err != nil { + t.Fatal(err) + } + return storage +} + +func testUploadHandlerDeps(storage string) uploadHandlerDeps { + return uploadHandlerDeps{ + authenticate: func(*gin.Context) (util.JWTMessage, error) { + return util.JWTMessage{}, nil + }, + permission: func(string, util.JWTMessage, *gin.Context) model.FilePermission { + return model.ReadWrite + }, + redirect: func(_ *gin.Context, logicalPath string, _ util.JWTMessage) (string, error) { + if logicalPath == testLogicalUserRoot { + return testRealUserRoot, nil + } + return testRealUserRoot + "/" + strings.TrimPrefix(logicalPath, testLogicalUserRoot+"/"), nil + }, + openTarget: openUploadTarget, + stagePublish: stageAndPublishFile, + storageRoot: storage, + } +} + +func serveUpload( + t *testing.T, + deps uploadHandlerDeps, + logicalPath string, + query string, + body io.Reader, +) *httptest.ResponseRecorder { + t.Helper() + router := gin.New() + router.POST("/upload/*path", func(c *gin.Context) { + uploadFileWithDeps(c, deps) + }) + target := "/upload/" + logicalPath + if query != "" { + target += "?" + query + } + request := httptest.NewRequest(http.MethodPost, target, body) + recorder := httptest.NewRecorder() + router.ServeHTTP(recorder, request) + return recorder +} + +func assertUploadEnvelope(t *testing.T, recorder *httptest.ResponseRecorder, wantStatus, wantCode int) { + t.Helper() + if recorder.Code != wantStatus { + t.Fatalf("status = %d, want %d; body=%s", recorder.Code, wantStatus, recorder.Body.String()) + } + var envelope struct { + Code int `json:"code"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &envelope); err != nil { + t.Fatalf("decode response: %v; body=%s", err, recorder.Body.String()) + } + if envelope.Code != wantCode { + t.Fatalf("code = %d, want %d; body=%s", envelope.Code, wantCode, recorder.Body.String()) + } +} + +func assertStoredFile(t *testing.T, path string, want []byte) { + t.Helper() + got, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, want) { + t.Fatalf("%s = %v, want %v", path, got, want) + } +} + +func assertNoUploadTemps(t *testing.T, directory string) { + t.Helper() + matches, err := filepath.Glob(filepath.Join(directory, ".crater-upload-*")) + if err != nil { + t.Fatal(err) + } + if len(matches) != 0 { + t.Fatalf("temporary upload entries remain: %#v", matches) + } +} diff --git a/cli/cmd/file.go b/cli/cmd/file.go new file mode 100644 index 000000000..6104f5ce4 --- /dev/null +++ b/cli/cmd/file.go @@ -0,0 +1,343 @@ +package cmd + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "unicode" + + "github.com/spf13/cobra" + + "github.com/raids-lab/crater/cli/internal/api" + "github.com/raids-lab/crater/cli/internal/clierror" + "github.com/raids-lab/crater/cli/internal/completion" + "github.com/raids-lab/crater/cli/internal/i18n" + "github.com/raids-lab/crater/cli/internal/output" + "github.com/raids-lab/crater/cli/pkg/errorcodes" +) + +var fileRemoteRoots = []string{"user", "public", "account"} + +var fileCmd = &cobra.Command{ + Use: "file", + Short: "Manage remote files", + Long: "Manage files and directories in user, public, and account storage spaces.", + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) > 0 { + return errUnknownSubcommand(cmd, args[0]) + } + return cmd.Help() + }, +} + +var fileUploadCmd = &cobra.Command{ + Use: "upload ", + Short: "Upload one local file", + Args: fileUploadArgs, + RunE: runFileUpload, +} + +type fileUploadDeps struct { + client func() (api.FileUploadClient, error) + stdout io.Writer + json bool +} + +type fileUploadInput struct { + localPath string + remotePath string + overwrite bool +} + +type fileUploadResult struct { + LocalPath string + RemotePath string + Bytes int64 + Overwrite bool + Overwritten bool +} + +func fileUploadArgs(cmd *cobra.Command, args []string) error { + if len(args) > 2 { + return errTooManyArgs(cmd, len(args), 2) + } + if len(args) < 2 { + field := "local-file" + label := i18n.T("file_label_local_file") + if len(args) == 1 { + field = "remote-path" + label = i18n.T("file_label_remote_file") + } + return errUsageFromIssues([]usageIssue{{ + Code: errorcodes.ErrMissingRequiredFlag, + Message: i18n.T("err_missing_required_arg", label, field), + Field: field, + }}) + } + return nil +} + +func runFileUpload(cmd *cobra.Command, args []string) error { + return runFileUploadWith(cmd, args, fileUploadDeps{ + client: activeFileUploadClient, + stdout: os.Stdout, + json: outputJSON, + }) +} + +func activeFileUploadClient() (api.FileUploadClient, error) { + return activeAPIClient() +} + +func runFileUploadWith(cmd *cobra.Command, args []string, deps fileUploadDeps) error { + remotePath, err := normalizeRemotePath(args[1], false) + if err != nil { + return err + } + if len(strings.Split(remotePath, "/")) < 2 { + return invalidRemotePathIssue(i18n.T("err_file_path_not_file", args[1])) + } + + source, err := openUploadSource(args[0]) + if err != nil { + return err + } + defer source.Close() + + overwrite, _ := cmd.Flags().GetBool("overwrite") + result, err := uploadRemoteFile(cmd.Context(), deps.client, source, fileUploadInput{ + localPath: args[0], + remotePath: remotePath, + overwrite: overwrite, + }) + if err != nil { + return err + } + return writeFileUploadResult(deps.stdout, deps.json, result) +} + +func openUploadSource(localPath string) (*os.File, error) { + pathInfo, err := os.Stat(localPath) + if err != nil { + return nil, localFileError("err_file_local_stat", localPath, err) + } + if !pathInfo.Mode().IsRegular() { + return nil, invalidLocalPathIssue(i18n.T("err_file_local_not_regular", localPath)) + } + + source, err := openUploadFileNoBlock(localPath) + if err != nil { + return nil, localFileError("err_file_local_open", localPath, err) + } + info, err := source.Stat() + if err != nil { + _ = source.Close() + return nil, localFileError("err_file_local_stat", localPath, err) + } + if !info.Mode().IsRegular() { + _ = source.Close() + return nil, invalidLocalPathIssue(i18n.T("err_file_local_not_regular", localPath)) + } + return source, nil +} + +func uploadRemoteFile( + ctx context.Context, + clientFactory func() (api.FileUploadClient, error), + source io.Reader, + input fileUploadInput, +) (fileUploadResult, error) { + client, err := clientFactory() + if err != nil { + return fileUploadResult{}, err + } + + uploaded, err := client.UploadFile(ctx, input.remotePath, source, input.overwrite) + if err != nil { + var sourceErr *api.SourceReadError + if errors.As(err, &sourceErr) { + return fileUploadResult{}, localFileError("err_file_local_read", input.localPath, sourceErr.Cause) + } + return fileUploadResult{}, cliErrFromAPI(err) + } + return fileUploadResult{ + LocalPath: input.localPath, + RemotePath: uploaded.RemotePath, + Bytes: uploaded.Bytes, + Overwrite: input.overwrite, + Overwritten: uploaded.Overwritten, + }, nil +} + +func writeFileUploadResult(writer io.Writer, jsonOutput bool, result fileUploadResult) error { + if jsonOutput { + return output.WriteSuccessJSON(writer, output.SuccessEnvelope(map[string]interface{}{ + "local_path": result.LocalPath, + "remote_path": result.RemotePath, + "bytes": result.Bytes, + "overwrite": result.Overwrite, + "overwritten": result.Overwritten, + })) + } + _, err := fmt.Fprintln(writer, i18n.T("file_upload_success", result.LocalPath, result.RemotePath, result.Bytes)) + if err != nil { + return &clierror.Error{ + Category: errorcodes.CategorySystem, + Code: errorcodes.ErrCommandExecution, + Message: i18n.T("err_file_output", err.Error()), + Context: map[string]interface{}{"msg": err.Error()}, + } + } + return nil +} + +func normalizeRemotePath(remotePath string, allowEmpty bool) (string, error) { + if remotePath == "" { + if allowEmpty { + return "", nil + } + return "", invalidRemotePathIssue(i18n.T("err_file_path_invalid", remotePath)) + } + if strings.ContainsRune(remotePath, '\\') { + return "", invalidRemotePathIssue(i18n.T("err_file_path_invalid", remotePath)) + } + for _, character := range remotePath { + if unicode.IsControl(character) { + return "", invalidRemotePathIssue(i18n.T("err_file_path_invalid", remotePath)) + } + } + + trimmed := strings.Trim(remotePath, "/") + rawSegments := strings.Split(trimmed, "/") + segments := make([]string, 0, len(rawSegments)) + for _, segment := range rawSegments { + if segment == ".." { + return "", invalidRemotePathIssue(i18n.T("err_file_path_invalid", remotePath)) + } + if segment == "" || segment == "." { + continue + } + segments = append(segments, segment) + } + if len(segments) == 0 { + if allowEmpty { + return "", nil + } + return "", invalidRemotePathIssue(i18n.T("err_file_path_invalid", remotePath)) + } + if !isFileRemoteRoot(segments[0]) { + return "", invalidRemotePathIssue(i18n.T("err_file_path_root", remotePath)) + } + return strings.Join(segments, "/"), nil +} + +func invalidRemotePathIssue(message string) error { + return errUsageFromIssues([]usageIssue{{ + Code: errorcodes.ErrInvalidFlagValue, + Message: message, + Field: "remote-path", + }}) +} + +func invalidLocalPathIssue(message string) error { + return errUsageFromIssues([]usageIssue{{ + Code: errorcodes.ErrInvalidFlagValue, + Message: message, + Field: "local-file", + }}) +} + +func localFileError(key, localPath string, cause error) *clierror.Error { + return &clierror.Error{ + Category: errorcodes.CategorySystem, + Code: errorcodes.ErrCommandExecution, + Message: i18n.T(key, localPath, cause.Error()), + Context: map[string]interface{}{ + "path": localPath, + "msg": cause.Error(), + }, + } +} + +func isFileRemoteRoot(value string) bool { + for _, root := range fileRemoteRoots { + if value == root { + return true + } + } + return false +} + +func fileRemoteRootCompleter(ctx completion.Context) ([]completion.Candidate, error) { + prefix := strings.ToLower(completion.CurrentWordPrefix(ctx)) + candidates := make([]completion.Candidate, 0, len(fileRemoteRoots)) + for _, root := range fileRemoteRoots { + if prefix != "" && !strings.HasPrefix(root, prefix) { + continue + } + candidates = append(candidates, completion.Candidate{ + Value: root, + Description: i18n.T("file_root_" + root + "_desc"), + }) + } + return candidates, nil +} + +func fileLocalPathCompleter(ctx completion.Context) ([]completion.Candidate, error) { + prefix := completion.CurrentWordPrefix(ctx) + directoryPrefix, namePrefix := filepath.Split(prefix) + directory := directoryPrefix + if directory == "" { + directory = "." + } + entries, err := os.ReadDir(directory) + if err != nil { + return nil, nil + } + + candidates := make([]completion.Candidate, 0, len(entries)) + for _, entry := range entries { + name := entry.Name() + if !strings.HasPrefix(name, namePrefix) || + (namePrefix == "" && strings.HasPrefix(name, ".")) { + continue + } + fullPath := filepath.Join(directory, name) + info, err := os.Stat(fullPath) + if err != nil { + continue + } + value := directoryPrefix + name + description := i18n.T("file_local_regular_desc") + switch { + case info.IsDir(): + value += string(filepath.Separator) + description = i18n.T("file_local_directory_desc") + case !info.Mode().IsRegular(): + continue + } + candidates = append(candidates, completion.Candidate{ + Value: value, + Description: description, + }) + } + return candidates, nil +} + +func init() { + fileUploadCmd.Flags().Bool("overwrite", false, "Replace an existing remote file") + fileRemoveCmd.Flags().Bool("recursive", false, "Remove a directory and all of its contents") + fileRemoveCmd.Flags().BoolP("yes", "y", false, "Remove without confirmation") + fileCmd.AddCommand(fileMkdirCmd, fileMoveCmd, fileRemoveCmd, fileUploadCmd) + rootCmd.AddCommand(fileCmd) + completion.RegisterPositional([]string{"file", "mkdir"}, 0, fileRemoteRootCompleter) + completion.RegisterPositional([]string{"file", "mv"}, 0, fileRemoteRootCompleter) + completion.RegisterPositional([]string{"file", "mv"}, 1, fileRemoteRootCompleter) + completion.RegisterPositional([]string{"file", "rm"}, 0, fileRemoteRootCompleter) + completion.RegisterPositional([]string{"file", "upload"}, 0, fileLocalPathCompleter) + completion.RegisterPositional([]string{"file", "upload"}, 1, fileRemoteRootCompleter) +} diff --git a/cli/cmd/file_mutation.go b/cli/cmd/file_mutation.go new file mode 100644 index 000000000..24bdf7c88 --- /dev/null +++ b/cli/cmd/file_mutation.go @@ -0,0 +1,198 @@ +package cmd + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "strings" + + "github.com/spf13/cobra" + + "github.com/raids-lab/crater/cli/internal/api" + "github.com/raids-lab/crater/cli/internal/clierror" + "github.com/raids-lab/crater/cli/internal/i18n" + "github.com/raids-lab/crater/cli/internal/output" + "github.com/raids-lab/crater/cli/pkg/errorcodes" +) + +var fileMkdirCmd = &cobra.Command{ + Use: "mkdir ", + Short: "Create one remote directory", + Long: "Create exactly one directory below user, public, or account storage without creating missing parents.", + Args: fileMkdirArgs, + RunE: runFileMkdir, +} + +var fileMoveCmd = &cobra.Command{ + Use: "mv ", + Short: "Move one remote file or directory", + Long: "Move one remote storage entry to an exact destination path without replacing an existing entry.", + Args: fileMoveArgs, + RunE: runFileMove, +} + +type fileMutationDeps struct { + client func() (api.FileMutationClient, error) + stdout io.Writer + json bool +} + +func fileMkdirArgs(cmd *cobra.Command, args []string) error { + return exactFileMutationArgs(cmd, args, []fileArgument{{ + field: "remote-path", + label: i18n.T("file_label_remote_directory"), + }}) +} + +func fileMoveArgs(cmd *cobra.Command, args []string) error { + return exactFileMutationArgs(cmd, args, []fileArgument{ + {field: "source-path", label: i18n.T("file_label_source_path")}, + {field: "destination-path", label: i18n.T("file_label_destination_path")}, + }) +} + +type fileArgument struct { + field string + label string +} + +func exactFileMutationArgs(cmd *cobra.Command, args []string, required []fileArgument) error { + if len(args) > len(required) { + return errTooManyArgs(cmd, len(args), len(required)) + } + if len(args) < len(required) { + missing := required[len(args)] + return errUsageFromIssues([]usageIssue{{ + Code: errorcodes.ErrMissingRequiredFlag, + Message: i18n.T("err_missing_required_arg", missing.label, missing.field), + Field: missing.field, + }}) + } + return nil +} + +func runFileMkdir(cmd *cobra.Command, args []string) error { + return runFileMkdirWith(cmd.Context(), args, fileMutationDeps{ + client: activeFileMutationClient, + stdout: os.Stdout, + json: outputJSON, + }) +} + +func runFileMove(cmd *cobra.Command, args []string) error { + return runFileMoveWith(cmd.Context(), args, fileMutationDeps{ + client: activeFileMutationClient, + stdout: os.Stdout, + json: outputJSON, + }) +} + +func activeFileMutationClient() (api.FileMutationClient, error) { + return activeAPIClient() +} + +func runFileMkdirWith(ctx context.Context, args []string, deps fileMutationDeps) error { + remotePath, issue := normalizeMutationPath(args[0], "remote-path") + if issue != nil { + return errUsageFromIssues([]usageIssue{*issue}) + } + client, err := deps.client() + if err != nil { + return err + } + if err := client.CreateDirectory(ctx, remotePath); err != nil { + return cliErrFromAPI(err) + } + return writeFileMutationResult( + deps.stdout, + deps.json, + map[string]interface{}{"remote_path": remotePath}, + i18n.T("file_mkdir_success", remotePath), + ) +} + +func runFileMoveWith(ctx context.Context, args []string, deps fileMutationDeps) error { + sourcePath, sourceIssue := normalizeMutationPath(args[0], "source-path") + destinationPath, destinationIssue := normalizeMutationPath(args[1], "destination-path") + issues := make([]usageIssue, 0, 2) + if sourceIssue != nil { + issues = append(issues, *sourceIssue) + } + if destinationIssue != nil { + issues = append(issues, *destinationIssue) + } + if len(issues) > 0 { + return errUsageFromIssues(issues) + } + switch { + case sourcePath == destinationPath: + return errUsageFromIssues([]usageIssue{invalidIssue( + "destination-path", + i18n.T("err_file_move_same", destinationPath), + )}) + case strings.HasPrefix(destinationPath, sourcePath+"/"): + return errUsageFromIssues([]usageIssue{invalidIssue( + "destination-path", + i18n.T("err_file_move_descendant", destinationPath, sourcePath), + )}) + } + + client, err := deps.client() + if err != nil { + return err + } + if err := client.MoveFile(ctx, sourcePath, destinationPath); err != nil { + return cliErrFromAPI(err) + } + return writeFileMutationResult( + deps.stdout, + deps.json, + map[string]interface{}{ + "source_path": sourcePath, + "destination_path": destinationPath, + }, + i18n.T("file_move_success", sourcePath, destinationPath), + ) +} + +func normalizeMutationPath(rawPath, field string) (string, *usageIssue) { + normalized, err := normalizeRemotePath(rawPath, false) + if err != nil { + var cliErr *clierror.Error + code := errorcodes.ErrInvalidFlagValue + message := err.Error() + if errors.As(err, &cliErr) { + code = cliErr.Code + message = cliErr.Message + } + issue := usageIssue{Code: code, Message: message, Field: field} + return "", &issue + } + if len(strings.Split(normalized, "/")) < 2 { + issue := invalidIssue(field, i18n.T("err_file_path_not_entry", rawPath)) + return "", &issue + } + return normalized, nil +} + +func writeFileMutationResult( + writer io.Writer, + jsonOutput bool, + data map[string]interface{}, + humanMessage string, +) error { + if jsonOutput { + return output.WriteSuccessJSON(writer, output.SuccessEnvelope(data)) + } + if _, err := fmt.Fprintln(writer, humanMessage); err != nil { + return &clierror.Error{ + Category: errorcodes.CategorySystem, + Code: errorcodes.ErrCommandExecution, + Message: i18n.T("err_file_output", err.Error()), + Context: map[string]interface{}{"msg": err.Error()}, + } + } + return nil +} diff --git a/cli/cmd/file_mutation_test.go b/cli/cmd/file_mutation_test.go new file mode 100644 index 000000000..9bdc3cb87 --- /dev/null +++ b/cli/cmd/file_mutation_test.go @@ -0,0 +1,298 @@ +package cmd + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "reflect" + "strings" + "testing" + + "github.com/spf13/cobra" + + "github.com/raids-lab/crater/cli/internal/api" + "github.com/raids-lab/crater/cli/internal/clierror" + "github.com/raids-lab/crater/cli/internal/completion" + "github.com/raids-lab/crater/cli/internal/i18n" + "github.com/raids-lab/crater/cli/pkg/errorcodes" +) + +type fakeFileMutationClient struct { + createDirectory func(context.Context, string) error + moveFile func(context.Context, string, string) error +} + +func (client fakeFileMutationClient) CreateDirectory(ctx context.Context, remotePath string) error { + return client.createDirectory(ctx, remotePath) +} + +func (client fakeFileMutationClient) MoveFile( + ctx context.Context, + sourcePath string, + destinationPath string, +) error { + return client.moveFile(ctx, sourcePath, destinationPath) +} + +func TestFileMutationArgs(t *testing.T) { + mkdir := &cobra.Command{Use: "mkdir"} + if err := fileMkdirArgs(mkdir, nil); err == nil { + t.Fatal("missing mkdir path should fail") + } + if err := fileMkdirArgs(mkdir, []string{"user/results"}); err != nil { + t.Fatalf("valid mkdir args: %v", err) + } + if err := fileMkdirArgs(mkdir, []string{"a", "b"}); err == nil { + t.Fatal("extra mkdir arg should fail") + } + + move := &cobra.Command{Use: "mv"} + if err := fileMoveArgs(move, nil); err == nil { + t.Fatal("missing move source should fail") + } + if err := fileMoveArgs(move, []string{"user/source"}); err == nil { + t.Fatal("missing move destination should fail") + } + if err := fileMoveArgs(move, []string{"user/source", "user/destination"}); err != nil { + t.Fatalf("valid move args: %v", err) + } + if err := fileMoveArgs(move, []string{"a", "b", "c"}); err == nil { + t.Fatal("extra move arg should fail") + } +} + +func TestRunFileMkdirNormalizesPathAndWritesHumanOutput(t *testing.T) { + var stdout bytes.Buffer + var gotPath string + err := runFileMkdirWith( + context.Background(), + []string{"/user//runs/./new directory/"}, + fileMutationDeps{ + client: func() (api.FileMutationClient, error) { + return fakeFileMutationClient{ + createDirectory: func(_ context.Context, remotePath string) error { + gotPath = remotePath + return nil + }, + }, nil + }, + stdout: &stdout, + }, + ) + if err != nil { + t.Fatalf("runFileMkdirWith: %v", err) + } + if gotPath != "user/runs/new directory" { + t.Fatalf("remote path = %q", gotPath) + } + if !strings.Contains(stdout.String(), gotPath) { + t.Fatalf("stdout = %q", stdout.String()) + } +} + +func TestRunFileMkdirWritesJSON(t *testing.T) { + var stdout bytes.Buffer + err := runFileMkdirWith( + context.Background(), + []string{"account/results"}, + fileMutationDeps{ + client: func() (api.FileMutationClient, error) { + return fakeFileMutationClient{ + createDirectory: func(context.Context, string) error { return nil }, + }, nil + }, + stdout: &stdout, + json: true, + }, + ) + if err != nil { + t.Fatalf("runFileMkdirWith: %v", err) + } + var envelope struct { + Status string `json:"status"` + Data struct { + RemotePath string `json:"remote_path"` + } `json:"data"` + } + if err := json.Unmarshal(stdout.Bytes(), &envelope); err != nil { + t.Fatalf("decode stdout: %v\n%s", err, stdout.String()) + } + if envelope.Status != "OK" || envelope.Data.RemotePath != "account/results" { + t.Fatalf("envelope = %#v", envelope) + } +} + +func TestRunFileMoveNormalizesPathsAndWritesJSON(t *testing.T) { + var stdout bytes.Buffer + var gotSource, gotDestination string + err := runFileMoveWith( + context.Background(), + []string{"/user//runs/./source", "account/results/目标 file"}, + fileMutationDeps{ + client: func() (api.FileMutationClient, error) { + return fakeFileMutationClient{ + moveFile: func(_ context.Context, sourcePath, destinationPath string) error { + gotSource = sourcePath + gotDestination = destinationPath + return nil + }, + }, nil + }, + stdout: &stdout, + json: true, + }, + ) + if err != nil { + t.Fatalf("runFileMoveWith: %v", err) + } + if gotSource != "user/runs/source" || gotDestination != "account/results/目标 file" { + t.Fatalf("source=%q destination=%q", gotSource, gotDestination) + } + var envelope struct { + Status string `json:"status"` + Data struct { + Source string `json:"source_path"` + Destination string `json:"destination_path"` + } `json:"data"` + } + if err := json.Unmarshal(stdout.Bytes(), &envelope); err != nil { + t.Fatalf("decode stdout: %v\n%s", err, stdout.String()) + } + if envelope.Status != "OK" || + envelope.Data.Source != gotSource || + envelope.Data.Destination != gotDestination { + t.Fatalf("envelope = %#v", envelope) + } +} + +func TestFileMutationRejectsInvalidPathsBeforeClient(t *testing.T) { + for _, test := range []struct { + name string + run func(fileMutationDeps) error + }{ + { + name: "mkdir logical root", + run: func(deps fileMutationDeps) error { + return runFileMkdirWith(context.Background(), []string{"user"}, deps) + }, + }, + { + name: "move same path", + run: func(deps fileMutationDeps) error { + return runFileMoveWith( + context.Background(), + []string{"user/source", "/user//./source"}, + deps, + ) + }, + }, + { + name: "move below source", + run: func(deps fileMutationDeps) error { + return runFileMoveWith( + context.Background(), + []string{"user/source", "user/source/nested"}, + deps, + ) + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + factoryCalls := 0 + err := test.run(fileMutationDeps{ + client: func() (api.FileMutationClient, error) { + factoryCalls++ + return nil, errors.New("must not be called") + }, + stdout: io.Discard, + }) + if err == nil { + t.Fatal("invalid path should fail") + } + if factoryCalls != 0 { + t.Fatalf("factory calls = %d, want 0", factoryCalls) + } + }) + } +} + +func TestRunFileMoveAggregatesInvalidOperandsBeforeClient(t *testing.T) { + factoryCalls := 0 + err := runFileMoveWith( + context.Background(), + []string{"user/../public/source", `account\destination`}, + fileMutationDeps{ + client: func() (api.FileMutationClient, error) { + factoryCalls++ + return nil, errors.New("must not be called") + }, + stdout: io.Discard, + }, + ) + if factoryCalls != 0 { + t.Fatalf("factory calls = %d, want 0", factoryCalls) + } + var cliErr *clierror.Error + if !errors.As(err, &cliErr) || cliErr.Category != errorcodes.CategoryUsage { + t.Fatalf("error = %T %v, want usage error", err, err) + } + issues, ok := cliErr.Context["issues"].([]map[string]interface{}) + if !ok || len(issues) != 2 { + t.Fatalf("issues = %#v, want two", cliErr.Context["issues"]) + } + if issues[0]["field"] != "source-path" || issues[1]["field"] != "destination-path" { + t.Fatalf("issues = %#v", issues) + } +} + +func TestFileMutationAPIFailureWritesNoSuccessOutput(t *testing.T) { + var stdout bytes.Buffer + err := runFileMoveWith( + context.Background(), + []string{"user/source", "user/destination"}, + fileMutationDeps{ + client: func() (api.FileMutationClient, error) { + return fakeFileMutationClient{ + moveFile: func(context.Context, string, string) error { + return &api.RequestError{ + HTTPStatus: 409, + CraterCode: 40901, + Msg: "destination path already exists", + } + }, + }, nil + }, + stdout: &stdout, + }, + ) + if err == nil { + t.Fatal("API conflict should fail") + } + if stdout.Len() != 0 { + t.Fatalf("stdout = %q, want empty", stdout.String()) + } + var cliErr *clierror.Error + if !errors.As(err, &cliErr) || cliErr.Category != errorcodes.CategoryAPI { + t.Fatalf("error = %T %v, want API error", err, err) + } +} + +func TestFileMutationRootCompletion(t *testing.T) { + i18n.SetLanguage("en") + candidates, err := fileRemoteRootCompleter(completion.Context{ + Words: []string{"crater", "file", "mv", "user/source", "p"}, + Current: 5, + }) + if err != nil { + t.Fatal(err) + } + got := make([]string, len(candidates)) + for index := range candidates { + got[index] = candidates[index].Value + } + if !reflect.DeepEqual(got, []string{"public"}) { + t.Fatalf("candidate values = %#v", got) + } +} diff --git a/cli/cmd/file_open_other.go b/cli/cmd/file_open_other.go new file mode 100644 index 000000000..818be74c8 --- /dev/null +++ b/cli/cmd/file_open_other.go @@ -0,0 +1,9 @@ +//go:build !aix && !darwin && !dragonfly && !freebsd && !linux && !netbsd && !openbsd && !solaris + +package cmd + +import "os" + +func openUploadFileNoBlock(path string) (*os.File, error) { + return os.Open(path) +} diff --git a/cli/cmd/file_open_unix.go b/cli/cmd/file_open_unix.go new file mode 100644 index 000000000..90c74e547 --- /dev/null +++ b/cli/cmd/file_open_unix.go @@ -0,0 +1,12 @@ +//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris + +package cmd + +import ( + "os" + "syscall" +) + +func openUploadFileNoBlock(path string) (*os.File, error) { + return os.OpenFile(path, os.O_RDONLY|syscall.O_NONBLOCK, 0) +} diff --git a/cli/cmd/file_open_unix_test.go b/cli/cmd/file_open_unix_test.go new file mode 100644 index 000000000..fbc2dbb3f --- /dev/null +++ b/cli/cmd/file_open_unix_test.go @@ -0,0 +1,34 @@ +//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris + +package cmd + +import ( + "syscall" + "testing" + "time" +) + +func TestOpenUploadSourceRejectsFIFOWithoutBlocking(t *testing.T) { + path := t.TempDir() + "/source.pipe" + if err := syscall.Mkfifo(path, 0o600); err != nil { + t.Fatal(err) + } + + done := make(chan error, 1) + go func() { + source, err := openUploadSource(path) + if source != nil { + _ = source.Close() + } + done <- err + }() + + select { + case err := <-done: + if err == nil { + t.Fatal("FIFO source should fail") + } + case <-time.After(time.Second): + t.Fatal("opening a FIFO blocked instead of rejecting it") + } +} diff --git a/cli/cmd/file_remove.go b/cli/cmd/file_remove.go new file mode 100644 index 000000000..72267847e --- /dev/null +++ b/cli/cmd/file_remove.go @@ -0,0 +1,155 @@ +package cmd + +import ( + "context" + "io" + "os" + "strings" + "unicode" + + "github.com/AlecAivazis/survey/v2" + "github.com/spf13/cobra" + + "github.com/raids-lab/crater/cli/internal/api" + "github.com/raids-lab/crater/cli/internal/clierror" + "github.com/raids-lab/crater/cli/internal/i18n" + "github.com/raids-lab/crater/cli/pkg/errorcodes" +) + +var fileRemoveCmd = &cobra.Command{ + Use: "rm ", + Short: "Remove one remote file or directory", + Long: "Remove exactly one remote entry below user, public, or account storage. Directories require --recursive.", + Args: fileRemoveArgs, + RunE: runFileRemove, +} + +type fileRemoveDeps struct { + client func() (api.FileRemoveClient, error) + confirm func(string) (bool, error) + stdout io.Writer + json bool + noInteractive bool +} + +func fileRemoveArgs(cmd *cobra.Command, args []string) error { + return exactFileMutationArgs(cmd, args, []fileArgument{{ + field: "remote-path", + label: i18n.T("file_label_remote_path"), + }}) +} + +func runFileRemove(cmd *cobra.Command, args []string) error { + return runFileRemoveWith(cmd.Context(), cmd, args, fileRemoveDeps{ + client: activeFileRemoveClient, + confirm: confirmFileRemove, + stdout: os.Stdout, + json: outputJSON, + noInteractive: noInteractive, + }) +} + +func activeFileRemoveClient() (api.FileRemoveClient, error) { + return activeAPIClient() +} + +func runFileRemoveWith( + ctx context.Context, + cmd *cobra.Command, + args []string, + deps fileRemoveDeps, +) error { + remotePath, issue := normalizeRemovePath(args[0]) + if issue != nil { + return errUsageFromIssues([]usageIssue{*issue}) + } + + recursive, _ := cmd.Flags().GetBool("recursive") + yes, _ := cmd.Flags().GetBool("yes") + if !yes { + if deps.json || deps.noInteractive { + return &clierror.Error{ + Category: errorcodes.CategoryUsage, + Code: errorcodes.ErrMissingRequiredFlag, + Message: i18n.T("err_confirm_required"), + } + } + confirmKey := "file_remove_confirm" + if recursive { + confirmKey = "file_remove_recursive_confirm" + } + confirmed, err := deps.confirm(i18n.T(confirmKey, remotePath)) + if err != nil { + return errSurveyOrSame(err) + } + if !confirmed { + return errOperationCancelled() + } + } + + client, err := deps.client() + if err != nil { + return err + } + result, err := client.RemoveFile(ctx, remotePath, recursive) + if err != nil { + return cliErrFromAPI(err) + } + return writeFileMutationResult( + deps.stdout, + deps.json, + map[string]interface{}{ + "remote_path": result.RemotePath, + "recursive": result.Recursive, + }, + i18n.T("file_remove_success", result.RemotePath), + ) +} + +func confirmFileRemove(message string) (bool, error) { + var confirmed bool + prompt := &survey.Confirm{Message: message, Default: false} + if err := survey.AskOne(prompt, &confirmed); err != nil { + return false, err + } + return confirmed, nil +} + +// normalizeRemovePath is intentionally stricter than the shared file path +// normalizer: deletion rejects every raw "." or ".." segment instead of +// silently canonicalizing it. +func normalizeRemovePath(rawPath string) (string, *usageIssue) { + invalid := func(message string) (string, *usageIssue) { + issue := invalidIssue("remote-path", message) + return "", &issue + } + if rawPath == "" || strings.ContainsRune(rawPath, '\\') { + return invalid(i18n.T("err_file_path_invalid", rawPath)) + } + for _, character := range rawPath { + if unicode.IsControl(character) { + return invalid(i18n.T("err_file_path_invalid", rawPath)) + } + } + + rawSegments := strings.Split(strings.Trim(rawPath, "/"), "/") + segments := make([]string, 0, len(rawSegments)) + for _, segment := range rawSegments { + if segment == "." || segment == ".." { + return invalid(i18n.T("err_file_remove_ambiguous_path", rawPath)) + } + if segment != "" { + segments = append(segments, segment) + } + } + if len(segments) == 0 { + return invalid(i18n.T("err_file_path_invalid", rawPath)) + } + if !isFileRemoteRoot(segments[0]) { + return invalid(i18n.T("err_file_path_root", rawPath)) + } + if len(segments) < 2 { + return invalid(i18n.T("err_file_remove_root", rawPath)) + } + return strings.Join(segments, "/"), nil +} diff --git a/cli/cmd/file_remove_test.go b/cli/cmd/file_remove_test.go new file mode 100644 index 000000000..1dbbb6bcd --- /dev/null +++ b/cli/cmd/file_remove_test.go @@ -0,0 +1,421 @@ +package cmd + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "strings" + "testing" + + "github.com/AlecAivazis/survey/v2/terminal" + "github.com/spf13/cobra" + + "github.com/raids-lab/crater/cli/internal/api" + "github.com/raids-lab/crater/cli/internal/clierror" + "github.com/raids-lab/crater/cli/internal/i18n" + "github.com/raids-lab/crater/cli/pkg/errorcodes" +) + +type fakeFileRemoveClient struct { + removeFile func(context.Context, string, bool) (api.FileRemoveResult, error) +} + +func (client fakeFileRemoveClient) RemoveFile( + ctx context.Context, + remotePath string, + recursive bool, +) (api.FileRemoveResult, error) { + return client.removeFile(ctx, remotePath, recursive) +} + +func newFileRemoveTestCommand(t *testing.T, recursive, yes bool) *cobra.Command { + t.Helper() + cmd := &cobra.Command{Use: "rm"} + cmd.Flags().Bool("recursive", false, "") + cmd.Flags().BoolP("yes", "y", false, "") + if recursive { + if err := cmd.Flags().Set("recursive", "true"); err != nil { + t.Fatal(err) + } + } + if yes { + if err := cmd.Flags().Set("yes", "true"); err != nil { + t.Fatal(err) + } + } + return cmd +} + +func TestFileRemoveArgs(t *testing.T) { + cmd := &cobra.Command{Use: "rm"} + if err := fileRemoveArgs(cmd, nil); err == nil { + t.Fatal("missing remote path should fail") + } + if err := fileRemoveArgs(cmd, []string{"user/results.txt"}); err != nil { + t.Fatalf("valid arguments: %v", err) + } + if err := fileRemoveArgs(cmd, []string{"user/a", "user/b"}); err == nil { + t.Fatal("extra argument should fail") + } +} + +func TestRunFileRemoveConfirmsNormalizedPathBeforeCreatingClient(t *testing.T) { + i18n.SetLanguage("en") + var stdout bytes.Buffer + var events []string + var gotPath string + var gotRecursive bool + cmd := newFileRemoveTestCommand(t, true, false) + + err := runFileRemoveWith( + context.Background(), + cmd, + []string{"/user//runs/result file.txt/"}, + fileRemoveDeps{ + confirm: func(message string) (bool, error) { + events = append(events, "confirm") + if message != `Recursively remove remote entry "user/runs/result file.txt"?` { + t.Fatalf("confirmation = %q", message) + } + return true, nil + }, + client: func() (api.FileRemoveClient, error) { + events = append(events, "client") + return fakeFileRemoveClient{ + removeFile: func( + _ context.Context, + remotePath string, + recursive bool, + ) (api.FileRemoveResult, error) { + events = append(events, "remove") + gotPath = remotePath + gotRecursive = recursive + return api.FileRemoveResult{ + RemotePath: remotePath, + Recursive: recursive, + }, nil + }, + }, nil + }, + stdout: &stdout, + }, + ) + if err != nil { + t.Fatalf("runFileRemoveWith: %v", err) + } + if strings.Join(events, ",") != "confirm,client,remove" { + t.Fatalf("events = %#v", events) + } + if gotPath != "user/runs/result file.txt" || !gotRecursive { + t.Fatalf("path = %q recursive = %t", gotPath, gotRecursive) + } + if !strings.Contains(stdout.String(), gotPath) { + t.Fatalf("stdout = %q", stdout.String()) + } +} + +func TestRunFileRemoveYesSkipsConfirmationAndWritesJSON(t *testing.T) { + var stdout bytes.Buffer + confirmCalls := 0 + cmd := newFileRemoveTestCommand(t, false, true) + + err := runFileRemoveWith( + context.Background(), + cmd, + []string{"account/output.txt"}, + fileRemoveDeps{ + confirm: func(string) (bool, error) { + confirmCalls++ + return false, errors.New("must not be called") + }, + client: func() (api.FileRemoveClient, error) { + return fakeFileRemoveClient{ + removeFile: func( + _ context.Context, + remotePath string, + recursive bool, + ) (api.FileRemoveResult, error) { + return api.FileRemoveResult{ + RemotePath: remotePath, + Recursive: recursive, + }, nil + }, + }, nil + }, + stdout: &stdout, + json: true, + }, + ) + if err != nil { + t.Fatalf("runFileRemoveWith: %v", err) + } + if confirmCalls != 0 { + t.Fatalf("confirmation calls = %d", confirmCalls) + } + var envelope struct { + Status string `json:"status"` + Data struct { + RemotePath string `json:"remote_path"` + Recursive bool `json:"recursive"` + } `json:"data"` + } + if err := json.Unmarshal(stdout.Bytes(), &envelope); err != nil { + t.Fatalf("decode stdout: %v\n%s", err, stdout.String()) + } + if envelope.Status != "OK" || + envelope.Data.RemotePath != "account/output.txt" || + envelope.Data.Recursive { + t.Fatalf("envelope = %#v", envelope) + } +} + +func TestRunFileRemoveNonInteractiveRequiresYesBeforeClient(t *testing.T) { + for _, test := range []struct { + name string + json bool + noInteractive bool + }{ + {name: "JSON", json: true}, + {name: "no interactive", noInteractive: true}, + } { + t.Run(test.name, func(t *testing.T) { + clientCalls := 0 + confirmCalls := 0 + err := runFileRemoveWith( + context.Background(), + newFileRemoveTestCommand(t, false, false), + []string{"user/output.txt"}, + fileRemoveDeps{ + client: func() (api.FileRemoveClient, error) { + clientCalls++ + return nil, errors.New("must not be called") + }, + confirm: func(string) (bool, error) { + confirmCalls++ + return true, nil + }, + stdout: io.Discard, + json: test.json, + noInteractive: test.noInteractive, + }, + ) + var cliErr *clierror.Error + if !errors.As(err, &cliErr) { + t.Fatalf("error = %T %v, want *clierror.Error", err, err) + } + if cliErr.Category != errorcodes.CategoryUsage || + cliErr.Code != errorcodes.ErrMissingRequiredFlag || + cliErr.Message != i18n.T("err_confirm_required") { + t.Fatalf("error = %#v", cliErr) + } + if exitCodeFor(err) != errorcodes.ExitUsage { + t.Fatalf("exit code = %d", exitCodeFor(err)) + } + if clientCalls != 0 || confirmCalls != 0 { + t.Fatalf("client calls = %d, confirmation calls = %d", clientCalls, confirmCalls) + } + }) + } +} + +func TestRunFileRemoveCancellationNeverCreatesClient(t *testing.T) { + for _, test := range []struct { + name string + confirm func(string) (bool, error) + }{ + { + name: "No", + confirm: func(string) (bool, error) { + return false, nil + }, + }, + { + name: "Ctrl-C", + confirm: func(string) (bool, error) { + return false, terminal.InterruptErr + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + clientCalls := 0 + var stdout bytes.Buffer + err := runFileRemoveWith( + context.Background(), + newFileRemoveTestCommand(t, false, false), + []string{"public/output.txt"}, + fileRemoveDeps{ + client: func() (api.FileRemoveClient, error) { + clientCalls++ + return nil, errors.New("must not be called") + }, + confirm: test.confirm, + stdout: &stdout, + }, + ) + var cliErr *clierror.Error + if !errors.As(err, &cliErr) || + cliErr.Category != errorcodes.CategoryCancelled || + cliErr.Code != errorcodes.ErrOperationCancelled { + t.Fatalf("error = %T %#v, want cancellation", err, cliErr) + } + if exitCodeFor(err) != errorcodes.ExitCancelled { + t.Fatalf("exit code = %d", exitCodeFor(err)) + } + if clientCalls != 0 { + t.Fatalf("client calls = %d", clientCalls) + } + if stdout.Len() != 0 { + t.Fatalf("stdout = %q, want empty", stdout.String()) + } + }) + } +} + +func TestRunFileRemoveRejectsUnsafePathsBeforeConfirmationOrClient(t *testing.T) { + for _, remotePath := range []string{ + "", + "/", + ".", + "..", + "user", + "/public/", + "account/.", + "user/../public/file", + "./user/file", + `user\file`, + "user/\nfile", + "admin-user/alice/file", + "admin-public/file", + "admin-account/team/file", + } { + t.Run(strings.ReplaceAll(remotePath, "/", "_"), func(t *testing.T) { + confirmCalls := 0 + clientCalls := 0 + err := runFileRemoveWith( + context.Background(), + newFileRemoveTestCommand(t, false, true), + []string{remotePath}, + fileRemoveDeps{ + confirm: func(string) (bool, error) { + confirmCalls++ + return true, nil + }, + client: func() (api.FileRemoveClient, error) { + clientCalls++ + return nil, errors.New("must not be called") + }, + stdout: io.Discard, + }, + ) + var cliErr *clierror.Error + if !errors.As(err, &cliErr) || + cliErr.Category != errorcodes.CategoryUsage || + cliErr.Code != errorcodes.ErrInvalidFlagValue { + t.Fatalf("path %q: error = %T %#v", remotePath, err, cliErr) + } + if confirmCalls != 0 || clientCalls != 0 { + t.Fatalf( + "path %q: confirmation calls = %d, client calls = %d", + remotePath, + confirmCalls, + clientCalls, + ) + } + }) + } +} + +func TestRunFileRemoveAPIFailureWritesNoSuccessOutput(t *testing.T) { + var stdout bytes.Buffer + err := runFileRemoveWith( + context.Background(), + newFileRemoveTestCommand(t, false, true), + []string{"user/output.txt"}, + fileRemoveDeps{ + client: func() (api.FileRemoveClient, error) { + return fakeFileRemoveClient{ + removeFile: func( + context.Context, + string, + bool, + ) (api.FileRemoveResult, error) { + return api.FileRemoveResult{}, &api.RequestError{ + HTTPStatus: 404, + CraterCode: 40404, + Msg: "storage resource not found", + } + }, + }, nil + }, + confirm: func(string) (bool, error) { + return true, nil + }, + stdout: &stdout, + }, + ) + if err == nil { + t.Fatal("API failure should fail") + } + if stdout.Len() != 0 { + t.Fatalf("stdout = %q, want empty", stdout.String()) + } + var cliErr *clierror.Error + if !errors.As(err, &cliErr) || cliErr.Category != errorcodes.CategoryAPI { + t.Fatalf("error = %T %v, want API error", err, err) + } +} + +func TestFileRemoveCommandFlags(t *testing.T) { + recursive := fileRemoveCmd.Flags().Lookup("recursive") + if recursive == nil { + t.Fatal("--recursive is not registered") + } + yes := fileRemoveCmd.Flags().Lookup("yes") + if yes == nil || yes.Shorthand != "y" { + t.Fatalf("--yes flag = %#v", yes) + } +} + +func TestFileRemoveTranslations(t *testing.T) { + for _, test := range []struct { + language i18n.Language + short string + confirm string + recursive string + success string + }{ + { + language: i18n.En, + short: "Remove one remote file or directory", + confirm: `Remove remote entry "user/result.txt"?`, + recursive: `Recursively remove remote entry "user/result.txt"?`, + success: "Removed remote entry user/result.txt", + }, + { + language: i18n.ZhCN, + short: "删除单个远端文件或目录", + confirm: "确定删除远端条目“user/result.txt”？", + recursive: "确定递归删除远端条目“user/result.txt”？", + success: "已删除远端条目 user/result.txt", + }, + } { + t.Run(string(test.language), func(t *testing.T) { + i18n.SetLanguage(string(test.language)) + if got := i18n.T("file_rm_short"); got != test.short { + t.Fatalf("short = %q", got) + } + if got := i18n.T("file_remove_confirm", "user/result.txt"); got != test.confirm { + t.Fatalf("confirmation = %q", got) + } + if got := i18n.T("file_remove_recursive_confirm", "user/result.txt"); got != test.recursive { + t.Fatalf("recursive confirmation = %q", got) + } + if got := i18n.T("file_remove_success", "user/result.txt"); got != test.success { + t.Fatalf("success = %q", got) + } + }) + } + i18n.SetLanguage("en") +} diff --git a/cli/cmd/file_test.go b/cli/cmd/file_test.go new file mode 100644 index 000000000..7746dd1df --- /dev/null +++ b/cli/cmd/file_test.go @@ -0,0 +1,407 @@ +package cmd + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/raids-lab/crater/cli/internal/api" + "github.com/raids-lab/crater/cli/internal/clierror" + "github.com/raids-lab/crater/cli/internal/completion" + "github.com/raids-lab/crater/cli/pkg/errorcodes" + "github.com/spf13/cobra" +) + +type fakeFileUploadClient struct { + upload func(context.Context, string, io.Reader, bool) (api.FileUploadResult, error) +} + +func (client fakeFileUploadClient) UploadFile( + ctx context.Context, + remotePath string, + source io.Reader, + overwrite bool, +) (api.FileUploadResult, error) { + return client.upload(ctx, remotePath, source, overwrite) +} + +func testFileUploadCommand(t *testing.T, overwrite bool) *cobra.Command { + t.Helper() + command := &cobra.Command{Use: "upload"} + command.Flags().Bool("overwrite", false, "") + if overwrite { + if err := command.Flags().Set("overwrite", "true"); err != nil { + t.Fatal(err) + } + } + return command +} + +func TestFileUploadArgs(t *testing.T) { + command := &cobra.Command{Use: "upload"} + if err := fileUploadArgs(command, nil); err == nil { + t.Fatal("missing local file should fail") + } + if err := fileUploadArgs(command, []string{"local.bin"}); err == nil { + t.Fatal("missing remote file should fail") + } + if err := fileUploadArgs(command, []string{"local.bin", "user/local.bin"}); err != nil { + t.Fatalf("valid args: %v", err) + } + if err := fileUploadArgs(command, []string{"a", "b", "c"}); err == nil { + t.Fatal("extra args should fail") + } +} + +func TestNormalizeRemoteUploadPath(t *testing.T) { + tests := []struct { + name string + input string + want string + wantErr bool + }{ + {name: "nested file", input: "user/results/model.bin", want: "user/results/model.bin"}, + {name: "unicode and spaces", input: "/account/实验 data/结果.bin/", want: "account/实验 data/结果.bin"}, + {name: "safe normalization", input: "public//runs/./out.bin", want: "public/runs/out.bin"}, + {name: "empty", wantErr: true}, + {name: "unknown root", input: "admin/secret.bin", wantErr: true}, + {name: "parent traversal", input: "user/../public/secret.bin", wantErr: true}, + {name: "backslash", input: `user\secret.bin`, wantErr: true}, + {name: "control character", input: "user/file\nname", wantErr: true}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := normalizeRemotePath(test.input, false) + if (err != nil) != test.wantErr { + t.Fatalf("normalizeRemotePath(%q) error = %v, wantErr %v", test.input, err, test.wantErr) + } + if got != test.want { + t.Fatalf("normalizeRemotePath(%q) = %q, want %q", test.input, got, test.want) + } + }) + } +} + +func TestRunFileUploadRejectsInvalidRemoteBeforeClient(t *testing.T) { + factoryCalls := 0 + err := runFileUploadWith( + testFileUploadCommand(t, false), + []string{"does-not-need-to-exist", "user/../public/secret.bin"}, + fileUploadDeps{ + client: func() (api.FileUploadClient, error) { + factoryCalls++ + return nil, errors.New("must not be called") + }, + stdout: io.Discard, + }, + ) + if err == nil { + t.Fatal("invalid remote path should fail") + } + if factoryCalls != 0 { + t.Fatalf("factory calls = %d, want 0", factoryCalls) + } +} + +func TestRunFileUploadRejectsLogicalRootBeforeClient(t *testing.T) { + local := writeUploadFixture(t, []byte("data")) + factoryCalls := 0 + err := runFileUploadWith( + testFileUploadCommand(t, false), + []string{local, "user"}, + fileUploadDeps{ + client: func() (api.FileUploadClient, error) { + factoryCalls++ + return nil, errors.New("must not be called") + }, + stdout: io.Discard, + }, + ) + if err == nil { + t.Fatal("logical root should not be an upload target") + } + if factoryCalls != 0 { + t.Fatalf("factory calls = %d, want 0", factoryCalls) + } +} + +func TestRunFileUploadRejectsNonRegularSourceBeforeClient(t *testing.T) { + factoryCalls := 0 + err := runFileUploadWith( + testFileUploadCommand(t, false), + []string{t.TempDir(), "user/result.bin"}, + fileUploadDeps{ + client: func() (api.FileUploadClient, error) { + factoryCalls++ + return nil, errors.New("must not be called") + }, + stdout: io.Discard, + }, + ) + if err == nil { + t.Fatal("directory source should fail") + } + if factoryCalls != 0 { + t.Fatalf("factory calls = %d, want 0", factoryCalls) + } +} + +func TestRunFileUploadExistingTargetNeedsOverwrite(t *testing.T) { + local := writeUploadFixture(t, []byte("new")) + uploadCalls := 0 + err := runFileUploadWith( + testFileUploadCommand(t, false), + []string{local, "user/results/result.bin"}, + fileUploadDeps{ + client: func() (api.FileUploadClient, error) { + return fakeFileUploadClient{ + upload: func(_ context.Context, remotePath string, _ io.Reader, overwrite bool) (api.FileUploadResult, error) { + uploadCalls++ + if remotePath != "user/results/result.bin" || overwrite { + t.Fatalf("remotePath=%q overwrite=%v", remotePath, overwrite) + } + return api.FileUploadResult{}, &api.RequestError{ + HTTPStatus: 409, + CraterCode: 40901, + Msg: "target file already exists", + } + }, + }, nil + }, + stdout: io.Discard, + }, + ) + if err == nil { + t.Fatal("existing target should fail without --overwrite") + } + if uploadCalls != 1 { + t.Fatalf("upload calls = %d, want 1", uploadCalls) + } + var cliErr *clierror.Error + if !errors.As(err, &cliErr) || cliErr.Category != errorcodes.CategoryAPI { + t.Fatalf("error = %T %v, want API error", err, err) + } +} + +func TestRunFileUploadRejectsRemoteDirectoryEvenWithOverwrite(t *testing.T) { + local := writeUploadFixture(t, []byte("new")) + uploadCalls := 0 + err := runFileUploadWith( + testFileUploadCommand(t, true), + []string{local, "public/results"}, + fileUploadDeps{ + client: func() (api.FileUploadClient, error) { + return fakeFileUploadClient{ + upload: func(context.Context, string, io.Reader, bool) (api.FileUploadResult, error) { + uploadCalls++ + return api.FileUploadResult{}, &api.RequestError{ + HTTPStatus: 409, + CraterCode: 40902, + Msg: "target path is not a regular file", + } + }, + }, nil + }, + stdout: io.Discard, + }, + ) + if err == nil { + t.Fatal("remote directory should fail") + } + if uploadCalls != 1 { + t.Fatalf("upload calls = %d, want 1", uploadCalls) + } +} + +func TestRunFileUploadStreamsBinaryAndWritesJSONMetadata(t *testing.T) { + payload := []byte{0x00, 0xff, 'C', 'L', 'I'} + local := writeUploadFixture(t, payload) + var stdout bytes.Buffer + err := runFileUploadWith( + testFileUploadCommand(t, false), + []string{local, "/account//实验 data/./result.bin"}, + fileUploadDeps{ + client: func() (api.FileUploadClient, error) { + return fakeFileUploadClient{ + upload: func(_ context.Context, remotePath string, source io.Reader, overwrite bool) (api.FileUploadResult, error) { + if remotePath != "account/实验 data/result.bin" { + t.Fatalf("remote path = %q", remotePath) + } + if overwrite { + t.Fatal("overwrite = true, want false") + } + got, readErr := io.ReadAll(source) + if readErr != nil { + return api.FileUploadResult{}, readErr + } + if !bytes.Equal(got, payload) { + t.Fatalf("payload = %v, want %v", got, payload) + } + return api.FileUploadResult{ + RemotePath: remotePath, + Bytes: int64(len(got)), + }, nil + }, + }, nil + }, + stdout: &stdout, + json: true, + }, + ) + if err != nil { + t.Fatalf("runFileUploadWith: %v", err) + } + if bytes.Contains(stdout.Bytes(), payload) { + t.Fatalf("stdout contains binary payload: %q", stdout.Bytes()) + } + var envelope struct { + Status string `json:"status"` + Data struct { + LocalPath string `json:"local_path"` + RemotePath string `json:"remote_path"` + Bytes int64 `json:"bytes"` + Overwrite bool `json:"overwrite"` + Overwritten bool `json:"overwritten"` + } `json:"data"` + } + if err := json.Unmarshal(stdout.Bytes(), &envelope); err != nil { + t.Fatalf("decode stdout: %v\n%s", err, stdout.String()) + } + if envelope.Status != "OK" || envelope.Data.LocalPath != local || + envelope.Data.RemotePath != "account/实验 data/result.bin" || + envelope.Data.Bytes != int64(len(payload)) || envelope.Data.Overwrite { + t.Fatalf("envelope = %#v", envelope) + } +} + +func TestRunFileUploadOverwritePassesExplicitAuthorization(t *testing.T) { + local := writeUploadFixture(t, []byte("replacement")) + overwriteSeen := false + err := runFileUploadWith( + testFileUploadCommand(t, true), + []string{local, "user/result.bin"}, + fileUploadDeps{ + client: func() (api.FileUploadClient, error) { + return fakeFileUploadClient{ + upload: func(_ context.Context, remotePath string, source io.Reader, overwrite bool) (api.FileUploadResult, error) { + overwriteSeen = overwrite + written, readErr := io.Copy(io.Discard, source) + return api.FileUploadResult{ + RemotePath: remotePath, + Bytes: written, + Overwritten: true, + }, readErr + }, + }, nil + }, + stdout: io.Discard, + }, + ) + if err != nil { + t.Fatalf("runFileUploadWith: %v", err) + } + if !overwriteSeen { + t.Fatal("overwrite flag was not passed to API client") + } +} + +func TestRunFileUploadSourceReadErrorIsSystemError(t *testing.T) { + sentinel := errors.New("local read failed") + _, err := uploadRemoteFile( + context.Background(), + func() (api.FileUploadClient, error) { + return fakeFileUploadClient{ + upload: func(context.Context, string, io.Reader, bool) (api.FileUploadResult, error) { + return api.FileUploadResult{}, &api.SourceReadError{Cause: sentinel} + }, + }, nil + }, + bytes.NewReader(nil), + fileUploadInput{ + localPath: "local.bin", + remotePath: "user/remote.bin", + }, + ) + var cliErr *clierror.Error + if !errors.As(err, &cliErr) || cliErr.Category != errorcodes.CategorySystem { + t.Fatalf("error = %T %v, want system cli error", err, err) + } +} + +func TestFileRemoteRootCompleter(t *testing.T) { + candidates, err := fileRemoteRootCompleter(completion.Context{ + Words: []string{"crater", "file", "upload", "local.bin", "a"}, + Current: 5, + }) + if err != nil { + t.Fatal(err) + } + got := make([]string, len(candidates)) + for index := range candidates { + got[index] = candidates[index].Value + } + if !reflect.DeepEqual(got, []string{"account"}) { + t.Fatalf("candidate values = %#v", got) + } +} + +func TestFileLocalPathCompleter(t *testing.T) { + directory := t.TempDir() + if err := os.WriteFile(filepath.Join(directory, "alpha.txt"), []byte("a"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(directory, ".hidden"), []byte("h"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(filepath.Join(directory, "artifacts"), 0o700); err != nil { + t.Fatal(err) + } + + prefix := filepath.Join(directory, "a") + candidates, err := fileLocalPathCompleter(completion.Context{ + Words: []string{"crater", "file", "upload", prefix}, + Current: 4, + }) + if err != nil { + t.Fatal(err) + } + got := make([]string, len(candidates)) + for index := range candidates { + got[index] = candidates[index].Value + } + want := []string{ + filepath.Join(directory, "alpha.txt"), + filepath.Join(directory, "artifacts") + string(filepath.Separator), + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("candidate values = %#v, want %#v", got, want) + } + + hiddenPrefix := filepath.Join(directory, ".h") + candidates, err = fileLocalPathCompleter(completion.Context{ + Words: []string{"crater", "file", "upload", hiddenPrefix}, + Current: 4, + }) + if err != nil { + t.Fatal(err) + } + if len(candidates) != 1 || candidates[0].Value != filepath.Join(directory, ".hidden") { + t.Fatalf("hidden candidates = %#v", candidates) + } +} + +func writeUploadFixture(t *testing.T, data []byte) string { + t.Helper() + path := filepath.Join(t.TempDir(), "fixture.bin") + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + return path +} diff --git a/cli/docs/COMMANDS.md b/cli/docs/COMMANDS.md index 4b75a41c3..51646e8cb 100644 --- a/cli/docs/COMMANDS.md +++ b/cli/docs/COMMANDS.md @@ -772,3 +772,89 @@ This section records the read-only API surface covered by the CLI after the broa - `crater admin cronjobs`: `/api/v1/admin/operations/cronjob`. - `crater admin whitelist`: `/api/v1/admin/operations/whitelist`. - These commands surface existing admin GET APIs only. They do not perform update/delete/reconcile actions. + +--- + +## 8. 远端文件模块 (file) + +本模块面向普通用户访问 storage service 暴露的逻辑文件空间。远端路径不是本机路径，只允许以 `user`、`public` 或 `account` 为首段；CLI 会规范化安全的 `.`、重复分隔符和首尾分隔符，逐段进行 URL 编码，保留合法的空格与非 ASCII 文件名，并在请求前拒绝任何 `..` 段、反斜杠和控制字符。 + +### `crater file mkdir ` + +- **描述**：在远端逻辑文件空间创建一个目录。 +- **位置参数**： + - ``（必填）：`user`、`public` 或 `account` 下的完整目标目录路径，不能只给逻辑根。 +- **处理逻辑**： + - 调用 `MKCOL /api/ss/*path`。 + - 只创建目标目录；父目录必须预先存在，不会递归补齐。 + - 已存在的文件或目录按冲突处理，不会被修改。 + - 权限和目标路径由 storage service 再次校验；CLI 只有在收到精确的 HTTP 201 后才报告成功。 +- **输出格式**： + - 默认模式：展示规范化后的已创建目录路径。 + - `--json`：stdout 仅输出成功信封。 +- **`--json` 的 `data`**：`remote_path`（字符串）。 +- **状态**：[x] Completed + +### `crater file mv ` + +- **描述**：把一个远端文件或目录移动到另一个精确目标路径。 +- **位置参数**： + - ``（必填）：现有文件或目录的完整逻辑路径。 + - ``（必填）：移动后的完整目标路径，不是仅包含目标父目录的路径。 +- **处理逻辑**： + - 调用 `POST /api/ss/move/*source-path`，请求体中的 `dst` 为完整目标路径。 + - 源和目标都必须位于 `user`、`public` 或 `account` 下，且必须具备写权限。 + - 拒绝同路径移动，也拒绝把目录移动到自身的后代路径。 + - 目标父目录必须预先存在，不会自动创建。 + - 不提供覆盖选项；目标已存在时始终返回冲突。服务端仅在文件系统支持原生原子 no-clobber rename 时执行移动；不支持该能力时安全失败，不会退化为可能覆盖并发目标的普通重命名。 +- **输出格式**： + - 默认模式：展示源路径和目标路径。 + - `--json`：stdout 仅输出成功信封。 +- **`--json` 的 `data`**：`source_path`（字符串）、`destination_path`（字符串）。 +- **状态**：[x] Completed + +### `crater file rm ` + +- **描述**：删除普通用户逻辑空间中的一个精确远端路径。 +- **位置参数**： + - ``（必填）：`user`、`public` 或 `account` 下的完整目标路径；不能是逻辑根，也不能包含原始 `.`、`..`、反斜杠、控制字符或平台保留根。 +- **选项**： + - `--recursive`（bool）：允许删除目录及其内容；删除目录时必须显式提供。 + - `--yes, -y`（bool）：跳过交互确认。`--json` 或 `--no-interactive` 模式下必须显式提供。 +- **处理逻辑**： + - 调用专用安全接口 `DELETE /api/ss/files/*path?recursive=`，不会回退到旧的无条件递归删除接口。 + - 交互模式在发送请求前展示规范化后的精确目标，默认选择 No；取消时不创建 API client，也不发送请求。 + - 普通文件和最终 symlink 只删除该条目且不跟随链接；目录只有同时提供 `--recursive --yes` 才允许递归删除。 + - 权限、授权根和条目类型由 storage service 再次校验；删除过程中类型发生变化时安全失败，不会自动升级为递归删除。 + - 仅支持单目标，不支持 glob、批量删除、Trash、恢复或管理员跨用户删除。 +- **输出格式**： + - 默认模式：展示已删除的规范化远端路径。 + - `--json`：stdout 仅输出成功信封；失败时 stdout 为空。 +- **`--json` 的 `data`**：`remote_path`（字符串）、`recursive`（布尔）。 +- **状态**：[x] Completed + +### `crater file upload ` + +- **描述**：把一个本地普通文件流式上传到远端逻辑路径。 +- **位置参数**： + - ``（必填）：本地普通文件。目录、管道、设备和 socket 会在请求前被拒绝。 + - ``（必填）：`user`、`public` 或 `account` 下的完整目标文件路径，不能只给逻辑根。 +- **选项**： + - `--overwrite`（bool）：允许替换已存在的远端普通文件；默认拒绝覆盖。 +- **处理逻辑**： + - 调用 `POST /api/ss/upload/*path?overwrite=`，请求体直接流式读取本地文件，不把完整内容载入内存。 + - storage service 在目标同目录写入临时文件，完成 `chmod`、`sync` 和 `close` 后才发布。新文件通过原子 no-clobber 链接发布；显式覆盖通过同目录原子重命名替换。 + - 服务端是覆盖策略的最终裁决者：即使预检后并发出现同名文件，未指定 `--overwrite` 也不会覆盖；上传失败不会暴露部分新文件或截断旧文件。 + - 父目录必须预先存在，本命令不会自动创建目录。 + - 不支持递归目录、glob、多文件、断点续传、分片或进度条。 +- **输出格式**： + - 默认模式：成功后展示本地路径、远端路径和已上传字节数。 + - `--json`：stdout 仅输出结果元数据，不包含文件内容。 +- **`--json` 的 `data`**： + - `local_path`（字符串）：本地输入路径。 + - `remote_path`（字符串）：规范化后的远端逻辑路径。 + - `bytes`（整数）：服务端完整接收并发布的字节数。 + - `overwrite`（布尔）：本次是否显式启用了覆盖选项。 + - `overwritten`（布尔）：本次是否实际替换了已有普通文件。 +- **兼容性**：安全上传端点随本功能新增。旧 storage service 会返回 404，CLI 不会回退到可能截断文件的旧 WebDAV PUT。 +- **状态**：[x] Completed diff --git a/cli/internal/api/file.go b/cli/internal/api/file.go new file mode 100644 index 000000000..716f166b0 --- /dev/null +++ b/cli/internal/api/file.go @@ -0,0 +1,199 @@ +package api + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/url" + "strconv" + "strings" + + "github.com/imroc/req/v3" +) + +const maxFileErrorBody = 32 << 10 + +// FileUploadClient exposes the ordinary-user APIs needed to upload one file. +type FileUploadClient interface { + UploadFile(ctx context.Context, remotePath string, source io.Reader, overwrite bool) (FileUploadResult, error) +} + +// NewFileUploadClient creates a typed remote-file upload client. +func NewFileUploadClient(baseURL, token string) FileUploadClient { + return NewClient(baseURL).SetToken(token) +} + +// FileUploadResult is the server-confirmed metadata for an atomic upload. +type FileUploadResult struct { + RemotePath string `json:"remote_path"` + Bytes int64 `json:"bytes"` + Overwritten bool `json:"overwritten"` +} + +// SourceReadError identifies a failure reading the caller-owned upload source. +type SourceReadError struct { + Cause error +} + +func (e *SourceReadError) Error() string { + return "read upload source: " + e.Cause.Error() +} + +func (e *SourceReadError) Unwrap() error { + if e == nil { + return nil + } + return e.Cause +} + +// UploadFile streams one source to the storage service without buffering it in +// memory. The dedicated endpoint stages the body and atomically publishes it. +func (c *Client) UploadFile( + ctx context.Context, + remotePath string, + source io.Reader, + overwrite bool, +) (FileUploadResult, error) { + requestPath := FileUploadPath + "/" + escapeRemotePath(remotePath) + tracked := &uploadSourceReader{source: source} + request := c.httpClient.R(). + SetContext(ctx). + SetContentType("application/octet-stream"). + SetQueryParam("overwrite", strconv.FormatBool(overwrite)). + SetBody(tracked). + DisableAutoReadResponse() + + resp, err := request.Post(requestPath) + if err != nil { + if tracked.readErr != nil { + return FileUploadResult{}, &SourceReadError{Cause: tracked.readErr} + } + if resp != nil && resp.Response != nil { + return FileUploadResult{}, uploadProtocolError(resp, "failed to process upload response: "+err.Error()) + } + return FileUploadResult{}, &NetworkError{Cause: err} + } + if resp.Body != nil { + defer resp.Body.Close() + } + if tracked.readErr != nil { + return FileUploadResult{}, &SourceReadError{Cause: tracked.readErr} + } + if !resp.IsSuccessState() { + if resp.Body == nil { + return FileUploadResult{}, &RequestError{ + HTTPStatus: resp.GetStatusCode(), + Msg: http.StatusText(resp.GetStatusCode()), + } + } + return FileUploadResult{}, rawFileRequestError(resp) + } + if resp.Body == nil { + return FileUploadResult{}, uploadProtocolError(resp, "upload response body is empty") + } + body, readErr := io.ReadAll(io.LimitReader(resp.Body, maxFileErrorBody+1)) + if readErr != nil { + return FileUploadResult{}, uploadProtocolError(resp, "failed to read upload response: "+readErr.Error()) + } + if len(body) > maxFileErrorBody { + return FileUploadResult{}, uploadProtocolError(resp, "upload response exceeds size limit") + } + var result Response[FileUploadResult] + if err := json.Unmarshal(body, &result); err != nil { + return FileUploadResult{}, uploadProtocolError(resp, "invalid upload response: "+err.Error()) + } + if err := errorFromResponse(resp, result.Code, result.Message); err != nil { + return FileUploadResult{}, err + } + if result.Data.RemotePath != remotePath { + return FileUploadResult{}, uploadProtocolError(resp, "upload response remote_path does not match the request") + } + if result.Data.Bytes < 0 || result.Data.Bytes != tracked.read { + return FileUploadResult{}, uploadProtocolError(resp, "upload response byte count does not match the streamed source") + } + if result.Data.Overwritten && !overwrite { + return FileUploadResult{}, uploadProtocolError(resp, "server reported an overwrite without client authorization") + } + return result.Data, nil +} + +func uploadProtocolError(resp *req.Response, message string) *RequestError { + status := 0 + if resp != nil && resp.Response != nil { + status = resp.GetStatusCode() + } + return &RequestError{ + HTTPStatus: status, + Msg: message, + } +} + +type uploadSourceReader struct { + source io.Reader + read int64 + readErr error +} + +func (reader *uploadSourceReader) Read(data []byte) (int, error) { + read, err := reader.source.Read(data) + reader.read += int64(read) + if err != nil && !errors.Is(err, io.EOF) { + reader.readErr = err + } + return read, err +} + +func rawFileRequestError(resp *req.Response) error { + if resp == nil || resp.Response == nil { + return &NetworkError{Cause: errors.New("storage service returned no HTTP response")} + } + if resp.Body == nil { + message := http.StatusText(resp.GetStatusCode()) + if message == "" { + message = "storage service returned an empty error response" + } + return &RequestError{ + HTTPStatus: resp.GetStatusCode(), + Msg: message, + } + } + body, readErr := io.ReadAll(io.LimitReader(resp.Body, maxFileErrorBody+1)) + if readErr != nil { + message := http.StatusText(resp.GetStatusCode()) + if message == "" { + message = "failed to read error response" + } + return &RequestError{ + HTTPStatus: resp.GetStatusCode(), + Msg: message + ": " + readErr.Error(), + } + } + if len(body) > maxFileErrorBody { + body = body[:maxFileErrorBody] + } + + var envelope Response[json.RawMessage] + _ = json.Unmarshal(body, &envelope) + message := strings.TrimSpace(envelope.Message) + if message == "" { + message = strings.TrimSpace(strings.ToValidUTF8(string(body), "\uFFFD")) + } + if message == "" { + message = http.StatusText(resp.GetStatusCode()) + } + return &RequestError{ + HTTPStatus: resp.GetStatusCode(), + CraterCode: envelope.Code, + Msg: message, + } +} + +func escapeRemotePath(remotePath string) string { + segments := strings.Split(remotePath, "/") + for index := range segments { + segments[index] = url.PathEscape(segments[index]) + } + return strings.Join(segments, "/") +} diff --git a/cli/internal/api/file_mutation.go b/cli/internal/api/file_mutation.go new file mode 100644 index 000000000..849eed55e --- /dev/null +++ b/cli/internal/api/file_mutation.go @@ -0,0 +1,100 @@ +package api + +import ( + "context" + "encoding/json" + "io" + "net/http" + + "github.com/imroc/req/v3" +) + +// FileMutationClient exposes the ordinary-user APIs for creating directories +// and moving one remote storage entry. +type FileMutationClient interface { + CreateDirectory(ctx context.Context, remotePath string) error + MoveFile(ctx context.Context, sourcePath, destinationPath string) error +} + +// NewFileMutationClient creates a typed remote-storage mutation client. +func NewFileMutationClient(baseURL, token string) FileMutationClient { + return NewClient(baseURL).SetToken(token) +} + +type moveFileRequest struct { + Destination string `json:"dst"` +} + +// CreateDirectory creates exactly one directory. Parent directories are never +// created implicitly, and HTTP 201 is the only accepted success status. +func (c *Client) CreateDirectory(ctx context.Context, remotePath string) error { + requestPath := StoragePrefix + "/" + escapeRemotePath(remotePath) + resp, err := c.httpClient.R(). + SetContext(ctx). + DisableAutoReadResponse(). + Send("MKCOL", requestPath) + if resp != nil && resp.Response != nil && resp.Body != nil { + defer resp.Body.Close() + } + if err != nil { + return fileRequestTransportError(resp, err) + } + if resp.GetStatusCode() != http.StatusCreated { + return rawFileRequestError(resp) + } + return nil +} + +// MoveFile moves one file or directory to the exact destination path. The +// storage service must reject an existing destination. +func (c *Client) MoveFile(ctx context.Context, sourcePath, destinationPath string) error { + requestPath := FileMovePath + "/" + escapeRemotePath(sourcePath) + resp, err := c.httpClient.R(). + SetContext(ctx). + SetContentType("application/json"). + SetBody(moveFileRequest{Destination: destinationPath}). + DisableAutoReadResponse(). + Post(requestPath) + if resp != nil && resp.Response != nil && resp.Body != nil { + defer resp.Body.Close() + } + if err != nil { + return fileRequestTransportError(resp, err) + } + if !resp.IsSuccessState() { + return rawFileRequestError(resp) + } + if resp.Body == nil { + return fileProtocolError(resp, "move response body is empty") + } + body, readErr := io.ReadAll(io.LimitReader(resp.Body, maxFileErrorBody+1)) + if readErr != nil { + return fileProtocolError(resp, "failed to read move response: "+readErr.Error()) + } + if len(body) > maxFileErrorBody { + return fileProtocolError(resp, "move response exceeds size limit") + } + var envelope Response[json.RawMessage] + if err := json.Unmarshal(body, &envelope); err != nil { + return fileProtocolError(resp, "invalid move response: "+err.Error()) + } + return errorFromResponse(resp, envelope.Code, envelope.Message) +} + +func fileRequestTransportError(resp *req.Response, err error) error { + if resp != nil && resp.Response != nil { + return rawFileRequestError(resp) + } + return &NetworkError{Cause: err} +} + +func fileProtocolError(resp *req.Response, message string) *RequestError { + status := 0 + if resp != nil && resp.Response != nil { + status = resp.GetStatusCode() + } + return &RequestError{ + HTTPStatus: status, + Msg: message, + } +} diff --git a/cli/internal/api/file_mutation_test.go b/cli/internal/api/file_mutation_test.go new file mode 100644 index 000000000..c7ce93a98 --- /dev/null +++ b/cli/internal/api/file_mutation_test.go @@ -0,0 +1,212 @@ +package api + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/imroc/req/v3" +) + +func TestCreateDirectoryUsesMKCOLAndEncodesPathSegments(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.Method != "MKCOL" { + t.Errorf("method = %q, want MKCOL", request.Method) + } + if request.URL.EscapedPath() != "/api/ss/user/%E5%AE%9E%E9%AA%8C%20%231" { + t.Errorf("escaped path = %q", request.URL.EscapedPath()) + } + if request.Header.Get("Authorization") != "Bearer secret" { + t.Errorf("authorization = %q", request.Header.Get("Authorization")) + } + writer.WriteHeader(http.StatusCreated) + })) + defer server.Close() + + if err := NewClient(server.URL).SetToken("secret").CreateDirectory( + context.Background(), + "user/实验 #1", + ); err != nil { + t.Fatalf("CreateDirectory: %v", err) + } +} + +func TestCreateDirectoryRequiresHTTP201(t *testing.T) { + for _, test := range []struct { + name string + status int + body string + wantCode int + wantMessage string + }{ + { + name: "unexpected 200", status: http.StatusOK, + wantMessage: "OK", + }, + { + name: "Crater conflict", status: http.StatusConflict, + body: `{"code":40901,"data":null,"msg":"directory already exists"}`, + wantCode: 40901, wantMessage: "directory already exists", + }, + { + name: "WebDAV conflict", status: http.StatusConflict, + body: "parent directory does not exist", + wantMessage: "parent directory does not exist", + }, + } { + t.Run(test.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + writer.WriteHeader(test.status) + _, _ = io.WriteString(writer, test.body) + })) + defer server.Close() + + err := NewClient(server.URL).CreateDirectory(context.Background(), "user/results") + var requestErr *RequestError + if !errors.As(err, &requestErr) { + t.Fatalf("error = %T %v, want *RequestError", err, err) + } + if requestErr.HTTPStatus != test.status || + requestErr.CraterCode != test.wantCode || + requestErr.Msg != test.wantMessage { + t.Fatalf("request error = %#v", requestErr) + } + }) + } +} + +func TestMoveFileUsesExactDestinationAndEncodesSource(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.Method != http.MethodPost { + t.Errorf("method = %q, want POST", request.Method) + } + if request.URL.EscapedPath() != "/api/ss/move/user/%E6%BA%90%20%231.txt" { + t.Errorf("escaped path = %q", request.URL.EscapedPath()) + } + if request.Header.Get("Authorization") != "Bearer secret" { + t.Errorf("authorization = %q", request.Header.Get("Authorization")) + } + var body struct { + Destination string `json:"dst"` + } + if err := json.NewDecoder(request.Body).Decode(&body); err != nil { + t.Errorf("decode body: %v", err) + } + if body.Destination != "account/目标/result.txt" { + t.Errorf("destination = %q", body.Destination) + } + writer.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(writer, `{"code":0,"data":"move files successfully","msg":""}`) + })) + defer server.Close() + + if err := NewClient(server.URL).SetToken("secret").MoveFile( + context.Background(), + "user/源 #1.txt", + "account/目标/result.txt", + ); err != nil { + t.Fatalf("MoveFile: %v", err) + } +} + +func TestMoveFilePreservesServerErrors(t *testing.T) { + for _, test := range []struct { + name string + status int + body string + wantCode int + wantMsg string + }{ + { + name: "forbidden", status: http.StatusForbidden, + body: `{"code":40301,"data":null,"msg":"write permission is required"}`, + wantCode: 40301, wantMsg: "write permission is required", + }, + { + name: "destination exists", status: http.StatusConflict, + body: `{"code":40901,"data":null,"msg":"destination path already exists"}`, + wantCode: 40901, wantMsg: "destination path already exists", + }, + } { + t.Run(test.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + writer.Header().Set("Content-Type", "application/json") + writer.WriteHeader(test.status) + _, _ = io.WriteString(writer, test.body) + })) + defer server.Close() + + err := NewClient(server.URL).MoveFile( + context.Background(), + "user/source", + "user/destination", + ) + var requestErr *RequestError + if !errors.As(err, &requestErr) { + t.Fatalf("error = %T %v, want *RequestError", err, err) + } + if requestErr.HTTPStatus != test.status || + requestErr.CraterCode != test.wantCode || + requestErr.Msg != test.wantMsg { + t.Fatalf("request error = %#v", requestErr) + } + }) + } +} + +func TestMoveFileRejectsInvalidSuccessResponseWithHTTPStatus(t *testing.T) { + for _, body := range []string{ + "", + `{"code":`, + `{"code":40901,"data":null,"msg":"unexpected conflict"}`, + strings.Repeat("x", maxFileErrorBody+1), + } { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + writer.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(writer, body) + })) + + err := NewClient(server.URL).MoveFile(context.Background(), "user/source", "user/destination") + server.Close() + var requestErr *RequestError + if !errors.As(err, &requestErr) { + t.Fatalf("body %q: error = %T %v, want *RequestError", body, err, err) + } + if requestErr.HTTPStatus != http.StatusOK { + t.Fatalf("body %q: status = %d, want 200", body, requestErr.HTTPStatus) + } + } +} + +func TestFileMutationNetworkErrorsHaveNoHTTPStatus(t *testing.T) { + sentinel := errors.New("network unavailable") + client := NewClient("https://example.invalid") + client.httpClient.GetTransport().WrapRoundTripFunc(func(_ http.RoundTripper) req.HttpRoundTripFunc { + return func(*http.Request) (*http.Response, error) { + return nil, sentinel + } + }) + + for _, call := range []func() error{ + func() error { + return client.CreateDirectory(context.Background(), "user/results") + }, + func() error { + return client.MoveFile(context.Background(), "user/source", "user/destination") + }, + } { + err := call() + var networkErr *NetworkError + if !errors.As(err, &networkErr) { + t.Fatalf("error = %T %v, want *NetworkError", err, err) + } + if !errors.Is(networkErr, sentinel) { + t.Fatalf("error = %v, want sentinel", networkErr) + } + } +} diff --git a/cli/internal/api/file_remove.go b/cli/internal/api/file_remove.go new file mode 100644 index 000000000..8889100c3 --- /dev/null +++ b/cli/internal/api/file_remove.go @@ -0,0 +1,119 @@ +package api + +import ( + "context" + "encoding/json" + "io" + "net/http" + "strconv" + + "github.com/imroc/req/v3" +) + +// FileRemoveClient exposes the ordinary-user API for removing one remote +// storage entry. +type FileRemoveClient interface { + RemoveFile(ctx context.Context, remotePath string, recursive bool) (FileRemoveResult, error) +} + +// FileRemoveResult is the server-confirmed metadata for one removal. +type FileRemoveResult struct { + RemotePath string `json:"remote_path"` + Recursive bool `json:"recursive"` +} + +// NewFileRemoveClient creates a typed remote-file removal client. +func NewFileRemoveClient(baseURL, token string) FileRemoveClient { + return NewClient(baseURL).SetToken(token) +} + +type removeFileResponseData struct { + RemotePath *string `json:"remote_path"` + Recursive *bool `json:"recursive"` +} + +type removeFileResponse struct { + Code *int `json:"code"` + Data removeFileResponseData `json:"data"` + Message string `json:"msg"` +} + +// RemoveFile removes exactly one remote entry. Directories are accepted only +// when recursive is true. This method intentionally has no fallback to the +// legacy, unconditional /delete endpoint. +func (c *Client) RemoveFile( + ctx context.Context, + remotePath string, + recursive bool, +) (FileRemoveResult, error) { + requestPath := FileRemovePath + "/" + escapeRemotePath(remotePath) + resp, err := c.httpClient.R(). + SetContext(ctx). + SetQueryParam("recursive", strconv.FormatBool(recursive)). + DisableAutoReadResponse(). + Delete(requestPath) + if resp != nil && resp.Response != nil && resp.Body != nil { + defer resp.Body.Close() + } + if err != nil { + return FileRemoveResult{}, fileRequestTransportError(resp, err) + } + if resp.GetStatusCode() != http.StatusOK { + return FileRemoveResult{}, rawFileRequestError(resp) + } + if resp.Body == nil { + return FileRemoveResult{}, fileRemoveProtocolError(resp, "remove response body is empty") + } + + body, readErr := io.ReadAll(io.LimitReader(resp.Body, maxFileErrorBody+1)) + if readErr != nil { + return FileRemoveResult{}, fileRemoveProtocolError( + resp, + "failed to read remove response: "+readErr.Error(), + ) + } + if len(body) > maxFileErrorBody { + return FileRemoveResult{}, fileRemoveProtocolError(resp, "remove response exceeds size limit") + } + + var envelope removeFileResponse + if err := json.Unmarshal(body, &envelope); err != nil { + return FileRemoveResult{}, fileRemoveProtocolError( + resp, + "invalid remove response: "+err.Error(), + ) + } + if envelope.Code == nil { + return FileRemoveResult{}, fileRemoveProtocolError(resp, "remove response code is missing") + } + if err := errorFromResponse(resp, *envelope.Code, envelope.Message); err != nil { + return FileRemoveResult{}, err + } + if envelope.Data.RemotePath == nil || *envelope.Data.RemotePath != remotePath { + return FileRemoveResult{}, fileRemoveProtocolError( + resp, + "remove response remote_path does not match the request", + ) + } + if envelope.Data.Recursive == nil || *envelope.Data.Recursive != recursive { + return FileRemoveResult{}, fileRemoveProtocolError( + resp, + "remove response recursive value does not match the request", + ) + } + return FileRemoveResult{ + RemotePath: *envelope.Data.RemotePath, + Recursive: *envelope.Data.Recursive, + }, nil +} + +func fileRemoveProtocolError(resp *req.Response, message string) *RequestError { + status := 0 + if resp != nil && resp.Response != nil { + status = resp.GetStatusCode() + } + return &RequestError{ + HTTPStatus: status, + Msg: message, + } +} diff --git a/cli/internal/api/file_remove_test.go b/cli/internal/api/file_remove_test.go new file mode 100644 index 000000000..6846aeba4 --- /dev/null +++ b/cli/internal/api/file_remove_test.go @@ -0,0 +1,229 @@ +package api + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + + "github.com/imroc/req/v3" +) + +func TestRemoveFileUsesDedicatedEndpointAndEncodesEachPathSegment(t *testing.T) { + for _, recursive := range []bool{false, true} { + t.Run("recursive="+strconv.FormatBool(recursive), func(t *testing.T) { + recursiveValue := strconv.FormatBool(recursive) + requests := 0 + server := httptest.NewServer(http.HandlerFunc( + func(writer http.ResponseWriter, request *http.Request) { + requests++ + if request.Method != http.MethodDelete { + t.Errorf("method = %q, want DELETE", request.Method) + } + if request.URL.EscapedPath() != + "/api/ss/files/user/%E5%AE%9E%E9%AA%8C%20%231/result%3F.txt" { + t.Errorf("escaped path = %q", request.URL.EscapedPath()) + } + if request.URL.RawQuery != "recursive="+recursiveValue { + t.Errorf("raw query = %q", request.URL.RawQuery) + } + if request.Header.Get("Authorization") != "Bearer secret" { + t.Errorf("authorization = %q", request.Header.Get("Authorization")) + } + writer.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString( + writer, + `{"code":0,"data":{"remote_path":"user/实验 #1/result?.txt","recursive":`+ + recursiveValue+ + `},"msg":""}`, + ) + }, + )) + defer server.Close() + + result, err := NewClient(server.URL).SetToken("secret").RemoveFile( + context.Background(), + "user/实验 #1/result?.txt", + recursive, + ) + if err != nil { + t.Fatalf("RemoveFile: %v", err) + } + if result.RemotePath != "user/实验 #1/result?.txt" || + result.Recursive != recursive { + t.Fatalf("result = %#v", result) + } + if requests != 1 { + t.Fatalf("requests = %d, want exactly one", requests) + } + }) + } +} + +func TestRemoveFileAcceptsOnlyHTTP200AndCodeZero(t *testing.T) { + for _, test := range []struct { + name string + status int + body string + wantCode int + }{ + { + name: "created is not success", + status: http.StatusCreated, + body: `{"code":0,"data":{"remote_path":"user/file","recursive":false},"msg":""}`, + }, + { + name: "no content is not success", + status: http.StatusNoContent, + }, + { + name: "nonzero Crater code", + status: http.StatusOK, + body: `{"code":40902,"data":null,"msg":"recursive removal required"}`, + wantCode: 40902, + }, + { + name: "server conflict", + status: http.StatusConflict, + body: `{"code":40902,"data":null,"msg":"recursive removal required"}`, + wantCode: 40902, + }, + } { + t.Run(test.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc( + func(writer http.ResponseWriter, _ *http.Request) { + writer.Header().Set("Content-Type", "application/json") + writer.WriteHeader(test.status) + _, _ = io.WriteString(writer, test.body) + }, + )) + defer server.Close() + + _, err := NewClient(server.URL).RemoveFile( + context.Background(), + "user/file", + false, + ) + var requestErr *RequestError + if !errors.As(err, &requestErr) { + t.Fatalf("error = %T %v, want *RequestError", err, err) + } + if requestErr.HTTPStatus != test.status || requestErr.CraterCode != test.wantCode { + t.Fatalf("request error = %#v", requestErr) + } + }) + } +} + +func TestRemoveFileRejectsInvalidSuccessMetadata(t *testing.T) { + for _, test := range []struct { + name string + body string + recursive bool + }{ + {name: "empty body", body: ""}, + {name: "malformed JSON", body: `{"code":`}, + { + name: "missing code", + body: `{"data":{"remote_path":"user/file","recursive":false},"msg":""}`, + }, + {name: "null data", body: `{"code":0,"data":null,"msg":""}`}, + { + name: "missing remote path", + body: `{"code":0,"data":{"recursive":false},"msg":""}`, + }, + { + name: "mismatched remote path", + body: `{"code":0,"data":{"remote_path":"user/other","recursive":false},"msg":""}`, + }, + { + name: "missing recursive", + body: `{"code":0,"data":{"remote_path":"user/file"},"msg":""}`, + }, + { + name: "mismatched recursive", + body: `{"code":0,"data":{"remote_path":"user/file","recursive":false},"msg":""}`, + recursive: true, + }, + {name: "oversized body", body: strings.Repeat("x", maxFileErrorBody+1)}, + } { + t.Run(test.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc( + func(writer http.ResponseWriter, _ *http.Request) { + writer.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(writer, test.body) + }, + )) + defer server.Close() + + _, err := NewClient(server.URL).RemoveFile( + context.Background(), + "user/file", + test.recursive, + ) + var requestErr *RequestError + if !errors.As(err, &requestErr) { + t.Fatalf("error = %T %v, want *RequestError", err, err) + } + if requestErr.HTTPStatus != http.StatusOK { + t.Fatalf("status = %d, want 200", requestErr.HTTPStatus) + } + }) + } +} + +func TestRemoveFilePreservesServerError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc( + func(writer http.ResponseWriter, _ *http.Request) { + writer.Header().Set("Content-Type", "application/json") + writer.WriteHeader(http.StatusForbidden) + _, _ = io.WriteString( + writer, + `{"code":40301,"data":null,"msg":"write permission is required"}`, + ) + }, + )) + defer server.Close() + + _, err := NewClient(server.URL).RemoveFile(context.Background(), "public/file", false) + var requestErr *RequestError + if !errors.As(err, &requestErr) { + t.Fatalf("error = %T %v, want *RequestError", err, err) + } + if requestErr.HTTPStatus != http.StatusForbidden || + requestErr.CraterCode != 40301 || + requestErr.Msg != "write permission is required" { + t.Fatalf("request error = %#v", requestErr) + } +} + +func TestRemoveFileNetworkErrorHasNoHTTPStatusAndNoFallback(t *testing.T) { + sentinel := errors.New("network unavailable") + requests := 0 + client := NewClient("https://example.invalid") + client.httpClient.GetTransport().WrapRoundTripFunc(func(_ http.RoundTripper) req.HttpRoundTripFunc { + return func(request *http.Request) (*http.Response, error) { + requests++ + if request.URL.Path != "/api/ss/files/user/file" { + t.Errorf("request path = %q", request.URL.Path) + } + return nil, sentinel + } + }) + + _, err := client.RemoveFile(context.Background(), "user/file", false) + var networkErr *NetworkError + if !errors.As(err, &networkErr) { + t.Fatalf("error = %T %v, want *NetworkError", err, err) + } + if !errors.Is(networkErr, sentinel) { + t.Fatalf("error = %v, want sentinel", networkErr) + } + if requests != 1 { + t.Fatalf("requests = %d, want exactly one and no fallback", requests) + } +} diff --git a/cli/internal/api/file_test.go b/cli/internal/api/file_test.go new file mode 100644 index 000000000..f5aea02ac --- /dev/null +++ b/cli/internal/api/file_test.go @@ -0,0 +1,379 @@ +package api + +import ( + "bytes" + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/imroc/req/v3" +) + +func TestUploadFileStreamsAndEncodesPathSegments(t *testing.T) { + requestStarted := make(chan struct{}) + firstChunkRead := make(chan struct{}) + handlerDone := make(chan error, 1) + firstChunk := []byte{0x00, 0x01, 0xff, 'A'} + secondChunk := []byte("第二块") + + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + close(requestStarted) + if request.Method != http.MethodPost { + handlerDone <- errors.New("unexpected method: " + request.Method) + return + } + if request.URL.EscapedPath() != "/api/ss/upload/user/%E5%AE%9E%E9%AA%8C%20%231/100%25.bin" { + handlerDone <- errors.New("unexpected escaped path: " + request.URL.EscapedPath()) + return + } + if request.URL.Query().Get("overwrite") != "false" { + handlerDone <- errors.New("unexpected overwrite query") + return + } + if request.Header.Get("Content-Type") != "application/octet-stream" { + handlerDone <- errors.New("unexpected content type: " + request.Header.Get("Content-Type")) + return + } + if request.Header.Get("Authorization") != "Bearer secret" { + handlerDone <- errors.New("unexpected authorization header") + return + } + gotFirst := make([]byte, len(firstChunk)) + if _, err := io.ReadFull(request.Body, gotFirst); err != nil { + handlerDone <- err + return + } + if !bytes.Equal(gotFirst, firstChunk) { + handlerDone <- errors.New("unexpected first chunk") + return + } + close(firstChunkRead) + gotRest, err := io.ReadAll(request.Body) + if err != nil { + handlerDone <- err + return + } + if !bytes.Equal(gotRest, secondChunk) { + handlerDone <- errors.New("unexpected second chunk") + return + } + writer.WriteHeader(http.StatusCreated) + _, _ = io.WriteString(writer, `{"code":0,"data":{"remote_path":"user/实验 #1/100%.bin","bytes":13,"overwritten":false},"msg":""}`) + handlerDone <- nil + })) + defer server.Close() + + reader, writer := io.Pipe() + type uploadResult struct { + upload FileUploadResult + err error + } + result := make(chan uploadResult, 1) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + go func() { + upload, err := NewClient(server.URL).SetToken("secret").UploadFile( + ctx, + "user/实验 #1/100%.bin", + reader, + false, + ) + result <- uploadResult{upload: upload, err: err} + }() + + select { + case <-requestStarted: + case <-time.After(time.Second): + t.Fatal("request did not start before the complete source was available") + } + if _, err := writer.Write(firstChunk); err != nil { + t.Fatal(err) + } + select { + case <-firstChunkRead: + case <-time.After(time.Second): + t.Fatal("server did not receive the first chunk incrementally") + } + if _, err := writer.Write(secondChunk); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + + got := <-result + if got.err != nil { + t.Fatalf("UploadFile: %v", got.err) + } + if handlerErr := <-handlerDone; handlerErr != nil { + t.Fatal(handlerErr) + } + wantBytes := int64(len(firstChunk) + len(secondChunk)) + if got.upload.Bytes != wantBytes || got.upload.RemotePath != "user/实验 #1/100%.bin" || got.upload.Overwritten { + t.Fatalf("upload = %#v, want %d bytes", got.upload, wantBytes) + } +} + +func TestUploadFileOverwriteSetsExplicitQuery(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.URL.Query().Get("overwrite") != "true" { + t.Errorf("overwrite query = %q", request.URL.Query().Get("overwrite")) + } + _, _ = io.Copy(io.Discard, request.Body) + writer.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(writer, `{"code":0,"data":{"remote_path":"user/result.bin","bytes":11,"overwritten":true},"msg":""}`) + })) + defer server.Close() + + upload, err := NewClient(server.URL).UploadFile( + context.Background(), + "user/result.bin", + bytes.NewBufferString("replacement"), + true, + ) + if err != nil { + t.Fatalf("UploadFile: %v", err) + } + if upload.Bytes != int64(len("replacement")) || !upload.Overwritten { + t.Fatalf("upload = %#v", upload) + } +} + +func TestUploadFileAcceptsEmptySource(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + body, err := io.ReadAll(request.Body) + if err != nil { + t.Errorf("read body: %v", err) + } + if len(body) != 0 { + t.Errorf("body = %v, want empty", body) + } + writer.Header().Set("Content-Type", "application/json") + writer.WriteHeader(http.StatusCreated) + _, _ = io.WriteString(writer, `{"code":0,"data":{"remote_path":"user/empty.bin","bytes":0,"overwritten":false},"msg":""}`) + })) + defer server.Close() + + upload, err := NewClient(server.URL).UploadFile( + context.Background(), + "user/empty.bin", + bytes.NewReader(nil), + false, + ) + if err != nil { + t.Fatalf("UploadFile: %v", err) + } + if upload.Bytes != 0 || upload.Overwritten { + t.Fatalf("upload = %#v", upload) + } +} + +func TestUploadFileDecodesJSONAndPlainTextErrors(t *testing.T) { + tests := []struct { + name string + status int + body string + content string + wantCode int + wantMsg string + }{ + { + name: "Crater envelope", + status: http.StatusConflict, + body: `{"code":40901,"data":null,"msg":"target file already exists"}`, + content: "application/json", + wantCode: 40901, + wantMsg: "target file already exists", + }, + { + name: "plain text", + status: http.StatusConflict, + body: "parent directory does not exist", + content: "text/plain", + wantMsg: "parent directory does not exist", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + _, _ = io.Copy(io.Discard, request.Body) + writer.Header().Set("Content-Type", test.content) + writer.WriteHeader(test.status) + _, _ = io.WriteString(writer, test.body) + })) + defer server.Close() + + _, err := NewClient(server.URL).UploadFile( + context.Background(), + "user/result.bin", + bytes.NewBufferString("data"), + false, + ) + var requestErr *RequestError + if !errors.As(err, &requestErr) { + t.Fatalf("error = %T %v, want *RequestError", err, err) + } + if requestErr.HTTPStatus != test.status || + requestErr.CraterCode != test.wantCode || + requestErr.Msg != test.wantMsg { + t.Fatalf("request error = %#v", requestErr) + } + }) + } +} + +func TestUploadFileRejectsInvalidSuccessResponsesWithHTTPStatus(t *testing.T) { + tests := []struct { + name string + body string + overwrite bool + }{ + {name: "empty body"}, + {name: "malformed JSON", body: `{"code":`}, + {name: "null metadata", body: `{"code":0,"data":null,"msg":""}`}, + {name: "wrong path", body: `{"code":0,"data":{"remote_path":"user/other.bin","bytes":4,"overwritten":false},"msg":""}`}, + {name: "wrong byte count", body: `{"code":0,"data":{"remote_path":"user/result.bin","bytes":3,"overwritten":false},"msg":""}`}, + {name: "negative byte count", body: `{"code":0,"data":{"remote_path":"user/result.bin","bytes":-1,"overwritten":false},"msg":""}`}, + {name: "unauthorized overwrite", body: `{"code":0,"data":{"remote_path":"user/result.bin","bytes":4,"overwritten":true},"msg":""}`}, + {name: "oversized body", body: strings.Repeat("x", maxFileErrorBody+1)}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + _, _ = io.Copy(io.Discard, request.Body) + writer.Header().Set("Content-Type", "application/json") + writer.WriteHeader(http.StatusCreated) + _, _ = io.WriteString(writer, test.body) + })) + defer server.Close() + + _, err := NewClient(server.URL).UploadFile( + context.Background(), + "user/result.bin", + bytes.NewBufferString("data"), + test.overwrite, + ) + var requestErr *RequestError + if !errors.As(err, &requestErr) { + t.Fatalf("error = %T %v, want *RequestError", err, err) + } + if requestErr.HTTPStatus != http.StatusCreated { + t.Fatalf("HTTP status = %d, want %d", requestErr.HTTPStatus, http.StatusCreated) + } + }) + } +} + +type failingUploadReader struct { + err error +} + +func (reader failingUploadReader) Read([]byte) (int, error) { + return 0, reader.err +} + +func TestUploadFileIdentifiesSourceReadFailure(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + _, _ = io.Copy(io.Discard, request.Body) + writer.WriteHeader(http.StatusBadRequest) + })) + defer server.Close() + + sentinel := errors.New("local disk failed") + _, err := NewClient(server.URL).UploadFile( + context.Background(), + "user/result.bin", + failingUploadReader{err: sentinel}, + false, + ) + var sourceErr *SourceReadError + if !errors.As(err, &sourceErr) { + t.Fatalf("error = %T %v, want *SourceReadError", err, err) + } + if !errors.Is(sourceErr, sentinel) { + t.Fatalf("error = %v, want sentinel", sourceErr) + } +} + +type failingFileReadCloser struct { + err error +} + +func (reader failingFileReadCloser) Read([]byte) (int, error) { + return 0, reader.err +} + +func (failingFileReadCloser) Close() error { + return nil +} + +func TestUploadFileKeepsHTTPStatusWhenErrorBodyReadFails(t *testing.T) { + sentinel := errors.New("broken error body") + client := NewClient("https://example.invalid") + client.httpClient.GetTransport().WrapRoundTripFunc(func(_ http.RoundTripper) req.HttpRoundTripFunc { + return func(request *http.Request) (*http.Response, error) { + _, _ = io.Copy(io.Discard, request.Body) + return &http.Response{ + StatusCode: http.StatusConflict, + Status: "409 Conflict", + Header: make(http.Header), + Body: failingFileReadCloser{err: sentinel}, + Request: request, + }, nil + } + }) + + _, err := client.UploadFile( + context.Background(), + "user/result.bin", + bytes.NewBufferString("data"), + false, + ) + var requestErr *RequestError + if !errors.As(err, &requestErr) { + t.Fatalf("error = %T %v, want *RequestError", err, err) + } + if requestErr.HTTPStatus != http.StatusConflict || + requestErr.Msg != "Conflict: broken error body" { + t.Fatalf("request error = %#v", requestErr) + } +} + +func TestUploadFileKeepsHTTPStatusWhenSuccessBodyReadFails(t *testing.T) { + sentinel := errors.New("broken success body") + client := NewClient("https://example.invalid") + client.httpClient.GetTransport().WrapRoundTripFunc(func(_ http.RoundTripper) req.HttpRoundTripFunc { + return func(request *http.Request) (*http.Response, error) { + _, _ = io.Copy(io.Discard, request.Body) + return &http.Response{ + StatusCode: http.StatusCreated, + Status: "201 Created", + Header: make(http.Header), + Body: failingFileReadCloser{err: sentinel}, + Request: request, + }, nil + } + }) + + _, err := client.UploadFile( + context.Background(), + "user/result.bin", + bytes.NewBufferString("data"), + false, + ) + var requestErr *RequestError + if !errors.As(err, &requestErr) { + t.Fatalf("error = %T %v, want *RequestError", err, err) + } + if requestErr.HTTPStatus != http.StatusCreated || + requestErr.Msg != "failed to read upload response: broken success body" { + t.Fatalf("request error = %#v", requestErr) + } +} diff --git a/cli/internal/api/paths.go b/cli/internal/api/paths.go index 6ccc6fce4..1e465503d 100644 --- a/cli/internal/api/paths.go +++ b/cli/internal/api/paths.go @@ -34,6 +34,7 @@ const ( SPJobsPrefix = "/api/v1/spjobs" VCJobsPrefix = "/api/v1/vcjobs" AdminVCJobsPrefix = "/api/v1/admin/vcjobs" + StoragePrefix = "/api/ss" ) const CompatibilityPath = CompatibilityPrefix + "/compatibility" @@ -54,4 +55,7 @@ const ( VCJobListPath = VCJobsPrefix VCJobBillingPath = VCJobsPrefix + "/billing" AdminVCJobBillingPath = AdminVCJobsPrefix + "/billing" + FileUploadPath = StoragePrefix + "/upload" + FileMovePath = StoragePrefix + "/move" + FileRemovePath = StoragePrefix + "/files" ) diff --git a/cli/internal/i18n/catalog_file.go b/cli/internal/i18n/catalog_file.go new file mode 100644 index 000000000..e2fb81fa1 --- /dev/null +++ b/cli/internal/i18n/catalog_file.go @@ -0,0 +1,94 @@ +package i18n + +var catalogFile = map[Language]map[string]string{ + En: { + "file_short": "Manage remote files", + "file_long": "Manage files and directories in user, public, and account storage spaces.", + "file_mkdir_short": "Create one remote directory", + "file_mkdir_long": "Create exactly one directory below user, public, or account storage without creating missing parents.", + "file_mv_short": "Move one remote file or directory", + "file_mv_long": "Move one remote storage entry to an exact destination path without replacing an existing entry.", + "file_rm_short": "Remove one remote file or directory", + "file_rm_long": "Remove exactly one remote entry below user, public, or account storage. Directories require --recursive.", + "file_rm_flag_recursive": "Remove a directory and all of its contents", + "file_rm_flag_yes": "Remove without confirmation", + "file_upload_short": "Upload one local file", + "file_upload_long": "Stream one local regular file to a path below user, public, or account storage.", + "file_upload_flag_overwrite": "Replace an existing remote file", + "file_label_local_file": "local file", + "file_label_remote_file": "remote path", + "file_label_remote_directory": "remote directory", + "file_label_source_path": "source path", + "file_label_destination_path": "destination path", + "file_label_remote_path": "remote path", + "err_file_path_invalid": "invalid remote path %q", + "err_file_path_root": "remote path must start with user, public, or account: %q", + "err_file_path_not_file": "remote file must name an entry below user, public, or account: %q", + "err_file_path_not_entry": "remote path must name an entry below user, public, or account: %q", + "err_file_move_same": "move destination must differ from the source: %q", + "err_file_move_descendant": "move destination %q cannot be inside source %q", + "err_file_remove_ambiguous_path": "remote path for removal must not contain . or .. segments: %q", + "err_file_remove_root": "refusing to remove a storage root: %q", + "err_file_local_open": "failed to open local file %q: %s", + "err_file_local_stat": "failed to inspect local file %q: %s", + "err_file_local_not_regular": "local path is not a regular file: %q", + "err_file_local_read": "failed to read local file %q: %s", + "err_file_output": "failed to write command output: %s", + "file_upload_success": "Uploaded %s to %s (%d bytes)", + "file_mkdir_success": "Created remote directory %s", + "file_move_success": "Moved %s to %s", + "file_remove_confirm": "Remove remote entry %q?", + "file_remove_recursive_confirm": "Recursively remove remote entry %q?", + "file_remove_success": "Removed remote entry %s", + "file_root_user_desc": "Your private user storage.", + "file_root_public_desc": "Shared public storage.", + "file_root_account_desc": "Storage for the current account.", + "file_local_regular_desc": "Local regular file.", + "file_local_directory_desc": "Local directory.", + }, + ZhCN: { + "file_short": "管理远端文件", + "file_long": "管理用户、公共及当前账户存储空间中的文件和目录。", + "file_mkdir_short": "创建单个远端目录", + "file_mkdir_long": "在 user、public 或 account 存储空间下创建单个目录，且不会自动创建缺失的父目录。", + "file_mv_short": "移动单个远端文件或目录", + "file_mv_long": "将单个远端存储条目移动到指定目标路径，且不会覆盖已存在的条目。", + "file_rm_short": "删除单个远端文件或目录", + "file_rm_long": "删除 user、public 或 account 存储空间下的单个远端条目。删除目录时必须提供 --recursive。", + "file_rm_flag_recursive": "删除目录及其中的全部内容", + "file_rm_flag_yes": "跳过确认并执行删除", + "file_upload_short": "上传单个本地文件", + "file_upload_long": "将一个本地普通文件流式上传到 user、public 或 account 存储空间下的路径。", + "file_upload_flag_overwrite": "替换已存在的远端文件", + "file_label_local_file": "本地文件", + "file_label_remote_file": "远端路径", + "file_label_remote_directory": "远端目录", + "file_label_source_path": "源路径", + "file_label_destination_path": "目标路径", + "file_label_remote_path": "远端路径", + "err_file_path_invalid": "无效的远端路径 %q", + "err_file_path_root": "远端路径必须以 user、public 或 account 开头：%q", + "err_file_path_not_file": "远端文件必须指向 user、public 或 account 下的具体条目：%q", + "err_file_path_not_entry": "远端路径必须指向 user、public 或 account 下的具体条目：%q", + "err_file_move_same": "移动目标必须与源路径不同：%q", + "err_file_move_descendant": "移动目标 %q 不能位于源路径 %q 内", + "err_file_remove_ambiguous_path": "待删除的远端路径不能包含 . 或 .. 路径段：%q", + "err_file_remove_root": "拒绝删除存储根目录：%q", + "err_file_local_open": "打开本地文件 %q 失败：%s", + "err_file_local_stat": "检查本地文件 %q 失败：%s", + "err_file_local_not_regular": "本地路径不是普通文件：%q", + "err_file_local_read": "读取本地文件 %q 失败：%s", + "err_file_output": "写入命令输出失败：%s", + "file_upload_success": "已将 %s 上传到 %s（%d 字节）", + "file_mkdir_success": "已创建远端目录 %s", + "file_move_success": "已将 %s 移动到 %s", + "file_remove_confirm": "确定删除远端条目“%s”？", + "file_remove_recursive_confirm": "确定递归删除远端条目“%s”？", + "file_remove_success": "已删除远端条目 %s", + "file_root_user_desc": "当前用户的私有存储空间", + "file_root_public_desc": "共享公共存储空间", + "file_root_account_desc": "当前账户的存储空间", + "file_local_regular_desc": "本地普通文件", + "file_local_directory_desc": "本地目录", + }, +} diff --git a/cli/internal/i18n/i18n.go b/cli/internal/i18n/i18n.go index c2a68e3a2..b2f2fa1d7 100644 --- a/cli/internal/i18n/i18n.go +++ b/cli/internal/i18n/i18n.go @@ -29,6 +29,7 @@ var translations = mergeCatalogs( catalogOrder, catalogErrors, catalogJob, + catalogFile, ) func mergeCatalogs(catalogs ...map[Language]map[string]string) map[Language]map[string]string { diff --git a/cli/skills/crater-cli-file/SKILL.md b/cli/skills/crater-cli-file/SKILL.md new file mode 100644 index 000000000..bbbb089d4 --- /dev/null +++ b/cli/skills/crater-cli-file/SKILL.md @@ -0,0 +1,105 @@ +--- +name: crater-cli-file +version: 0.3.0 +description: "Use Crater CLI to create, move, safely remove, or upload one entry in ordinary-user remote storage." +metadata: + requires: + bins: ["crater"] + cliHelp: "crater file --help" +--- + +# Crater CLI File Operations + +**CRITICAL — Before doing anything else, MUST read `crater-cli-shared` (possible path: [`../crater-cli-shared/SKILL.md`](../crater-cli-shared/SKILL.md)) for global options, non-interactive use, errors, and sensitive information handling.** + +Use `crater file` when a user wants to create a directory, move or safely remove one remote entry, or copy one local regular file into Crater storage. + +## Supported workflow + +- Create a new remote file: + + ```bash + crater file upload ./train.py user/jobs/train.py + ``` + +- Create exactly one remote directory: + + ```bash + crater file mkdir user/jobs/new-run + ``` + +- Move or rename one remote file: + + ```bash + crater file mv user/jobs/train.py user/jobs/archive/train.py + ``` + +- Move one remote directory: + + ```bash + crater file mv user/jobs/old-run account/archive/old-run + ``` + +- Remove one remote file after confirming the normalized target: + + ```bash + crater file rm user/jobs/archive/train.py + ``` + +- Recursively remove one remote directory only with both explicit safeguards: + + ```bash + crater file rm user/jobs/old-run --recursive --yes + ``` + +- Upload a binary file to current-account storage: + + ```bash + crater file upload ./weights.bin "account/模型/weights.bin" + ``` + +- Replace an existing regular remote file only after the user explicitly asks for it: + + ```bash + crater file upload ./train.py user/jobs/train.py --overwrite + ``` + +- Return structured metadata: + + ```bash + crater file upload ./train.py user/jobs/train.py --json --no-interactive + ``` + +## Safety + +- The local path must resolve to one open regular file. Directories, devices, sockets, and pipes are rejected before any API request. +- Remote paths must start with `user`, `public`, or `account` and must name an entry below that root. +- `mkdir` creates exactly one directory. Its parent must already exist. +- `mv` takes the complete source and complete destination path. The destination is not interpreted as a parent directory. +- `mv` never overwrites an existing destination and has no overwrite flag. Choose a different exact path when the server reports a conflict. +- `mv` fails closed when the backing filesystem cannot provide atomic no-clobber rename semantics. +- Do not move an entry to itself or below itself. +- `rm` accepts one exact path only. It rejects logical roots, raw `.` or `..` segments, reserved platform roots, globs, and bulk targets. +- Interactive `rm` shows the normalized exact path and defaults to No. Cancellation sends no request. +- `--json` and `--no-interactive` are non-interactive; add `--yes` explicitly or the command returns a usage error. +- Directories require `--recursive` in addition to confirmation. Never infer or add it unless recursive deletion is the user's stated intent. +- `rm` uses only the dedicated safe `/api/ss/files` endpoint. Never retry through the legacy `/api/ss/delete` endpoint. +- A failed recursive deletion can be partial on network filesystems. Inspect the remaining path before deciding whether to retry. +- Never add `--overwrite` unless replacing that exact remote target is part of the user's request. +- The server stages the complete stream in the target directory and atomically publishes it. A failed transfer never exposes a partial new file or truncates the previous file. +- Parent directories are never created automatically. +- This command uploads one file only. Do not pass a directory or shell glob. +- JSON stdout contains metadata only; it never includes file bytes. +- Do not ask the user to provide a token or Keyring content. + +## Troubleshooting + +1. Run `crater auth ls --json` and confirm an active context exists. +2. Use `crater file --help` and the selected subcommand's help to verify the local binary supports the operation. +3. If `mkdir` reports a missing parent, create each required parent explicitly from top to bottom. +4. If `mv` reports a conflict, choose another complete destination path; there is no overwrite mode. +5. If `rm` reports that a directory requires recursive authorization, verify the exact target and rerun with `--recursive`; add `--yes` only when the user has confirmed deletion. +6. If a recursive remove fails, inspect the path before retrying because some children may already be gone. +7. If an upload target exists, choose a new path or obtain explicit permission to add `--overwrite`. +8. A `404` from `/api/ss/upload` or `/api/ss/files` means the storage service may be older than the safe endpoint; upgrade it instead of falling back to an unsafe legacy route. +9. For API errors, inspect `category`, `code`, and `context.http_status` from JSON stderr without exposing credentials. diff --git a/cli/test/snapshots/file/file_test.go b/cli/test/snapshots/file/file_test.go new file mode 100644 index 000000000..4cb32d0fa --- /dev/null +++ b/cli/test/snapshots/file/file_test.go @@ -0,0 +1,80 @@ +package file_test + +import ( + "os" + "testing" + + "github.com/raids-lab/crater/cli/internal/snaptest" +) + +const goldenStemFileUpload = "file_upload" + +func TestFileUploadSnapshotsEN(t *testing.T) { + runFileUploadSnapshots(t, "en") +} + +func TestFileUploadSnapshotsZhCN(t *testing.T) { + runFileUploadSnapshots(t, "zh-CN") +} + +func runFileUploadSnapshots(t *testing.T, language string) { + t.Helper() + path := snaptest.GoldenFileT(t, "file", goldenStemFileUpload, language) + home := t.TempDir() + baseEnv := append(snaptest.EnvMinimal(home, language), "CRATER_TEST_SANDBOX_HTTP=error404") + binary := snaptest.CraterExecutable(t) + localFixture := ".snapshot-upload-" + language + ".bin" + if err := os.WriteFile(localFixture, []byte{0x00, 0xff, 'C', 'L', 'I'}, 0o600); err != nil { + t.Fatal(err) + } + defer os.Remove(localFixture) + + cases := []snaptest.Case{ + {ID: "01-file-typo-json", Args: []string{"file", "get", "--json", "--no-interactive"}}, + {ID: "01b-file-typo-text", Args: []string{"file", "get", "--no-interactive"}}, + {ID: "02-file-upload-missing-local-json", Args: []string{"file", "upload", "--json", "--no-interactive"}}, + {ID: "03-file-upload-missing-remote-json", Args: []string{"file", "upload", localFixture, "--json", "--no-interactive"}}, + {ID: "04-file-upload-extra-arg-json", Args: []string{"file", "upload", localFixture, "user/a.bin", "extra", "--json", "--no-interactive"}}, + {ID: "05-file-upload-traversal-json", Args: []string{"file", "upload", localFixture, "user/../public/a.bin", "--json", "--no-interactive"}}, + {ID: "06-file-upload-root-json", Args: []string{"file", "upload", localFixture, "user", "--json", "--no-interactive"}}, + {ID: "07-file-upload-directory-json", Args: []string{"file", "upload", ".", "user/a.bin", "--json", "--no-interactive"}}, + {ID: "08-file-upload-404-json", Args: []string{"file", "upload", localFixture, "user/实验 data/result.bin", "--json", "--no-interactive"}}, + {ID: "09-file-help", Args: []string{"file", "--help"}}, + {ID: "10-file-upload-help", Args: []string{"file", "upload", "--help"}}, + {ID: "11-file-mkdir-help", Args: []string{"file", "mkdir", "--help"}}, + {ID: "12-file-mkdir-missing-json", Args: []string{"file", "mkdir", "--json", "--no-interactive"}}, + {ID: "13-file-mkdir-root-json", Args: []string{"file", "mkdir", "user", "--json", "--no-interactive"}}, + {ID: "14-file-mkdir-404-json", Args: []string{"file", "mkdir", "user/实验 data", "--json", "--no-interactive"}}, + {ID: "15-file-mv-help", Args: []string{"file", "mv", "--help"}}, + {ID: "16-file-mv-missing-source-json", Args: []string{"file", "mv", "--json", "--no-interactive"}}, + {ID: "17-file-mv-missing-destination-json", Args: []string{"file", "mv", "user/source", "--json", "--no-interactive"}}, + {ID: "18-file-mv-invalid-operands-json", Args: []string{"file", "mv", "user/../source", `account\destination`, "--json", "--no-interactive"}}, + {ID: "19-file-mv-same-json", Args: []string{"file", "mv", "user/source", "/user//./source", "--json", "--no-interactive"}}, + {ID: "20-file-mv-descendant-json", Args: []string{"file", "mv", "user/source", "user/source/nested", "--json", "--no-interactive"}}, + {ID: "21-file-mv-404-json", Args: []string{"file", "mv", "user/source", "account/实验 data/result", "--json", "--no-interactive"}}, + {ID: "22-file-rm-help", Args: []string{"file", "rm", "--help"}}, + {ID: "23-file-rm-missing-json", Args: []string{"file", "rm", "--json", "--no-interactive"}}, + {ID: "24-file-rm-extra-arg-json", Args: []string{"file", "rm", "user/a", "user/b", "--json", "--no-interactive"}}, + {ID: "25-file-rm-root-json", Args: []string{"file", "rm", "user", "--yes", "--json", "--no-interactive"}}, + {ID: "26-file-rm-dot-json", Args: []string{"file", "rm", "user/./victim", "--yes", "--json", "--no-interactive"}}, + {ID: "27-file-rm-traversal-json", Args: []string{"file", "rm", "user/../victim", "--yes", "--json", "--no-interactive"}}, + {ID: "28-file-rm-missing-yes-json", Args: []string{"file", "rm", "user/victim", "--json", "--no-interactive"}}, + {ID: "29-file-rm-json-implies-noninteractive", Args: []string{"file", "rm", "user/victim", "--json"}}, + {ID: "30-file-rm-404-json", Args: []string{"file", "rm", "user/实验 data/result", "--yes", "--json", "--no-interactive"}}, + {ID: "31-file-rm-recursive-404-json", Args: []string{"file", "rm", "account/runs/old", "--recursive", "--yes", "--json", "--no-interactive"}}, + } + + results := make([]*snaptest.Result, len(cases)) + for index := range cases { + result, err := snaptest.Run(binary, baseEnv, cases[index].Args) + if err != nil { + t.Fatalf("case %s: %v", cases[index].ID, err) + } + results[index] = result + } + + update := os.Getenv("UPDATE_SNAPSHOTS") == "1" || os.Getenv("UPDATE_SNAPSHOTS") == "true" + if err := snaptest.MatchOrUpdateGolden(path, language, cases, results, update); err != nil { + t.Fatal(err) + } +} diff --git a/cli/testdata/snapshots/file/file_upload.en.txtar b/cli/testdata/snapshots/file/file_upload.en.txtar new file mode 100644 index 000000000..655e45638 --- /dev/null +++ b/cli/testdata/snapshots/file/file_upload.en.txtar @@ -0,0 +1,426 @@ +# Crater CLI snapshot bundle (txtar). Regenerate: make snapshot-update (or UPDATE_SNAPSHOTS=1 go test ./test/snapshots/...) +-- en/01-file-typo-json/argv -- +crater file get --json --no-interactive +-- en/01-file-typo-json/exit -- +2 +-- en/01-file-typo-json/stdout -- +-- en/01-file-typo-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_UNKNOWN_COMMAND", + "message": "unknown command \"get\" for \"crater file\"\nRun \"crater file --help\" for usage." +} +-- en/01b-file-typo-text/argv -- +crater file get --no-interactive +-- en/01b-file-typo-text/exit -- +2 +-- en/01b-file-typo-text/stdout -- +-- en/01b-file-typo-text/stderr -- +Error: + unknown command "get" for "crater file" + Run "crater file --help" for usage. +-- en/02-file-upload-missing-local-json/argv -- +crater file upload --json --no-interactive +-- en/02-file-upload-missing-local-json/exit -- +2 +-- en/02-file-upload-missing-local-json/stdout -- +-- en/02-file-upload-missing-local-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_MISSING_REQUIRED_FLAG", + "message": "local file is required (\u003clocal-file\u003e)" +} +-- en/03-file-upload-missing-remote-json/argv -- +crater file upload .snapshot-upload-en.bin --json --no-interactive +-- en/03-file-upload-missing-remote-json/exit -- +2 +-- en/03-file-upload-missing-remote-json/stdout -- +-- en/03-file-upload-missing-remote-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_MISSING_REQUIRED_FLAG", + "message": "remote path is required (\u003cremote-path\u003e)" +} +-- en/04-file-upload-extra-arg-json/argv -- +crater file upload .snapshot-upload-en.bin user/a.bin extra --json --no-interactive +-- en/04-file-upload-extra-arg-json/exit -- +2 +-- en/04-file-upload-extra-arg-json/stdout -- +-- en/04-file-upload-extra-arg-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_INVALID_FLAG_VALUE", + "message": "too many arguments for crater file upload: got 3, want at most 2" +} +-- en/05-file-upload-traversal-json/argv -- +crater file upload .snapshot-upload-en.bin user/../public/a.bin --json --no-interactive +-- en/05-file-upload-traversal-json/exit -- +2 +-- en/05-file-upload-traversal-json/stdout -- +-- en/05-file-upload-traversal-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_INVALID_FLAG_VALUE", + "message": "invalid remote path \"user/../public/a.bin\"" +} +-- en/06-file-upload-root-json/argv -- +crater file upload .snapshot-upload-en.bin user --json --no-interactive +-- en/06-file-upload-root-json/exit -- +2 +-- en/06-file-upload-root-json/stdout -- +-- en/06-file-upload-root-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_INVALID_FLAG_VALUE", + "message": "remote file must name an entry below user, public, or account: \"user\"" +} +-- en/07-file-upload-directory-json/argv -- +crater file upload . user/a.bin --json --no-interactive +-- en/07-file-upload-directory-json/exit -- +2 +-- en/07-file-upload-directory-json/stdout -- +-- en/07-file-upload-directory-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_INVALID_FLAG_VALUE", + "message": "local path is not a regular file: \".\"" +} +-- en/08-file-upload-404-json/argv -- +crater file upload .snapshot-upload-en.bin user/实验 data/result.bin --json --no-interactive +-- en/08-file-upload-404-json/exit -- +4 +-- en/08-file-upload-404-json/stdout -- +-- en/08-file-upload-404-json/stderr -- +{ + "category": "api_error", + "code": "ERR_NOT_FOUND_404", + "message": "HTTP 404: simulated", + "context": { + "crater_code": 404, + "http_status": 404, + "msg": "simulated" + } +} +-- en/09-file-help/argv -- +crater file --help +-- en/09-file-help/exit -- +0 +-- en/09-file-help/stdout -- +Manage files and directories in user, public, and account storage spaces. + +Usage: + crater file [flags] + crater file [command] + +Available Commands: + mkdir Create one remote directory + mv Move one remote file or directory + rm Remove one remote file or directory + upload Upload one local file + +Global Flags: + -h, --help Help for crater + --json Output in raw JSON format + --no-interactive Disable interactive prompts + +Use "crater file [command] --help" for more information about a command. +-- en/09-file-help/stderr -- +-- en/10-file-upload-help/argv -- +crater file upload --help +-- en/10-file-upload-help/exit -- +0 +-- en/10-file-upload-help/stdout -- +Stream one local regular file to a path below user, public, or account storage. + +Usage: + crater file upload [flags] + +Flags: + --overwrite Replace an existing remote file + +Global Flags: + -h, --help Help for crater + --json Output in raw JSON format + --no-interactive Disable interactive prompts +-- en/10-file-upload-help/stderr -- +-- en/11-file-mkdir-help/argv -- +crater file mkdir --help +-- en/11-file-mkdir-help/exit -- +0 +-- en/11-file-mkdir-help/stdout -- +Create exactly one directory below user, public, or account storage without creating missing parents. + +Usage: + crater file mkdir [flags] + +Global Flags: + -h, --help Help for crater + --json Output in raw JSON format + --no-interactive Disable interactive prompts +-- en/11-file-mkdir-help/stderr -- +-- en/12-file-mkdir-missing-json/argv -- +crater file mkdir --json --no-interactive +-- en/12-file-mkdir-missing-json/exit -- +2 +-- en/12-file-mkdir-missing-json/stdout -- +-- en/12-file-mkdir-missing-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_MISSING_REQUIRED_FLAG", + "message": "remote directory is required (\u003cremote-path\u003e)" +} +-- en/13-file-mkdir-root-json/argv -- +crater file mkdir user --json --no-interactive +-- en/13-file-mkdir-root-json/exit -- +2 +-- en/13-file-mkdir-root-json/stdout -- +-- en/13-file-mkdir-root-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_INVALID_FLAG_VALUE", + "message": "remote path must name an entry below user, public, or account: \"user\"" +} +-- en/14-file-mkdir-404-json/argv -- +crater file mkdir user/实验 data --json --no-interactive +-- en/14-file-mkdir-404-json/exit -- +4 +-- en/14-file-mkdir-404-json/stdout -- +-- en/14-file-mkdir-404-json/stderr -- +{ + "category": "api_error", + "code": "ERR_NOT_FOUND_404", + "message": "HTTP 404: simulated", + "context": { + "crater_code": 404, + "http_status": 404, + "msg": "simulated" + } +} +-- en/15-file-mv-help/argv -- +crater file mv --help +-- en/15-file-mv-help/exit -- +0 +-- en/15-file-mv-help/stdout -- +Move one remote storage entry to an exact destination path without replacing an existing entry. + +Usage: + crater file mv [flags] + +Global Flags: + -h, --help Help for crater + --json Output in raw JSON format + --no-interactive Disable interactive prompts +-- en/15-file-mv-help/stderr -- +-- en/16-file-mv-missing-source-json/argv -- +crater file mv --json --no-interactive +-- en/16-file-mv-missing-source-json/exit -- +2 +-- en/16-file-mv-missing-source-json/stdout -- +-- en/16-file-mv-missing-source-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_MISSING_REQUIRED_FLAG", + "message": "source path is required (\u003csource-path\u003e)" +} +-- en/17-file-mv-missing-destination-json/argv -- +crater file mv user/source --json --no-interactive +-- en/17-file-mv-missing-destination-json/exit -- +2 +-- en/17-file-mv-missing-destination-json/stdout -- +-- en/17-file-mv-missing-destination-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_MISSING_REQUIRED_FLAG", + "message": "destination path is required (\u003cdestination-path\u003e)" +} +-- en/18-file-mv-invalid-operands-json/argv -- +crater file mv user/../source account\destination --json --no-interactive +-- en/18-file-mv-invalid-operands-json/exit -- +2 +-- en/18-file-mv-invalid-operands-json/stdout -- +-- en/18-file-mv-invalid-operands-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_INVALID_FLAG_VALUE", + "message": "invalid remote path \"user/../source\"\ninvalid remote path \"account\\\\destination\"", + "context": { + "issues": [ + { + "code": "ERR_INVALID_FLAG_VALUE", + "field": "source-path", + "message": "invalid remote path \"user/../source\"" + }, + { + "code": "ERR_INVALID_FLAG_VALUE", + "field": "destination-path", + "message": "invalid remote path \"account\\\\destination\"" + } + ] + } +} +-- en/19-file-mv-same-json/argv -- +crater file mv user/source /user//./source --json --no-interactive +-- en/19-file-mv-same-json/exit -- +2 +-- en/19-file-mv-same-json/stdout -- +-- en/19-file-mv-same-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_INVALID_FLAG_VALUE", + "message": "move destination must differ from the source: \"user/source\"" +} +-- en/20-file-mv-descendant-json/argv -- +crater file mv user/source user/source/nested --json --no-interactive +-- en/20-file-mv-descendant-json/exit -- +2 +-- en/20-file-mv-descendant-json/stdout -- +-- en/20-file-mv-descendant-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_INVALID_FLAG_VALUE", + "message": "move destination \"user/source/nested\" cannot be inside source \"user/source\"" +} +-- en/21-file-mv-404-json/argv -- +crater file mv user/source account/实验 data/result --json --no-interactive +-- en/21-file-mv-404-json/exit -- +4 +-- en/21-file-mv-404-json/stdout -- +-- en/21-file-mv-404-json/stderr -- +{ + "category": "api_error", + "code": "ERR_NOT_FOUND_404", + "message": "HTTP 404: simulated", + "context": { + "crater_code": 404, + "http_status": 404, + "msg": "simulated" + } +} +-- en/22-file-rm-help/argv -- +crater file rm --help +-- en/22-file-rm-help/exit -- +0 +-- en/22-file-rm-help/stdout -- +Remove exactly one remote entry below user, public, or account storage. Directories require --recursive. + +Usage: + crater file rm [flags] + +Flags: + --recursive Remove a directory and all of its contents + -y, --yes Remove without confirmation + +Global Flags: + -h, --help Help for crater + --json Output in raw JSON format + --no-interactive Disable interactive prompts +-- en/22-file-rm-help/stderr -- +-- en/23-file-rm-missing-json/argv -- +crater file rm --json --no-interactive +-- en/23-file-rm-missing-json/exit -- +2 +-- en/23-file-rm-missing-json/stdout -- +-- en/23-file-rm-missing-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_MISSING_REQUIRED_FLAG", + "message": "remote path is required (\u003cremote-path\u003e)" +} +-- en/24-file-rm-extra-arg-json/argv -- +crater file rm user/a user/b --json --no-interactive +-- en/24-file-rm-extra-arg-json/exit -- +2 +-- en/24-file-rm-extra-arg-json/stdout -- +-- en/24-file-rm-extra-arg-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_INVALID_FLAG_VALUE", + "message": "too many arguments for crater file rm: got 2, want at most 1" +} +-- en/25-file-rm-root-json/argv -- +crater file rm user --yes --json --no-interactive +-- en/25-file-rm-root-json/exit -- +2 +-- en/25-file-rm-root-json/stdout -- +-- en/25-file-rm-root-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_INVALID_FLAG_VALUE", + "message": "refusing to remove a storage root: \"user\"" +} +-- en/26-file-rm-dot-json/argv -- +crater file rm user/./victim --yes --json --no-interactive +-- en/26-file-rm-dot-json/exit -- +2 +-- en/26-file-rm-dot-json/stdout -- +-- en/26-file-rm-dot-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_INVALID_FLAG_VALUE", + "message": "remote path for removal must not contain . or .. segments: \"user/./victim\"" +} +-- en/27-file-rm-traversal-json/argv -- +crater file rm user/../victim --yes --json --no-interactive +-- en/27-file-rm-traversal-json/exit -- +2 +-- en/27-file-rm-traversal-json/stdout -- +-- en/27-file-rm-traversal-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_INVALID_FLAG_VALUE", + "message": "remote path for removal must not contain . or .. segments: \"user/../victim\"" +} +-- en/28-file-rm-missing-yes-json/argv -- +crater file rm user/victim --json --no-interactive +-- en/28-file-rm-missing-yes-json/exit -- +2 +-- en/28-file-rm-missing-yes-json/stdout -- +-- en/28-file-rm-missing-yes-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_MISSING_REQUIRED_FLAG", + "message": "--yes is required in non-interactive mode" +} +-- en/29-file-rm-json-implies-noninteractive/argv -- +crater file rm user/victim --json +-- en/29-file-rm-json-implies-noninteractive/exit -- +2 +-- en/29-file-rm-json-implies-noninteractive/stdout -- +-- en/29-file-rm-json-implies-noninteractive/stderr -- +{ + "category": "usage_error", + "code": "ERR_MISSING_REQUIRED_FLAG", + "message": "--yes is required in non-interactive mode" +} +-- en/30-file-rm-404-json/argv -- +crater file rm user/实验 data/result --yes --json --no-interactive +-- en/30-file-rm-404-json/exit -- +4 +-- en/30-file-rm-404-json/stdout -- +-- en/30-file-rm-404-json/stderr -- +{ + "category": "api_error", + "code": "ERR_NOT_FOUND_404", + "message": "HTTP 404: simulated", + "context": { + "crater_code": 404, + "http_status": 404, + "msg": "simulated" + } +} +-- en/31-file-rm-recursive-404-json/argv -- +crater file rm account/runs/old --recursive --yes --json --no-interactive +-- en/31-file-rm-recursive-404-json/exit -- +4 +-- en/31-file-rm-recursive-404-json/stdout -- +-- en/31-file-rm-recursive-404-json/stderr -- +{ + "category": "api_error", + "code": "ERR_NOT_FOUND_404", + "message": "HTTP 404: simulated", + "context": { + "crater_code": 404, + "http_status": 404, + "msg": "simulated" + } +} diff --git a/cli/testdata/snapshots/file/file_upload.zh-CN.txtar b/cli/testdata/snapshots/file/file_upload.zh-CN.txtar new file mode 100644 index 000000000..78d147d77 --- /dev/null +++ b/cli/testdata/snapshots/file/file_upload.zh-CN.txtar @@ -0,0 +1,426 @@ +# Crater CLI snapshot bundle (txtar). Regenerate: make snapshot-update (or UPDATE_SNAPSHOTS=1 go test ./test/snapshots/...) +-- zh-CN/01-file-typo-json/argv -- +crater file get --json --no-interactive +-- zh-CN/01-file-typo-json/exit -- +2 +-- zh-CN/01-file-typo-json/stdout -- +-- zh-CN/01-file-typo-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_UNKNOWN_COMMAND", + "message": "unknown command \"get\" for \"crater file\"\nRun \"crater file --help\" for usage." +} +-- zh-CN/01b-file-typo-text/argv -- +crater file get --no-interactive +-- zh-CN/01b-file-typo-text/exit -- +2 +-- zh-CN/01b-file-typo-text/stdout -- +-- zh-CN/01b-file-typo-text/stderr -- +Error: + unknown command "get" for "crater file" + Run "crater file --help" for usage. +-- zh-CN/02-file-upload-missing-local-json/argv -- +crater file upload --json --no-interactive +-- zh-CN/02-file-upload-missing-local-json/exit -- +2 +-- zh-CN/02-file-upload-missing-local-json/stdout -- +-- zh-CN/02-file-upload-missing-local-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_MISSING_REQUIRED_FLAG", + "message": "缺少必要参数：本地文件 (\u003clocal-file\u003e)" +} +-- zh-CN/03-file-upload-missing-remote-json/argv -- +crater file upload .snapshot-upload-zh-CN.bin --json --no-interactive +-- zh-CN/03-file-upload-missing-remote-json/exit -- +2 +-- zh-CN/03-file-upload-missing-remote-json/stdout -- +-- zh-CN/03-file-upload-missing-remote-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_MISSING_REQUIRED_FLAG", + "message": "缺少必要参数：远端路径 (\u003cremote-path\u003e)" +} +-- zh-CN/04-file-upload-extra-arg-json/argv -- +crater file upload .snapshot-upload-zh-CN.bin user/a.bin extra --json --no-interactive +-- zh-CN/04-file-upload-extra-arg-json/exit -- +2 +-- zh-CN/04-file-upload-extra-arg-json/stdout -- +-- zh-CN/04-file-upload-extra-arg-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_INVALID_FLAG_VALUE", + "message": "crater file upload 的参数过多：收到 3 个，最多允许 2 个" +} +-- zh-CN/05-file-upload-traversal-json/argv -- +crater file upload .snapshot-upload-zh-CN.bin user/../public/a.bin --json --no-interactive +-- zh-CN/05-file-upload-traversal-json/exit -- +2 +-- zh-CN/05-file-upload-traversal-json/stdout -- +-- zh-CN/05-file-upload-traversal-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_INVALID_FLAG_VALUE", + "message": "无效的远端路径 \"user/../public/a.bin\"" +} +-- zh-CN/06-file-upload-root-json/argv -- +crater file upload .snapshot-upload-zh-CN.bin user --json --no-interactive +-- zh-CN/06-file-upload-root-json/exit -- +2 +-- zh-CN/06-file-upload-root-json/stdout -- +-- zh-CN/06-file-upload-root-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_INVALID_FLAG_VALUE", + "message": "远端文件必须指向 user、public 或 account 下的具体条目：\"user\"" +} +-- zh-CN/07-file-upload-directory-json/argv -- +crater file upload . user/a.bin --json --no-interactive +-- zh-CN/07-file-upload-directory-json/exit -- +2 +-- zh-CN/07-file-upload-directory-json/stdout -- +-- zh-CN/07-file-upload-directory-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_INVALID_FLAG_VALUE", + "message": "本地路径不是普通文件：\".\"" +} +-- zh-CN/08-file-upload-404-json/argv -- +crater file upload .snapshot-upload-zh-CN.bin user/实验 data/result.bin --json --no-interactive +-- zh-CN/08-file-upload-404-json/exit -- +4 +-- zh-CN/08-file-upload-404-json/stdout -- +-- zh-CN/08-file-upload-404-json/stderr -- +{ + "category": "api_error", + "code": "ERR_NOT_FOUND_404", + "message": "请求失败（HTTP 404）：simulated", + "context": { + "crater_code": 404, + "http_status": 404, + "msg": "simulated" + } +} +-- zh-CN/09-file-help/argv -- +crater file --help +-- zh-CN/09-file-help/exit -- +0 +-- zh-CN/09-file-help/stdout -- +管理用户、公共及当前账户存储空间中的文件和目录。 + +Usage: + crater file [flags] + crater file [command] + +Available Commands: + mkdir 创建单个远端目录 + mv 移动单个远端文件或目录 + rm 删除单个远端文件或目录 + upload 上传单个本地文件 + +Global Flags: + -h, --help 显示帮助信息 + --json 以原始 JSON 格式输出 + --no-interactive 禁用交互式提示 + +Use "crater file [command] --help" for more information about a command. +-- zh-CN/09-file-help/stderr -- +-- zh-CN/10-file-upload-help/argv -- +crater file upload --help +-- zh-CN/10-file-upload-help/exit -- +0 +-- zh-CN/10-file-upload-help/stdout -- +将一个本地普通文件流式上传到 user、public 或 account 存储空间下的路径。 + +Usage: + crater file upload [flags] + +Flags: + --overwrite 替换已存在的远端文件 + +Global Flags: + -h, --help 显示帮助信息 + --json 以原始 JSON 格式输出 + --no-interactive 禁用交互式提示 +-- zh-CN/10-file-upload-help/stderr -- +-- zh-CN/11-file-mkdir-help/argv -- +crater file mkdir --help +-- zh-CN/11-file-mkdir-help/exit -- +0 +-- zh-CN/11-file-mkdir-help/stdout -- +在 user、public 或 account 存储空间下创建单个目录，且不会自动创建缺失的父目录。 + +Usage: + crater file mkdir [flags] + +Global Flags: + -h, --help 显示帮助信息 + --json 以原始 JSON 格式输出 + --no-interactive 禁用交互式提示 +-- zh-CN/11-file-mkdir-help/stderr -- +-- zh-CN/12-file-mkdir-missing-json/argv -- +crater file mkdir --json --no-interactive +-- zh-CN/12-file-mkdir-missing-json/exit -- +2 +-- zh-CN/12-file-mkdir-missing-json/stdout -- +-- zh-CN/12-file-mkdir-missing-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_MISSING_REQUIRED_FLAG", + "message": "缺少必要参数：远端目录 (\u003cremote-path\u003e)" +} +-- zh-CN/13-file-mkdir-root-json/argv -- +crater file mkdir user --json --no-interactive +-- zh-CN/13-file-mkdir-root-json/exit -- +2 +-- zh-CN/13-file-mkdir-root-json/stdout -- +-- zh-CN/13-file-mkdir-root-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_INVALID_FLAG_VALUE", + "message": "远端路径必须指向 user、public 或 account 下的具体条目：\"user\"" +} +-- zh-CN/14-file-mkdir-404-json/argv -- +crater file mkdir user/实验 data --json --no-interactive +-- zh-CN/14-file-mkdir-404-json/exit -- +4 +-- zh-CN/14-file-mkdir-404-json/stdout -- +-- zh-CN/14-file-mkdir-404-json/stderr -- +{ + "category": "api_error", + "code": "ERR_NOT_FOUND_404", + "message": "请求失败（HTTP 404）：simulated", + "context": { + "crater_code": 404, + "http_status": 404, + "msg": "simulated" + } +} +-- zh-CN/15-file-mv-help/argv -- +crater file mv --help +-- zh-CN/15-file-mv-help/exit -- +0 +-- zh-CN/15-file-mv-help/stdout -- +将单个远端存储条目移动到指定目标路径，且不会覆盖已存在的条目。 + +Usage: + crater file mv [flags] + +Global Flags: + -h, --help 显示帮助信息 + --json 以原始 JSON 格式输出 + --no-interactive 禁用交互式提示 +-- zh-CN/15-file-mv-help/stderr -- +-- zh-CN/16-file-mv-missing-source-json/argv -- +crater file mv --json --no-interactive +-- zh-CN/16-file-mv-missing-source-json/exit -- +2 +-- zh-CN/16-file-mv-missing-source-json/stdout -- +-- zh-CN/16-file-mv-missing-source-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_MISSING_REQUIRED_FLAG", + "message": "缺少必要参数：源路径 (\u003csource-path\u003e)" +} +-- zh-CN/17-file-mv-missing-destination-json/argv -- +crater file mv user/source --json --no-interactive +-- zh-CN/17-file-mv-missing-destination-json/exit -- +2 +-- zh-CN/17-file-mv-missing-destination-json/stdout -- +-- zh-CN/17-file-mv-missing-destination-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_MISSING_REQUIRED_FLAG", + "message": "缺少必要参数：目标路径 (\u003cdestination-path\u003e)" +} +-- zh-CN/18-file-mv-invalid-operands-json/argv -- +crater file mv user/../source account\destination --json --no-interactive +-- zh-CN/18-file-mv-invalid-operands-json/exit -- +2 +-- zh-CN/18-file-mv-invalid-operands-json/stdout -- +-- zh-CN/18-file-mv-invalid-operands-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_INVALID_FLAG_VALUE", + "message": "无效的远端路径 \"user/../source\"\n无效的远端路径 \"account\\\\destination\"", + "context": { + "issues": [ + { + "code": "ERR_INVALID_FLAG_VALUE", + "field": "source-path", + "message": "无效的远端路径 \"user/../source\"" + }, + { + "code": "ERR_INVALID_FLAG_VALUE", + "field": "destination-path", + "message": "无效的远端路径 \"account\\\\destination\"" + } + ] + } +} +-- zh-CN/19-file-mv-same-json/argv -- +crater file mv user/source /user//./source --json --no-interactive +-- zh-CN/19-file-mv-same-json/exit -- +2 +-- zh-CN/19-file-mv-same-json/stdout -- +-- zh-CN/19-file-mv-same-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_INVALID_FLAG_VALUE", + "message": "移动目标必须与源路径不同：\"user/source\"" +} +-- zh-CN/20-file-mv-descendant-json/argv -- +crater file mv user/source user/source/nested --json --no-interactive +-- zh-CN/20-file-mv-descendant-json/exit -- +2 +-- zh-CN/20-file-mv-descendant-json/stdout -- +-- zh-CN/20-file-mv-descendant-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_INVALID_FLAG_VALUE", + "message": "移动目标 \"user/source/nested\" 不能位于源路径 \"user/source\" 内" +} +-- zh-CN/21-file-mv-404-json/argv -- +crater file mv user/source account/实验 data/result --json --no-interactive +-- zh-CN/21-file-mv-404-json/exit -- +4 +-- zh-CN/21-file-mv-404-json/stdout -- +-- zh-CN/21-file-mv-404-json/stderr -- +{ + "category": "api_error", + "code": "ERR_NOT_FOUND_404", + "message": "请求失败（HTTP 404）：simulated", + "context": { + "crater_code": 404, + "http_status": 404, + "msg": "simulated" + } +} +-- zh-CN/22-file-rm-help/argv -- +crater file rm --help +-- zh-CN/22-file-rm-help/exit -- +0 +-- zh-CN/22-file-rm-help/stdout -- +删除 user、public 或 account 存储空间下的单个远端条目。删除目录时必须提供 --recursive。 + +Usage: + crater file rm [flags] + +Flags: + --recursive 删除目录及其中的全部内容 + -y, --yes 跳过确认并执行删除 + +Global Flags: + -h, --help 显示帮助信息 + --json 以原始 JSON 格式输出 + --no-interactive 禁用交互式提示 +-- zh-CN/22-file-rm-help/stderr -- +-- zh-CN/23-file-rm-missing-json/argv -- +crater file rm --json --no-interactive +-- zh-CN/23-file-rm-missing-json/exit -- +2 +-- zh-CN/23-file-rm-missing-json/stdout -- +-- zh-CN/23-file-rm-missing-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_MISSING_REQUIRED_FLAG", + "message": "缺少必要参数：远端路径 (\u003cremote-path\u003e)" +} +-- zh-CN/24-file-rm-extra-arg-json/argv -- +crater file rm user/a user/b --json --no-interactive +-- zh-CN/24-file-rm-extra-arg-json/exit -- +2 +-- zh-CN/24-file-rm-extra-arg-json/stdout -- +-- zh-CN/24-file-rm-extra-arg-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_INVALID_FLAG_VALUE", + "message": "crater file rm 的参数过多：收到 2 个，最多允许 1 个" +} +-- zh-CN/25-file-rm-root-json/argv -- +crater file rm user --yes --json --no-interactive +-- zh-CN/25-file-rm-root-json/exit -- +2 +-- zh-CN/25-file-rm-root-json/stdout -- +-- zh-CN/25-file-rm-root-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_INVALID_FLAG_VALUE", + "message": "拒绝删除存储根目录：\"user\"" +} +-- zh-CN/26-file-rm-dot-json/argv -- +crater file rm user/./victim --yes --json --no-interactive +-- zh-CN/26-file-rm-dot-json/exit -- +2 +-- zh-CN/26-file-rm-dot-json/stdout -- +-- zh-CN/26-file-rm-dot-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_INVALID_FLAG_VALUE", + "message": "待删除的远端路径不能包含 . 或 .. 路径段：\"user/./victim\"" +} +-- zh-CN/27-file-rm-traversal-json/argv -- +crater file rm user/../victim --yes --json --no-interactive +-- zh-CN/27-file-rm-traversal-json/exit -- +2 +-- zh-CN/27-file-rm-traversal-json/stdout -- +-- zh-CN/27-file-rm-traversal-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_INVALID_FLAG_VALUE", + "message": "待删除的远端路径不能包含 . 或 .. 路径段：\"user/../victim\"" +} +-- zh-CN/28-file-rm-missing-yes-json/argv -- +crater file rm user/victim --json --no-interactive +-- zh-CN/28-file-rm-missing-yes-json/exit -- +2 +-- zh-CN/28-file-rm-missing-yes-json/stdout -- +-- zh-CN/28-file-rm-missing-yes-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_MISSING_REQUIRED_FLAG", + "message": "非交互模式下必须提供 --yes" +} +-- zh-CN/29-file-rm-json-implies-noninteractive/argv -- +crater file rm user/victim --json +-- zh-CN/29-file-rm-json-implies-noninteractive/exit -- +2 +-- zh-CN/29-file-rm-json-implies-noninteractive/stdout -- +-- zh-CN/29-file-rm-json-implies-noninteractive/stderr -- +{ + "category": "usage_error", + "code": "ERR_MISSING_REQUIRED_FLAG", + "message": "非交互模式下必须提供 --yes" +} +-- zh-CN/30-file-rm-404-json/argv -- +crater file rm user/实验 data/result --yes --json --no-interactive +-- zh-CN/30-file-rm-404-json/exit -- +4 +-- zh-CN/30-file-rm-404-json/stdout -- +-- zh-CN/30-file-rm-404-json/stderr -- +{ + "category": "api_error", + "code": "ERR_NOT_FOUND_404", + "message": "请求失败（HTTP 404）：simulated", + "context": { + "crater_code": 404, + "http_status": 404, + "msg": "simulated" + } +} +-- zh-CN/31-file-rm-recursive-404-json/argv -- +crater file rm account/runs/old --recursive --yes --json --no-interactive +-- zh-CN/31-file-rm-recursive-404-json/exit -- +4 +-- zh-CN/31-file-rm-recursive-404-json/stdout -- +-- zh-CN/31-file-rm-recursive-404-json/stderr -- +{ + "category": "api_error", + "code": "ERR_NOT_FOUND_404", + "message": "请求失败（HTTP 404）：simulated", + "context": { + "crater_code": 404, + "http_status": 404, + "msg": "simulated" + } +} diff --git a/frontend/src/services/error_code.ts b/frontend/src/services/error_code.ts index 058cb1271..47abf8294 100644 --- a/frontend/src/services/error_code.ts +++ b/frontend/src/services/error_code.ts @@ -41,6 +41,7 @@ export const ERROR_USER_BANNED: ErrorCode = 40303 export const ERROR_DATA_BASE_NOT_FOUND: ErrorCode = 40401 export const ERROR_SERVICE_SSHD_NOT_FOUND: ErrorCode = 40402 export const ERROR_K8S_RESOURCE_NOT_FOUND: ErrorCode = 40403 +export const ERROR_STORAGE_RESOURCE_NOT_FOUND: ErrorCode = 40404 // 405xx - Method Not Allowed export const ERROR_METHOD_NOT_ALLOWED: ErrorCode = 40501 diff --git a/output/playwright/issue-480-file-upload.png b/output/playwright/issue-480-file-upload.png new file mode 100644 index 000000000..70936666e Binary files /dev/null and b/output/playwright/issue-480-file-upload.png differ diff --git a/output/playwright/issue-481-file-mkdir-move.png b/output/playwright/issue-481-file-mkdir-move.png new file mode 100644 index 000000000..c209123f9 Binary files /dev/null and b/output/playwright/issue-481-file-mkdir-move.png differ diff --git a/output/playwright/issue-482-file-remove.png b/output/playwright/issue-482-file-remove.png new file mode 100644 index 000000000..8be03c0b2 Binary files /dev/null and b/output/playwright/issue-482-file-remove.png differ