diff --git a/backend/go.mod b/backend/go.mod index 09d21c0c8..54b5eb437 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -26,6 +26,7 @@ require ( golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 golang.org/x/net v0.41.0 golang.org/x/sync v0.16.0 + golang.org/x/sys v0.34.0 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 @@ -143,7 +144,6 @@ require ( golang.org/x/arch v0.18.0 // indirect golang.org/x/mod v0.25.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/sys v0.34.0 // indirect golang.org/x/term v0.33.0 // indirect golang.org/x/text v0.27.0 // indirect golang.org/x/time v0.9.0 // indirect diff --git a/backend/internal/storage/file.go b/backend/internal/storage/file.go index eed53600c..cd1203b6f 100644 --- a/backend/internal/storage/file.go +++ b/backend/internal/storage/file.go @@ -874,6 +874,7 @@ func RegisterFile(webdavGroup *gin.RouterGroup) { webdavGroup.GET("/admin/files", GetAllFiles) webdavGroup.GET("/admin/files/*path", GetAllFiles) webdavGroup.GET("/download/*path", Download) + webdavGroup.POST("/upload/*path", UploadFile) webdavGroup.DELETE("/delete/*path", DeleteFile) webdavGroup.GET("/userspace", GetUserSpace) webdavGroup.GET("/queuespace", GetAccountSpace) diff --git a/backend/internal/storage/upload.go b/backend/internal/storage/upload.go new file mode 100644 index 000000000..f203e92ba --- /dev/null +++ b/backend/internal/storage/upload.go @@ -0,0 +1,489 @@ +package storage + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "unicode" + + "github.com/gin-gonic/gin" + "k8s.io/klog/v2" + + "github.com/raids-lab/crater/dao/model" + "github.com/raids-lab/crater/internal/bizerr" + "github.com/raids-lab/crater/internal/resputil" + "github.com/raids-lab/crater/internal/util" +) + +const ( + uploadFileMode os.FileMode = 0o644 + uploadStageDirMode os.FileMode = 0o700 + uploadStageFileMode os.FileMode = 0o600 + uploadStageAttempts = 16 + uploadStageRandomBytes = 16 + uploadStagePayload = "payload" + parentPathSegment = ".." +) + +var ( + errUploadTargetExists = errors.New("upload target exists") + errUploadTargetNotRegular = errors.New("upload target is not a regular file") + errUploadParentInvalid = errors.New("upload parent is missing, invalid, or outside the authorized storage root") +) + +type uploadSourceError struct { + cause error +} + +func (e *uploadSourceError) Error() string { + return "read upload source: " + e.cause.Error() +} + +func (e *uploadSourceError) Unwrap() error { + if e == nil { + return nil + } + return e.cause +} + +type uploadOutcome struct { + Bytes int64 + Overwritten bool +} + +type uploadResponse struct { + RemotePath string `json:"remote_path"` + Bytes int64 `json:"bytes"` + Overwritten bool `json:"overwritten"` +} + +type uploadHandlerDeps struct { + authenticate func(*gin.Context) (util.JWTMessage, error) + permission func(string, util.JWTMessage, *gin.Context) model.FilePermission + redirect func(*gin.Context, string, util.JWTMessage) (string, error) + openTarget func(string, string, string) (*os.Root, string, error) + stagePublish func(io.Reader, *os.Root, string, bool, os.FileMode) (uploadOutcome, error) + storageRoot string +} + +func defaultUploadHandlerDeps() uploadHandlerDeps { + return uploadHandlerDeps{ + authenticate: CheckJWTToken, + permission: GetPermission, + redirect: Redirect, + openTarget: openUploadTarget, + stagePublish: stageAndPublishFile, + storageRoot: storageRootDir, + } +} + +// UploadFile atomically publishes one raw request body into an ordinary-user +// storage path. It deliberately uses a dedicated endpoint because the bundled +// WebDAV PUT handler truncates an existing target before the request completes. +func UploadFile(c *gin.Context) { + uploadFileWithDeps(c, defaultUploadHandlerDeps()) +} + +func uploadFileWithDeps(c *gin.Context, deps uploadHandlerDeps) { + token, err := deps.authenticate(c) + if err != nil { + resputil.HandleError(c, bizerr.Auth.TokenInvalid.New("invalid token")) + return + } + + overwrite, err := parseUploadOverwrite(c.Query("overwrite")) + if err != nil { + resputil.HandleError(c, bizerr.BadRequest.ParameterError.New("overwrite must be true or false")) + return + } + + logicalPath, err := normalizeUploadLogicalPath(c.Param("path")) + if err != nil { + resputil.HandleError(c, bizerr.BadRequest.ParameterError.New("invalid remote file path")) + return + } + + if permission := deps.permission(logicalPath, token, c); permission != model.ReadWrite { + resputil.HandleError(c, bizerr.Forbidden.PermissionDenied.New("write permission is required")) + return + } + + realPath, err := deps.redirect(c, logicalPath, token) + if err != nil { + klog.Errorf("resolve upload target: %v", err) + resputil.HandleError(c, bizerr.Internal.FileSystemError.Wrap(err, "failed to resolve upload target")) + return + } + logicalRoot := strings.Split(logicalPath, "/")[0] + realRoot, err := deps.redirect(c, logicalRoot, token) + if err != nil { + klog.Errorf("resolve upload storage root: %v", err) + resputil.HandleError(c, bizerr.Internal.FileSystemError.Wrap(err, "failed to resolve upload target")) + return + } + + parentRoot, targetName, err := deps.openTarget(deps.storageRoot, realRoot, realPath) + if err != nil { + if errors.Is(err, errUploadParentInvalid) { + resputil.HandleError(c, bizerr.Conflict.ResourceStatusError.New("upload parent directory is unavailable")) + return + } + klog.Errorf("open upload target: %v", err) + resputil.HandleError(c, bizerr.Internal.FileSystemError.Wrap(err, "failed to access upload storage")) + return + } + defer parentRoot.Close() + + outcome, err := deps.stagePublish(c.Request.Body, parentRoot, targetName, overwrite, uploadFileMode) + if err != nil { + switch { + case errors.Is(err, errUploadTargetExists): + resputil.HandleError(c, bizerr.Conflict.ResourceAlreadyExists.New("target file already exists")) + case errors.Is(err, errUploadTargetNotRegular): + resputil.HandleError(c, bizerr.Conflict.ResourceStatusError.New("target path is not a regular file")) + case errors.Is(err, errUploadParentInvalid): + resputil.HandleError(c, bizerr.Conflict.ResourceStatusError.New("upload parent directory is unavailable")) + default: + var sourceErr *uploadSourceError + if errors.As(err, &sourceErr) { + resputil.HandleError(c, bizerr.BadRequest.InvalidRequest.Wrap(sourceErr, "failed to read upload body")) + return + } + klog.Errorf("publish uploaded file: %v", err) + resputil.HandleError(c, bizerr.Internal.FileSystemError.Wrap(err, "failed to store uploaded file")) + } + return + } + + status := http.StatusCreated + if outcome.Overwritten { + status = http.StatusOK + } + c.JSON(status, resputil.Response[uploadResponse]{ + Code: resputil.OK, + Data: uploadResponse{ + RemotePath: logicalPath, + Bytes: outcome.Bytes, + Overwritten: outcome.Overwritten, + }, + Message: "", + }) +} + +func parseUploadOverwrite(raw string) (bool, error) { + switch raw { + case "", "false": + return false, nil + case "true": + return true, nil + default: + return false, errors.New("invalid overwrite value") + } +} + +func normalizeUploadLogicalPath(raw string) (string, error) { + if strings.ContainsRune(raw, '\\') { + return "", errors.New("backslashes are not allowed") + } + for _, character := range raw { + if unicode.IsControl(character) { + return "", errors.New("control characters are not allowed") + } + } + + trimmed := strings.Trim(raw, "/") + rawSegments := strings.Split(trimmed, "/") + segments := make([]string, 0, len(rawSegments)) + for _, segment := range rawSegments { + if segment == parentPathSegment { + return "", errors.New("parent traversal is not allowed") + } + if segment == "" || segment == "." { + continue + } + segments = append(segments, segment) + } + if len(segments) < 2 { + return "", errors.New("a file below a logical root is required") + } + switch segments[0] { + case model.UserPath, model.PublicPath, model.AccountPath: + default: + return "", errors.New("invalid logical root") + } + return strings.Join(segments, "/"), nil +} + +// openUploadTarget returns a directory handle anchored to the resolved target +// parent. os.Root rejects symlink traversal outside both the configured storage +// root and the caller's authorized real root, and remains anchored if a parent +// directory is renamed concurrently. +func openUploadTarget(storageRoot, authorizedRealRoot, targetRealPath string) (*os.Root, string, error) { + storage, err := os.OpenRoot(storageRoot) + if err != nil { + return nil, "", err + } + defer storage.Close() + + if !strings.HasPrefix(targetRealPath, strings.TrimSuffix(authorizedRealRoot, "/")+"/") { + return nil, "", errUploadParentInvalid + } + authorizedPath, err := cleanStorageRelativePath(authorizedRealRoot) + if err != nil { + return nil, "", errUploadParentInvalid + } + targetPath, err := cleanStorageRelativePath(targetRealPath) + if err != nil { + return nil, "", errUploadParentInvalid + } + targetRelative, err := filepath.Rel(authorizedPath, targetPath) + if err != nil || targetRelative == "." || pathEscapesRoot(targetRelative) { + return nil, "", errUploadParentInvalid + } + + authorized, err := storage.OpenRoot(authorizedPath) + if err != nil { + return nil, "", errUploadParentInvalid + } + defer authorized.Close() + + parentRelative := filepath.Dir(targetRelative) + targetName := filepath.Base(targetRelative) + if targetName == "." || targetName == string(filepath.Separator) { + return nil, "", errUploadParentInvalid + } + parent, err := authorized.OpenRoot(parentRelative) + if err != nil { + return nil, "", errUploadParentInvalid + } + return parent, targetName, nil +} + +func cleanStorageRelativePath(raw string) (string, error) { + if raw == "" || strings.ContainsRune(raw, '\\') { + return "", errUploadParentInvalid + } + for _, character := range raw { + if unicode.IsControl(character) { + return "", errUploadParentInvalid + } + } + + raw, err := trimLegacyStorageRootSlash(raw) + if err != nil { + return "", err + } + canonical, err := canonicalStorageSegments(raw) + if err != nil { + return "", err + } + normalized := filepath.FromSlash(canonical) + if filepath.Clean(normalized) != normalized || filepath.IsAbs(normalized) || pathEscapesRoot(normalized) { + return "", errUploadParentInvalid + } + return normalized, nil +} + +func trimLegacyStorageRootSlash(raw string) (string, error) { + if !strings.HasPrefix(raw, "/") { + return raw, nil + } + raw = strings.TrimPrefix(raw, "/") + if raw == "" || strings.HasPrefix(raw, "/") { + return "", errUploadParentInvalid + } + return raw, nil +} + +func canonicalStorageSegments(raw string) (string, error) { + // Historical User.Space and Account.Space records may start with "/". + // Redirect joins them after the configured prefix and produces one empty + // separator segment (for example "users//space/alice"). Accept exactly one + // such legacy marker, while rejecting every other ambiguous empty segment. + segments := strings.Split(raw, "/") + canonical := make([]string, 0, len(segments)) + legacyEmptySeen := false + for index, segment := range segments { + if segment == "" { + if legacyEmptySeen || index == 0 || index == len(segments)-1 { + return "", errUploadParentInvalid + } + legacyEmptySeen = true + continue + } + if segment == "." || segment == parentPathSegment { + return "", errUploadParentInvalid + } + canonical = append(canonical, segment) + } + return strings.Join(canonical, "/"), nil +} + +func pathEscapesRoot(path string) bool { + return path == parentPathSegment || + strings.HasPrefix(path, parentPathSegment+string(filepath.Separator)) +} + +func stageAndPublishFile( + source io.Reader, + parent *os.Root, + targetName string, + overwrite bool, + mode os.FileMode, +) (uploadOutcome, error) { + if err := validateUploadTarget(parent, targetName, overwrite); err != nil { + return uploadOutcome{}, err + } + + stageName, stageRoot, staged, err := createUploadStage(parent) + if err != nil { + return uploadOutcome{}, err + } + defer cleanupUploadStage(parent, stageRoot, stageName) + + written, err := writeUploadStage(source, staged, mode) + if err != nil { + return uploadOutcome{}, err + } + return publishStagedUpload(stageRoot, parent, targetName, overwrite, written) +} + +func validateUploadTarget(parent *os.Root, targetName string, overwrite bool) error { + if parent == nil || targetName == "" || targetName == "." || + targetName == parentPathSegment || filepath.Base(targetName) != targetName { + return errUploadParentInvalid + } + targetInfo, err := parent.Lstat(targetName) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + if !targetInfo.Mode().IsRegular() { + return errUploadTargetNotRegular + } + if !overwrite { + return errUploadTargetExists + } + return nil +} + +func writeUploadStage(source io.Reader, staged *os.File, mode os.FileMode) (int64, error) { + tracked := &trackedUploadSource{source: source} + written, copyErr := io.Copy(staged, tracked) + if copyErr != nil { + _ = staged.Close() + if tracked.readErr != nil { + return 0, &uploadSourceError{cause: tracked.readErr} + } + return 0, copyErr + } + if err := staged.Chmod(mode); err != nil { + _ = staged.Close() + return 0, err + } + if err := staged.Sync(); err != nil { + _ = staged.Close() + return 0, err + } + if err := staged.Close(); err != nil { + return 0, err + } + return written, nil +} + +func publishStagedUpload( + stageRoot, parent *os.Root, + targetName string, + overwrite bool, + written int64, +) (uploadOutcome, error) { + for retry := 0; retry < 2; retry++ { + if err := publishUploadNoClobber(stageRoot, parent, targetName); err == nil { + return uploadOutcome{Bytes: written}, nil + } else if !os.IsExist(err) { + return uploadOutcome{}, err + } + + targetInfo, statErr := parent.Lstat(targetName) + if os.IsNotExist(statErr) { + continue + } + if statErr != nil { + return uploadOutcome{}, statErr + } + if !targetInfo.Mode().IsRegular() { + return uploadOutcome{}, errUploadTargetNotRegular + } + if !overwrite { + return uploadOutcome{}, errUploadTargetExists + } + if err := publishUploadOverwrite(stageRoot, parent, targetName); err != nil { + return uploadOutcome{}, err + } + return uploadOutcome{Bytes: written, Overwritten: true}, nil + } + return uploadOutcome{}, errUploadTargetExists +} + +func createUploadStage(parent *os.Root) (string, *os.Root, *os.File, error) { + for attempt := 0; attempt < uploadStageAttempts; attempt++ { + random := make([]byte, uploadStageRandomBytes) + if _, err := rand.Read(random); err != nil { + return "", nil, nil, err + } + stageName := ".crater-upload-" + hex.EncodeToString(random) + if err := parent.Mkdir(stageName, uploadStageDirMode); err != nil { + if os.IsExist(err) { + continue + } + return "", nil, nil, err + } + stageRoot, err := parent.OpenRoot(stageName) + if err != nil { + _ = parent.Remove(stageName) + return "", nil, nil, err + } + staged, err := stageRoot.OpenFile( + uploadStagePayload, + os.O_WRONLY|os.O_CREATE|os.O_EXCL, + uploadStageFileMode, + ) + if err != nil { + _ = stageRoot.Close() + _ = parent.Remove(stageName) + return "", nil, nil, err + } + return stageName, stageRoot, staged, nil + } + return "", nil, nil, errors.New("could not allocate a private upload staging directory") +} + +func cleanupUploadStage(parent, stageRoot *os.Root, stageName string) { + if stageRoot != nil { + _ = stageRoot.Remove(uploadStagePayload) + _ = stageRoot.Close() + } + if parent != nil && stageName != "" { + _ = parent.Remove(stageName) + } +} + +type trackedUploadSource struct { + source io.Reader + readErr error +} + +func (reader *trackedUploadSource) Read(data []byte) (int, error) { + read, err := reader.source.Read(data) + if err != nil && !errors.Is(err, io.EOF) { + reader.readErr = err + } + return read, err +} diff --git a/backend/internal/storage/upload_publish_other.go b/backend/internal/storage/upload_publish_other.go new file mode 100644 index 000000000..2d2f3367a --- /dev/null +++ b/backend/internal/storage/upload_publish_other.go @@ -0,0 +1,18 @@ +//go:build !darwin && !dragonfly && !freebsd && !linux && !netbsd && !openbsd + +package storage + +import ( + "errors" + "os" +) + +var errUploadPublishUnsupported = errors.New("atomic upload publishing is unsupported on this platform") + +func publishUploadNoClobber(*os.Root, *os.Root, string) error { + return errUploadPublishUnsupported +} + +func publishUploadOverwrite(*os.Root, *os.Root, string) error { + return errUploadPublishUnsupported +} diff --git a/backend/internal/storage/upload_publish_unix.go b/backend/internal/storage/upload_publish_unix.go new file mode 100644 index 000000000..f438ec015 --- /dev/null +++ b/backend/internal/storage/upload_publish_unix.go @@ -0,0 +1,60 @@ +//go:build darwin || dragonfly || freebsd || linux || netbsd || openbsd + +package storage + +import ( + "os" + + "golang.org/x/sys/unix" +) + +// Publish from the already-open staging directory. Resolving the source +// relative to its directory descriptor prevents a writable-parent rename race +// from substituting another staging directory after the upload completes. +func publishUploadNoClobber(stage, parent *os.Root, targetName string) error { + stageDirectory, parentDirectory, err := openUploadDirectoryHandles(stage, parent) + if err != nil { + return err + } + defer stageDirectory.Close() + defer parentDirectory.Close() + return unix.Linkat( + int(stageDirectory.Fd()), + uploadStagePayload, + int(parentDirectory.Fd()), + targetName, + 0, + ) +} + +func publishUploadOverwrite(stage, parent *os.Root, targetName string) error { + stageDirectory, parentDirectory, err := openUploadDirectoryHandles(stage, parent) + if err != nil { + return err + } + defer stageDirectory.Close() + defer parentDirectory.Close() + return unix.Renameat( + int(stageDirectory.Fd()), + uploadStagePayload, + int(parentDirectory.Fd()), + targetName, + ) +} + +func openUploadDirectoryHandles(stage, parent *os.Root) ( + stageDirectory *os.File, + parentDirectory *os.File, + err error, +) { + stageDirectory, err = stage.Open(".") + if err != nil { + return nil, nil, err + } + parentDirectory, err = parent.Open(".") + if err != nil { + _ = stageDirectory.Close() + return nil, nil, err + } + return stageDirectory, parentDirectory, nil +} diff --git a/backend/internal/storage/upload_test.go b/backend/internal/storage/upload_test.go new file mode 100644 index 000000000..0b9afd288 --- /dev/null +++ b/backend/internal/storage/upload_test.go @@ -0,0 +1,669 @@ +package storage + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/gin-gonic/gin" + + "github.com/raids-lab/crater/dao/model" + "github.com/raids-lab/crater/internal/util" +) + +const testUploadMode os.FileMode = 0o640 + +func TestStageAndPublishFileStreamsBinaryAtomically(t *testing.T) { + directory := t.TempDir() + payload := []byte{0x00, 0xff, 'c', 'r', 'a', 't', 'e', 'r'} + + outcome, err := stageInDirectory(t, directory, "data.bin", bytes.NewReader(payload), false) + if err != nil { + t.Fatalf("stageAndPublishFile: %v", err) + } + if outcome.Bytes != int64(len(payload)) || outcome.Overwritten { + t.Fatalf("outcome = %#v", outcome) + } + target := filepath.Join(directory, "data.bin") + assertStoredFile(t, target, payload) + info, err := os.Stat(target) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o640 { + t.Fatalf("mode = %o, want 640", info.Mode().Perm()) + } + assertNoUploadTemps(t, directory) +} + +func TestStageAndPublishFilePublishesEmptyFile(t *testing.T) { + directory := t.TempDir() + outcome, err := stageInDirectory(t, directory, "empty.bin", bytes.NewReader(nil), false) + if err != nil { + t.Fatalf("stageAndPublishFile: %v", err) + } + if outcome.Bytes != 0 || outcome.Overwritten { + t.Fatalf("outcome = %#v", outcome) + } + assertStoredFile(t, filepath.Join(directory, "empty.bin"), nil) + assertNoUploadTemps(t, directory) +} + +func TestStageAndPublishFileNeedsExplicitOverwrite(t *testing.T) { + directory := t.TempDir() + target := filepath.Join(directory, "data.bin") + if err := os.WriteFile(target, []byte("original"), 0o600); err != nil { + t.Fatal(err) + } + + _, err := stageInDirectory(t, directory, "data.bin", bytes.NewBufferString("replacement"), false) + if !errors.Is(err, errUploadTargetExists) { + t.Fatalf("error = %v, want errUploadTargetExists", err) + } + assertStoredFile(t, target, []byte("original")) + assertNoUploadTemps(t, directory) + + outcome, err := stageInDirectory(t, directory, "data.bin", bytes.NewBufferString("replacement"), true) + if err != nil { + t.Fatalf("overwrite: %v", err) + } + if !outcome.Overwritten { + t.Fatalf("outcome = %#v, want overwritten", outcome) + } + assertStoredFile(t, target, []byte("replacement")) + assertNoUploadTemps(t, directory) +} + +func TestStageAndPublishFileOverwriteCreatesWhenAbsent(t *testing.T) { + directory := t.TempDir() + outcome, err := stageInDirectory(t, directory, "new.bin", bytes.NewBufferString("new"), true) + if err != nil { + t.Fatal(err) + } + if outcome.Overwritten { + t.Fatalf("outcome = %#v, want new file", outcome) + } + assertStoredFile(t, filepath.Join(directory, "new.bin"), []byte("new")) +} + +func TestStageAndPublishFileKeepsOldTargetDuringTransfer(t *testing.T) { + directory := t.TempDir() + target := filepath.Join(directory, "data.bin") + if err := os.WriteFile(target, []byte("old-complete"), 0o600); err != nil { + t.Fatal(err) + } + root, err := os.OpenRoot(directory) + if err != nil { + t.Fatal(err) + } + defer root.Close() + + source, writer := io.Pipe() + type result struct { + outcome uploadOutcome + err error + } + done := make(chan result, 1) + go func() { + outcome, err := stageAndPublishFile(source, root, "data.bin", true, 0o640) + done <- result{outcome: outcome, err: err} + }() + if _, err := writer.Write([]byte("new-part-1")); err != nil { + t.Fatal(err) + } + assertStoredFile(t, target, []byte("old-complete")) + if _, err := writer.Write([]byte("-part-2")); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + got := <-done + if got.err != nil { + t.Fatal(got.err) + } + if !got.outcome.Overwritten { + t.Fatalf("outcome = %#v", got.outcome) + } + assertStoredFile(t, target, []byte("new-part-1-part-2")) + assertNoUploadTemps(t, directory) +} + +func TestStageAndPublishFileCleansSourceFailureWithoutChangingTarget(t *testing.T) { + directory := t.TempDir() + target := filepath.Join(directory, "data.bin") + if err := os.WriteFile(target, []byte("old"), 0o600); err != nil { + t.Fatal(err) + } + sentinel := errors.New("source failed") + source := io.MultiReader(bytes.NewBufferString("partial"), uploadErrorReader{err: sentinel}) + + _, err := stageInDirectory(t, directory, "data.bin", source, true) + var sourceErr *uploadSourceError + if !errors.As(err, &sourceErr) || !errors.Is(sourceErr, sentinel) { + t.Fatalf("error = %T %v, want uploadSourceError", err, err) + } + assertStoredFile(t, target, []byte("old")) + assertNoUploadTemps(t, directory) +} + +func TestStageAndPublishFilePublishesOpenedStageAfterNameReplacement(t *testing.T) { + directory := t.TempDir() + var renamedStage string + var replacementStage string + source := &callbackEOFReader{ + data: []byte("safe"), + onEOF: func() { + matches, err := filepath.Glob(filepath.Join(directory, ".crater-upload-*")) + if err != nil || len(matches) != 1 { + t.Fatalf("staging entries = %#v, err=%v", matches, err) + } + replacementStage = matches[0] + renamedStage = replacementStage + "-renamed" + if err := os.Rename(replacementStage, renamedStage); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(replacementStage, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(replacementStage, uploadStagePayload), []byte("attacker"), 0o600); err != nil { + t.Fatal(err) + } + }, + } + + if _, err := stageInDirectory(t, directory, "result.bin", source, false); err != nil { + t.Fatal(err) + } + assertStoredFile(t, filepath.Join(directory, "result.bin"), []byte("safe")) + if renamedStage == "" || replacementStage == "" { + t.Fatal("replacement callback did not run") + } +} + +func TestStageAndPublishFileRejectsNonRegularTargets(t *testing.T) { + directory := t.TempDir() + targetDirectory := filepath.Join(directory, "target") + if err := os.Mkdir(targetDirectory, 0o700); err != nil { + t.Fatal(err) + } + if _, err := stageInDirectory(t, directory, "target", bytes.NewBufferString("x"), true); !errors.Is(err, errUploadTargetNotRegular) { + t.Fatalf("directory error = %v", err) + } + + if err := os.Symlink(targetDirectory, filepath.Join(directory, "link")); err != nil { + t.Fatal(err) + } + if _, err := stageInDirectory(t, directory, "link", bytes.NewBufferString("x"), true); !errors.Is(err, errUploadTargetNotRegular) { + t.Fatalf("symlink error = %v", err) + } +} + +func TestStageAndPublishFileRaceHasOneWinner(t *testing.T) { + directory := t.TempDir() + root, err := os.OpenRoot(directory) + if err != nil { + t.Fatal(err) + } + defer root.Close() + + const contenders = 8 + var wait sync.WaitGroup + wait.Add(contenders) + start := make(chan struct{}) + errorsSeen := make(chan error, contenders) + + for index := 0; index < contenders; index++ { + go func() { + defer wait.Done() + <-start + _, err := stageAndPublishFile( + bytes.NewBufferString(string(rune('a'+index))), + root, + "race.bin", + false, + 0o640, + ) + errorsSeen <- err + }() + } + close(start) + wait.Wait() + close(errorsSeen) + + successes := 0 + exists := 0 + for err := range errorsSeen { + switch { + case err == nil: + successes++ + case errors.Is(err, errUploadTargetExists): + exists++ + default: + t.Fatalf("unexpected error: %v", err) + } + } + if successes != 1 || exists != contenders-1 { + t.Fatalf("successes=%d exists=%d", successes, exists) + } + assertNoUploadTemps(t, directory) +} + +func TestOpenUploadTargetRejectsEscapingParentSymlink(t *testing.T) { + storage := t.TempDir() + authorized := filepath.Join(storage, "users", "alice") + outside := filepath.Join(storage, "users", "bob") + if err := os.MkdirAll(authorized, 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(outside, 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(authorized, "escape")); err != nil { + t.Fatal(err) + } + + _, _, err := openUploadTarget(storage, "users/alice", "users/alice/escape/secret.bin") + if !errors.Is(err, errUploadParentInvalid) { + t.Fatalf("error = %v, want errUploadParentInvalid", err) + } +} + +func TestOpenUploadTargetRejectsAuthorizedRootEscapingStorage(t *testing.T) { + storage := t.TempDir() + outside := t.TempDir() + if err := os.MkdirAll(filepath.Join(storage, "users"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(storage, "users", "alice")); err != nil { + t.Fatal(err) + } + + _, _, err := openUploadTarget(storage, "users/alice", "users/alice/secret.bin") + if !errors.Is(err, errUploadParentInvalid) { + t.Fatalf("error = %v, want errUploadParentInvalid", err) + } +} + +func TestOpenUploadTargetRejectsRawInternalTraversal(t *testing.T) { + for _, candidate := range []string{ + "users/../users/alice", + "users/alice/../alice/result.bin", + `users\..\users\alice`, + "users/.", + "users/alice/", + "//users/alice", + "users///alice", + "users/alice//runs//result.bin", + "users/alice\n", + } { + if _, err := cleanStorageRelativePath(candidate); !errors.Is(err, errUploadParentInvalid) { + t.Fatalf("cleanStorageRelativePath(%q) error = %v, want errUploadParentInvalid", candidate, err) + } + } +} + +func TestCleanStorageRelativePathSupportsOneLegacyAbsoluteSpaceMarker(t *testing.T) { + for input, want := range map[string]string{ + "/public/models": filepath.Join("public", "models"), + "users//space/zhouyh25": filepath.Join("users", "space", "zhouyh25"), + "users//space/zhouyh25/file": filepath.Join("users", "space", "zhouyh25", "file"), + } { + got, err := cleanStorageRelativePath(input) + if err != nil { + t.Fatalf("cleanStorageRelativePath(%q): %v", input, err) + } + if got != want { + t.Fatalf("cleanStorageRelativePath(%q) = %q, want %q", input, got, want) + } + } +} + +func TestOpenUploadTargetSupportsLegacyLeadingSlashSpace(t *testing.T) { + storage := t.TempDir() + if err := os.MkdirAll(filepath.Join(storage, "users", "space", "alice", "jobs"), 0o755); err != nil { + t.Fatal(err) + } + parent, targetName, err := openUploadTarget( + storage, + "users//space/alice", + "users//space/alice/jobs/result.bin", + ) + if err != nil { + t.Fatal(err) + } + defer parent.Close() + if targetName != "result.bin" { + t.Fatalf("targetName = %q", targetName) + } +} + +func TestOpenUploadTargetDoesNotCreateMissingParent(t *testing.T) { + storage := t.TempDir() + if err := os.MkdirAll(filepath.Join(storage, "users", "alice"), 0o755); err != nil { + t.Fatal(err) + } + + _, _, err := openUploadTarget(storage, "users/alice", "users/alice/missing/secret.bin") + if !errors.Is(err, errUploadParentInvalid) { + t.Fatalf("error = %v, want errUploadParentInvalid", err) + } + if _, err := os.Stat(filepath.Join(storage, "users", "alice", "missing")); !os.IsNotExist(err) { + t.Fatalf("missing parent was created: %v", err) + } +} + +func TestOpenedUploadTargetRemainsAnchoredAcrossParentRename(t *testing.T) { + storage := t.TempDir() + jobs := filepath.Join(storage, "users", "alice", "jobs") + original := filepath.Join(storage, "users", "alice", "jobs-original") + outside := t.TempDir() + if err := os.MkdirAll(jobs, 0o755); err != nil { + t.Fatal(err) + } + + parent, targetName, err := openUploadTarget(storage, "users/alice", "users/alice/jobs/result.bin") + if err != nil { + t.Fatal(err) + } + defer parent.Close() + if err := os.Rename(jobs, original); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, jobs); err != nil { + t.Fatal(err) + } + + if _, err := stageAndPublishFile(bytes.NewBufferString("safe"), parent, targetName, false, 0o640); err != nil { + t.Fatal(err) + } + assertStoredFile(t, filepath.Join(original, "result.bin"), []byte("safe")) + if _, err := os.Stat(filepath.Join(outside, "result.bin")); !os.IsNotExist(err) { + t.Fatalf("upload escaped through replacement symlink: %v", err) + } +} + +func TestUploadFileHandlerHTTPContract(t *testing.T) { + gin.SetMode(gin.TestMode) + t.Run("created", func(t *testing.T) { + storage := newUploadHandlerStorage(t) + recorder := serveUpload(t, testUploadHandlerDeps(storage), "user/result.bin", "overwrite=false", bytes.NewBufferString("data")) + assertUploadEnvelope(t, recorder, http.StatusCreated, 0) + assertStoredFile(t, filepath.Join(storage, "users", "alice", "result.bin"), []byte("data")) + info, err := os.Stat(filepath.Join(storage, "users", "alice", "result.bin")) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != uploadFileMode { + t.Fatalf("uploaded mode = %o, want %o", info.Mode().Perm(), uploadFileMode) + } + }) + + t.Run("overwritten", func(t *testing.T) { + storage := newUploadHandlerStorage(t) + target := filepath.Join(storage, "users", "alice", "result.bin") + if err := os.WriteFile(target, []byte("old"), 0o600); err != nil { + t.Fatal(err) + } + recorder := serveUpload(t, testUploadHandlerDeps(storage), "user/result.bin", "overwrite=true", bytes.NewBufferString("new")) + assertUploadEnvelope(t, recorder, http.StatusOK, 0) + assertStoredFile(t, target, []byte("new")) + }) + + tests := []struct { + name string + path string + query string + wantStatus int + wantCode int + mutate func(*testing.T, string, *uploadHandlerDeps) + }{ + {name: "invalid overwrite", path: "user/result.bin", query: "overwrite=yes", wantStatus: http.StatusBadRequest, wantCode: 40004}, + {name: "invalid path", path: "user", query: "overwrite=false", wantStatus: http.StatusBadRequest, wantCode: 40004}, + { + name: "unauthorized", path: "user/result.bin", query: "overwrite=false", + wantStatus: http.StatusUnauthorized, wantCode: 40102, + mutate: func(_ *testing.T, _ string, deps *uploadHandlerDeps) { + deps.authenticate = func(*gin.Context) (util.JWTMessage, error) { + return util.JWTMessage{}, errors.New("invalid token") + } + }, + }, + { + name: "forbidden", path: "user/result.bin", query: "overwrite=false", + wantStatus: http.StatusForbidden, wantCode: 40301, + mutate: func(_ *testing.T, _ string, deps *uploadHandlerDeps) { + deps.permission = func(string, util.JWTMessage, *gin.Context) model.FilePermission { + return model.ReadOnly + } + }, + }, + { + name: "existing target", path: "user/result.bin", query: "overwrite=false", + wantStatus: http.StatusConflict, wantCode: 40901, + mutate: func(t *testing.T, storage string, _ *uploadHandlerDeps) { + if err := os.WriteFile(filepath.Join(storage, "users", "alice", "result.bin"), []byte("old"), 0o600); err != nil { + t.Fatal(err) + } + }, + }, + { + name: "nonregular target", path: "user/result.bin", query: "overwrite=true", + wantStatus: http.StatusConflict, wantCode: 40902, + mutate: func(t *testing.T, storage string, _ *uploadHandlerDeps) { + if err := os.Mkdir(filepath.Join(storage, "users", "alice", "result.bin"), 0o700); err != nil { + t.Fatal(err) + } + }, + }, + { + name: "missing parent", path: "user/missing/result.bin", query: "overwrite=false", + wantStatus: http.StatusConflict, wantCode: 40902, + }, + { + name: "source read failure", path: "user/result.bin", query: "overwrite=false", + wantStatus: http.StatusBadRequest, wantCode: 40001, + mutate: func(_ *testing.T, _ string, deps *uploadHandlerDeps) { + deps.stagePublish = func(io.Reader, *os.Root, string, bool, os.FileMode) (uploadOutcome, error) { + return uploadOutcome{}, &uploadSourceError{cause: errors.New("broken body")} + } + }, + }, + { + name: "filesystem failure", path: "user/result.bin", query: "overwrite=false", + wantStatus: http.StatusInternalServerError, wantCode: 50005, + mutate: func(_ *testing.T, _ string, deps *uploadHandlerDeps) { + deps.openTarget = func(string, string, string) (*os.Root, string, error) { + return nil, "", errors.New("disk unavailable") + } + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + storage := newUploadHandlerStorage(t) + deps := testUploadHandlerDeps(storage) + if test.mutate != nil { + test.mutate(t, storage, &deps) + } + recorder := serveUpload(t, deps, test.path, test.query, bytes.NewBufferString("data")) + assertUploadEnvelope(t, recorder, test.wantStatus, test.wantCode) + }) + } +} + +func TestNormalizeUploadLogicalPath(t *testing.T) { + if got, err := normalizeUploadLogicalPath("/user//runs/./a.bin"); err != nil || got != "user/runs/a.bin" { + t.Fatalf("normalize = %q, %v", got, err) + } + for _, invalid := range []string{"user", "admin/file", "user/../public/file", `user\file`, "user/a\nb"} { + if _, err := normalizeUploadLogicalPath(invalid); err == nil { + t.Errorf("normalizeUploadLogicalPath(%q) accepted invalid path", invalid) + } + } +} + +func TestParseUploadOverwrite(t *testing.T) { + for _, valid := range []string{"", "false", "true"} { + if _, err := parseUploadOverwrite(valid); err != nil { + t.Errorf("parseUploadOverwrite(%q): %v", valid, err) + } + } + for _, invalid := range []string{"1", "TRUE", "yes"} { + if _, err := parseUploadOverwrite(invalid); err == nil { + t.Errorf("parseUploadOverwrite(%q) accepted invalid value", invalid) + } + } +} + +func TestRegisterRoutesServesUploadEndpoint(t *testing.T) { + gin.SetMode(gin.TestMode) + router := gin.New() + RegisterRoutes(router) + + request := httptest.NewRequest( + http.MethodPost, + "/api/ss/upload/user/test.bin?overwrite=false", + bytes.NewBufferString("data"), + ) + recorder := httptest.NewRecorder() + router.ServeHTTP(recorder, request) + assertUploadEnvelope(t, recorder, http.StatusUnauthorized, 40102) +} + +type uploadErrorReader struct { + err error +} + +func (reader uploadErrorReader) Read([]byte) (int, error) { + return 0, reader.err +} + +type callbackEOFReader struct { + data []byte + onEOF func() + read bool +} + +func (reader *callbackEOFReader) Read(buffer []byte) (int, error) { + if !reader.read { + reader.read = true + return copy(buffer, reader.data), nil + } + reader.onEOF() + return 0, io.EOF +} + +func stageInDirectory( + t *testing.T, + directory string, + targetName string, + source io.Reader, + overwrite bool, +) (uploadOutcome, error) { + t.Helper() + root, err := os.OpenRoot(directory) + if err != nil { + t.Fatal(err) + } + defer root.Close() + return stageAndPublishFile(source, root, targetName, overwrite, testUploadMode) +} + +func newUploadHandlerStorage(t *testing.T) string { + t.Helper() + storage := t.TempDir() + if err := os.MkdirAll(filepath.Join(storage, "users", "alice"), 0o755); err != nil { + t.Fatal(err) + } + return storage +} + +func testUploadHandlerDeps(storage string) uploadHandlerDeps { + return uploadHandlerDeps{ + authenticate: func(*gin.Context) (util.JWTMessage, error) { + return util.JWTMessage{}, nil + }, + permission: func(string, util.JWTMessage, *gin.Context) model.FilePermission { + return model.ReadWrite + }, + redirect: func(_ *gin.Context, logicalPath string, _ util.JWTMessage) (string, error) { + if logicalPath == "user" { + return "users/alice", nil + } + return "users/alice/" + strings.TrimPrefix(logicalPath, "user/"), nil + }, + openTarget: openUploadTarget, + stagePublish: stageAndPublishFile, + storageRoot: storage, + } +} + +func serveUpload( + t *testing.T, + deps uploadHandlerDeps, + logicalPath string, + query string, + body io.Reader, +) *httptest.ResponseRecorder { + t.Helper() + router := gin.New() + router.POST("/upload/*path", func(c *gin.Context) { + uploadFileWithDeps(c, deps) + }) + target := "/upload/" + logicalPath + if query != "" { + target += "?" + query + } + request := httptest.NewRequest(http.MethodPost, target, body) + recorder := httptest.NewRecorder() + router.ServeHTTP(recorder, request) + return recorder +} + +func assertUploadEnvelope(t *testing.T, recorder *httptest.ResponseRecorder, wantStatus, wantCode int) { + t.Helper() + if recorder.Code != wantStatus { + t.Fatalf("status = %d, want %d; body=%s", recorder.Code, wantStatus, recorder.Body.String()) + } + var envelope struct { + Code int `json:"code"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &envelope); err != nil { + t.Fatalf("decode response: %v; body=%s", err, recorder.Body.String()) + } + if envelope.Code != wantCode { + t.Fatalf("code = %d, want %d; body=%s", envelope.Code, wantCode, recorder.Body.String()) + } +} + +func assertStoredFile(t *testing.T, path string, want []byte) { + t.Helper() + got, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, want) { + t.Fatalf("%s = %v, want %v", path, got, want) + } +} + +func assertNoUploadTemps(t *testing.T, directory string) { + t.Helper() + matches, err := filepath.Glob(filepath.Join(directory, ".crater-upload-*")) + if err != nil { + t.Fatal(err) + } + if len(matches) != 0 { + t.Fatalf("temporary upload entries remain: %#v", matches) + } +} diff --git a/cli/cmd/file.go b/cli/cmd/file.go new file mode 100644 index 000000000..5ee22b9f6 --- /dev/null +++ b/cli/cmd/file.go @@ -0,0 +1,336 @@ +package cmd + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "unicode" + + "github.com/raids-lab/crater/cli/internal/api" + "github.com/raids-lab/crater/cli/internal/clierror" + "github.com/raids-lab/crater/cli/internal/completion" + "github.com/raids-lab/crater/cli/internal/i18n" + "github.com/raids-lab/crater/cli/internal/output" + "github.com/raids-lab/crater/cli/pkg/errorcodes" + "github.com/spf13/cobra" +) + +var fileRemoteRoots = []string{"user", "public", "account"} + +var fileCmd = &cobra.Command{ + Use: "file", + Short: "Upload remote files", + Long: "Upload files to user, public, and account storage spaces.", + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) > 0 { + return errUnknownSubcommand(cmd, args[0]) + } + return cmd.Help() + }, +} + +var fileUploadCmd = &cobra.Command{ + Use: "upload ", + Short: "Upload one local file", + Args: fileUploadArgs, + RunE: runFileUpload, +} + +type fileUploadDeps struct { + client func() (api.FileUploadClient, error) + stdout io.Writer + json bool +} + +type fileUploadInput struct { + localPath string + remotePath string + overwrite bool +} + +type fileUploadResult struct { + LocalPath string + RemotePath string + Bytes int64 + Overwrite bool + Overwritten bool +} + +func fileUploadArgs(cmd *cobra.Command, args []string) error { + if len(args) > 2 { + return errTooManyArgs(cmd, len(args), 2) + } + if len(args) < 2 { + field := "local-file" + label := i18n.T("file_label_local_file") + if len(args) == 1 { + field = "remote-path" + label = i18n.T("file_label_remote_file") + } + return errUsageFromIssues([]usageIssue{{ + Code: errorcodes.ErrMissingRequiredFlag, + Message: i18n.T("err_missing_required_arg", label, field), + Field: field, + }}) + } + return nil +} + +func runFileUpload(cmd *cobra.Command, args []string) error { + return runFileUploadWith(cmd, args, fileUploadDeps{ + client: activeFileUploadClient, + stdout: os.Stdout, + json: outputJSON, + }) +} + +func activeFileUploadClient() (api.FileUploadClient, error) { + return activeAPIClient() +} + +func runFileUploadWith(cmd *cobra.Command, args []string, deps fileUploadDeps) error { + remotePath, err := normalizeRemotePath(args[1], false) + if err != nil { + return err + } + if len(strings.Split(remotePath, "/")) < 2 { + return invalidRemotePathIssue(i18n.T("err_file_path_not_file", args[1])) + } + + source, err := openUploadSource(args[0]) + if err != nil { + return err + } + defer source.Close() + + overwrite, _ := cmd.Flags().GetBool("overwrite") + result, err := uploadRemoteFile(cmd.Context(), deps.client, source, fileUploadInput{ + localPath: args[0], + remotePath: remotePath, + overwrite: overwrite, + }) + if err != nil { + return err + } + return writeFileUploadResult(deps.stdout, deps.json, result) +} + +func openUploadSource(localPath string) (*os.File, error) { + pathInfo, err := os.Stat(localPath) + if err != nil { + return nil, localFileError("err_file_local_stat", localPath, err) + } + if !pathInfo.Mode().IsRegular() { + return nil, invalidLocalPathIssue(i18n.T("err_file_local_not_regular", localPath)) + } + + source, err := openUploadFileNoBlock(localPath) + if err != nil { + return nil, localFileError("err_file_local_open", localPath, err) + } + info, err := source.Stat() + if err != nil { + _ = source.Close() + return nil, localFileError("err_file_local_stat", localPath, err) + } + if !info.Mode().IsRegular() { + _ = source.Close() + return nil, invalidLocalPathIssue(i18n.T("err_file_local_not_regular", localPath)) + } + return source, nil +} + +func uploadRemoteFile( + ctx context.Context, + clientFactory func() (api.FileUploadClient, error), + source io.Reader, + input fileUploadInput, +) (fileUploadResult, error) { + client, err := clientFactory() + if err != nil { + return fileUploadResult{}, err + } + + uploaded, err := client.UploadFile(ctx, input.remotePath, source, input.overwrite) + if err != nil { + var sourceErr *api.SourceReadError + if errors.As(err, &sourceErr) { + return fileUploadResult{}, localFileError("err_file_local_read", input.localPath, sourceErr.Cause) + } + return fileUploadResult{}, cliErrFromAPI(err) + } + return fileUploadResult{ + LocalPath: input.localPath, + RemotePath: uploaded.RemotePath, + Bytes: uploaded.Bytes, + Overwrite: input.overwrite, + Overwritten: uploaded.Overwritten, + }, nil +} + +func writeFileUploadResult(writer io.Writer, jsonOutput bool, result fileUploadResult) error { + if jsonOutput { + return output.WriteSuccessJSON(writer, output.SuccessEnvelope(map[string]interface{}{ + "local_path": result.LocalPath, + "remote_path": result.RemotePath, + "bytes": result.Bytes, + "overwrite": result.Overwrite, + "overwritten": result.Overwritten, + })) + } + _, err := fmt.Fprintln(writer, i18n.T("file_upload_success", result.LocalPath, result.RemotePath, result.Bytes)) + if err != nil { + return &clierror.Error{ + Category: errorcodes.CategorySystem, + Code: errorcodes.ErrCommandExecution, + Message: i18n.T("err_file_output", err.Error()), + Context: map[string]interface{}{"msg": err.Error()}, + } + } + return nil +} + +func normalizeRemotePath(remotePath string, allowEmpty bool) (string, error) { + if remotePath == "" { + if allowEmpty { + return "", nil + } + return "", invalidRemotePathIssue(i18n.T("err_file_path_invalid", remotePath)) + } + if strings.ContainsRune(remotePath, '\\') { + return "", invalidRemotePathIssue(i18n.T("err_file_path_invalid", remotePath)) + } + for _, character := range remotePath { + if unicode.IsControl(character) { + return "", invalidRemotePathIssue(i18n.T("err_file_path_invalid", remotePath)) + } + } + + trimmed := strings.Trim(remotePath, "/") + rawSegments := strings.Split(trimmed, "/") + segments := make([]string, 0, len(rawSegments)) + for _, segment := range rawSegments { + if segment == ".." { + return "", invalidRemotePathIssue(i18n.T("err_file_path_invalid", remotePath)) + } + if segment == "" || segment == "." { + continue + } + segments = append(segments, segment) + } + if len(segments) == 0 { + if allowEmpty { + return "", nil + } + return "", invalidRemotePathIssue(i18n.T("err_file_path_invalid", remotePath)) + } + if !isFileRemoteRoot(segments[0]) { + return "", invalidRemotePathIssue(i18n.T("err_file_path_root", remotePath)) + } + return strings.Join(segments, "/"), nil +} + +func invalidRemotePathIssue(message string) error { + return errUsageFromIssues([]usageIssue{{ + Code: errorcodes.ErrInvalidFlagValue, + Message: message, + Field: "remote-path", + }}) +} + +func invalidLocalPathIssue(message string) error { + return errUsageFromIssues([]usageIssue{{ + Code: errorcodes.ErrInvalidFlagValue, + Message: message, + Field: "local-file", + }}) +} + +func localFileError(key, localPath string, cause error) *clierror.Error { + return &clierror.Error{ + Category: errorcodes.CategorySystem, + Code: errorcodes.ErrCommandExecution, + Message: i18n.T(key, localPath, cause.Error()), + Context: map[string]interface{}{ + "path": localPath, + "msg": cause.Error(), + }, + } +} + +func isFileRemoteRoot(value string) bool { + for _, root := range fileRemoteRoots { + if value == root { + return true + } + } + return false +} + +func fileRemoteRootCompleter(ctx completion.Context) ([]completion.Candidate, error) { + prefix := strings.ToLower(completion.CurrentWordPrefix(ctx)) + candidates := make([]completion.Candidate, 0, len(fileRemoteRoots)) + for _, root := range fileRemoteRoots { + if prefix != "" && !strings.HasPrefix(root, prefix) { + continue + } + candidates = append(candidates, completion.Candidate{ + Value: root, + Description: i18n.T("file_root_" + root + "_desc"), + }) + } + return candidates, nil +} + +func fileLocalPathCompleter(ctx completion.Context) ([]completion.Candidate, error) { + prefix := completion.CurrentWordPrefix(ctx) + directoryPrefix, namePrefix := filepath.Split(prefix) + directory := directoryPrefix + if directory == "" { + directory = "." + } + entries, err := os.ReadDir(directory) + if err != nil { + return nil, nil + } + + candidates := make([]completion.Candidate, 0, len(entries)) + for _, entry := range entries { + name := entry.Name() + if !strings.HasPrefix(name, namePrefix) || + (namePrefix == "" && strings.HasPrefix(name, ".")) { + continue + } + fullPath := filepath.Join(directory, name) + info, err := os.Stat(fullPath) + if err != nil { + continue + } + value := directoryPrefix + name + description := i18n.T("file_local_regular_desc") + switch { + case info.IsDir(): + value += string(filepath.Separator) + description = i18n.T("file_local_directory_desc") + case !info.Mode().IsRegular(): + continue + } + candidates = append(candidates, completion.Candidate{ + Value: value, + Description: description, + }) + } + return candidates, nil +} + +func init() { + fileUploadCmd.Flags().Bool("overwrite", false, "Replace an existing remote file") + fileCmd.AddCommand(fileUploadCmd) + rootCmd.AddCommand(fileCmd) + completion.RegisterPositional([]string{"file", "upload"}, 0, fileLocalPathCompleter) + completion.RegisterPositional([]string{"file", "upload"}, 1, fileRemoteRootCompleter) +} diff --git a/cli/cmd/file_open_other.go b/cli/cmd/file_open_other.go new file mode 100644 index 000000000..818be74c8 --- /dev/null +++ b/cli/cmd/file_open_other.go @@ -0,0 +1,9 @@ +//go:build !aix && !darwin && !dragonfly && !freebsd && !linux && !netbsd && !openbsd && !solaris + +package cmd + +import "os" + +func openUploadFileNoBlock(path string) (*os.File, error) { + return os.Open(path) +} diff --git a/cli/cmd/file_open_unix.go b/cli/cmd/file_open_unix.go new file mode 100644 index 000000000..90c74e547 --- /dev/null +++ b/cli/cmd/file_open_unix.go @@ -0,0 +1,12 @@ +//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris + +package cmd + +import ( + "os" + "syscall" +) + +func openUploadFileNoBlock(path string) (*os.File, error) { + return os.OpenFile(path, os.O_RDONLY|syscall.O_NONBLOCK, 0) +} diff --git a/cli/cmd/file_open_unix_test.go b/cli/cmd/file_open_unix_test.go new file mode 100644 index 000000000..fbc2dbb3f --- /dev/null +++ b/cli/cmd/file_open_unix_test.go @@ -0,0 +1,34 @@ +//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris + +package cmd + +import ( + "syscall" + "testing" + "time" +) + +func TestOpenUploadSourceRejectsFIFOWithoutBlocking(t *testing.T) { + path := t.TempDir() + "/source.pipe" + if err := syscall.Mkfifo(path, 0o600); err != nil { + t.Fatal(err) + } + + done := make(chan error, 1) + go func() { + source, err := openUploadSource(path) + if source != nil { + _ = source.Close() + } + done <- err + }() + + select { + case err := <-done: + if err == nil { + t.Fatal("FIFO source should fail") + } + case <-time.After(time.Second): + t.Fatal("opening a FIFO blocked instead of rejecting it") + } +} diff --git a/cli/cmd/file_test.go b/cli/cmd/file_test.go new file mode 100644 index 000000000..7746dd1df --- /dev/null +++ b/cli/cmd/file_test.go @@ -0,0 +1,407 @@ +package cmd + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/raids-lab/crater/cli/internal/api" + "github.com/raids-lab/crater/cli/internal/clierror" + "github.com/raids-lab/crater/cli/internal/completion" + "github.com/raids-lab/crater/cli/pkg/errorcodes" + "github.com/spf13/cobra" +) + +type fakeFileUploadClient struct { + upload func(context.Context, string, io.Reader, bool) (api.FileUploadResult, error) +} + +func (client fakeFileUploadClient) UploadFile( + ctx context.Context, + remotePath string, + source io.Reader, + overwrite bool, +) (api.FileUploadResult, error) { + return client.upload(ctx, remotePath, source, overwrite) +} + +func testFileUploadCommand(t *testing.T, overwrite bool) *cobra.Command { + t.Helper() + command := &cobra.Command{Use: "upload"} + command.Flags().Bool("overwrite", false, "") + if overwrite { + if err := command.Flags().Set("overwrite", "true"); err != nil { + t.Fatal(err) + } + } + return command +} + +func TestFileUploadArgs(t *testing.T) { + command := &cobra.Command{Use: "upload"} + if err := fileUploadArgs(command, nil); err == nil { + t.Fatal("missing local file should fail") + } + if err := fileUploadArgs(command, []string{"local.bin"}); err == nil { + t.Fatal("missing remote file should fail") + } + if err := fileUploadArgs(command, []string{"local.bin", "user/local.bin"}); err != nil { + t.Fatalf("valid args: %v", err) + } + if err := fileUploadArgs(command, []string{"a", "b", "c"}); err == nil { + t.Fatal("extra args should fail") + } +} + +func TestNormalizeRemoteUploadPath(t *testing.T) { + tests := []struct { + name string + input string + want string + wantErr bool + }{ + {name: "nested file", input: "user/results/model.bin", want: "user/results/model.bin"}, + {name: "unicode and spaces", input: "/account/实验 data/结果.bin/", want: "account/实验 data/结果.bin"}, + {name: "safe normalization", input: "public//runs/./out.bin", want: "public/runs/out.bin"}, + {name: "empty", wantErr: true}, + {name: "unknown root", input: "admin/secret.bin", wantErr: true}, + {name: "parent traversal", input: "user/../public/secret.bin", wantErr: true}, + {name: "backslash", input: `user\secret.bin`, wantErr: true}, + {name: "control character", input: "user/file\nname", wantErr: true}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := normalizeRemotePath(test.input, false) + if (err != nil) != test.wantErr { + t.Fatalf("normalizeRemotePath(%q) error = %v, wantErr %v", test.input, err, test.wantErr) + } + if got != test.want { + t.Fatalf("normalizeRemotePath(%q) = %q, want %q", test.input, got, test.want) + } + }) + } +} + +func TestRunFileUploadRejectsInvalidRemoteBeforeClient(t *testing.T) { + factoryCalls := 0 + err := runFileUploadWith( + testFileUploadCommand(t, false), + []string{"does-not-need-to-exist", "user/../public/secret.bin"}, + fileUploadDeps{ + client: func() (api.FileUploadClient, error) { + factoryCalls++ + return nil, errors.New("must not be called") + }, + stdout: io.Discard, + }, + ) + if err == nil { + t.Fatal("invalid remote path should fail") + } + if factoryCalls != 0 { + t.Fatalf("factory calls = %d, want 0", factoryCalls) + } +} + +func TestRunFileUploadRejectsLogicalRootBeforeClient(t *testing.T) { + local := writeUploadFixture(t, []byte("data")) + factoryCalls := 0 + err := runFileUploadWith( + testFileUploadCommand(t, false), + []string{local, "user"}, + fileUploadDeps{ + client: func() (api.FileUploadClient, error) { + factoryCalls++ + return nil, errors.New("must not be called") + }, + stdout: io.Discard, + }, + ) + if err == nil { + t.Fatal("logical root should not be an upload target") + } + if factoryCalls != 0 { + t.Fatalf("factory calls = %d, want 0", factoryCalls) + } +} + +func TestRunFileUploadRejectsNonRegularSourceBeforeClient(t *testing.T) { + factoryCalls := 0 + err := runFileUploadWith( + testFileUploadCommand(t, false), + []string{t.TempDir(), "user/result.bin"}, + fileUploadDeps{ + client: func() (api.FileUploadClient, error) { + factoryCalls++ + return nil, errors.New("must not be called") + }, + stdout: io.Discard, + }, + ) + if err == nil { + t.Fatal("directory source should fail") + } + if factoryCalls != 0 { + t.Fatalf("factory calls = %d, want 0", factoryCalls) + } +} + +func TestRunFileUploadExistingTargetNeedsOverwrite(t *testing.T) { + local := writeUploadFixture(t, []byte("new")) + uploadCalls := 0 + err := runFileUploadWith( + testFileUploadCommand(t, false), + []string{local, "user/results/result.bin"}, + fileUploadDeps{ + client: func() (api.FileUploadClient, error) { + return fakeFileUploadClient{ + upload: func(_ context.Context, remotePath string, _ io.Reader, overwrite bool) (api.FileUploadResult, error) { + uploadCalls++ + if remotePath != "user/results/result.bin" || overwrite { + t.Fatalf("remotePath=%q overwrite=%v", remotePath, overwrite) + } + return api.FileUploadResult{}, &api.RequestError{ + HTTPStatus: 409, + CraterCode: 40901, + Msg: "target file already exists", + } + }, + }, nil + }, + stdout: io.Discard, + }, + ) + if err == nil { + t.Fatal("existing target should fail without --overwrite") + } + if uploadCalls != 1 { + t.Fatalf("upload calls = %d, want 1", uploadCalls) + } + var cliErr *clierror.Error + if !errors.As(err, &cliErr) || cliErr.Category != errorcodes.CategoryAPI { + t.Fatalf("error = %T %v, want API error", err, err) + } +} + +func TestRunFileUploadRejectsRemoteDirectoryEvenWithOverwrite(t *testing.T) { + local := writeUploadFixture(t, []byte("new")) + uploadCalls := 0 + err := runFileUploadWith( + testFileUploadCommand(t, true), + []string{local, "public/results"}, + fileUploadDeps{ + client: func() (api.FileUploadClient, error) { + return fakeFileUploadClient{ + upload: func(context.Context, string, io.Reader, bool) (api.FileUploadResult, error) { + uploadCalls++ + return api.FileUploadResult{}, &api.RequestError{ + HTTPStatus: 409, + CraterCode: 40902, + Msg: "target path is not a regular file", + } + }, + }, nil + }, + stdout: io.Discard, + }, + ) + if err == nil { + t.Fatal("remote directory should fail") + } + if uploadCalls != 1 { + t.Fatalf("upload calls = %d, want 1", uploadCalls) + } +} + +func TestRunFileUploadStreamsBinaryAndWritesJSONMetadata(t *testing.T) { + payload := []byte{0x00, 0xff, 'C', 'L', 'I'} + local := writeUploadFixture(t, payload) + var stdout bytes.Buffer + err := runFileUploadWith( + testFileUploadCommand(t, false), + []string{local, "/account//实验 data/./result.bin"}, + fileUploadDeps{ + client: func() (api.FileUploadClient, error) { + return fakeFileUploadClient{ + upload: func(_ context.Context, remotePath string, source io.Reader, overwrite bool) (api.FileUploadResult, error) { + if remotePath != "account/实验 data/result.bin" { + t.Fatalf("remote path = %q", remotePath) + } + if overwrite { + t.Fatal("overwrite = true, want false") + } + got, readErr := io.ReadAll(source) + if readErr != nil { + return api.FileUploadResult{}, readErr + } + if !bytes.Equal(got, payload) { + t.Fatalf("payload = %v, want %v", got, payload) + } + return api.FileUploadResult{ + RemotePath: remotePath, + Bytes: int64(len(got)), + }, nil + }, + }, nil + }, + stdout: &stdout, + json: true, + }, + ) + if err != nil { + t.Fatalf("runFileUploadWith: %v", err) + } + if bytes.Contains(stdout.Bytes(), payload) { + t.Fatalf("stdout contains binary payload: %q", stdout.Bytes()) + } + var envelope struct { + Status string `json:"status"` + Data struct { + LocalPath string `json:"local_path"` + RemotePath string `json:"remote_path"` + Bytes int64 `json:"bytes"` + Overwrite bool `json:"overwrite"` + Overwritten bool `json:"overwritten"` + } `json:"data"` + } + if err := json.Unmarshal(stdout.Bytes(), &envelope); err != nil { + t.Fatalf("decode stdout: %v\n%s", err, stdout.String()) + } + if envelope.Status != "OK" || envelope.Data.LocalPath != local || + envelope.Data.RemotePath != "account/实验 data/result.bin" || + envelope.Data.Bytes != int64(len(payload)) || envelope.Data.Overwrite { + t.Fatalf("envelope = %#v", envelope) + } +} + +func TestRunFileUploadOverwritePassesExplicitAuthorization(t *testing.T) { + local := writeUploadFixture(t, []byte("replacement")) + overwriteSeen := false + err := runFileUploadWith( + testFileUploadCommand(t, true), + []string{local, "user/result.bin"}, + fileUploadDeps{ + client: func() (api.FileUploadClient, error) { + return fakeFileUploadClient{ + upload: func(_ context.Context, remotePath string, source io.Reader, overwrite bool) (api.FileUploadResult, error) { + overwriteSeen = overwrite + written, readErr := io.Copy(io.Discard, source) + return api.FileUploadResult{ + RemotePath: remotePath, + Bytes: written, + Overwritten: true, + }, readErr + }, + }, nil + }, + stdout: io.Discard, + }, + ) + if err != nil { + t.Fatalf("runFileUploadWith: %v", err) + } + if !overwriteSeen { + t.Fatal("overwrite flag was not passed to API client") + } +} + +func TestRunFileUploadSourceReadErrorIsSystemError(t *testing.T) { + sentinel := errors.New("local read failed") + _, err := uploadRemoteFile( + context.Background(), + func() (api.FileUploadClient, error) { + return fakeFileUploadClient{ + upload: func(context.Context, string, io.Reader, bool) (api.FileUploadResult, error) { + return api.FileUploadResult{}, &api.SourceReadError{Cause: sentinel} + }, + }, nil + }, + bytes.NewReader(nil), + fileUploadInput{ + localPath: "local.bin", + remotePath: "user/remote.bin", + }, + ) + var cliErr *clierror.Error + if !errors.As(err, &cliErr) || cliErr.Category != errorcodes.CategorySystem { + t.Fatalf("error = %T %v, want system cli error", err, err) + } +} + +func TestFileRemoteRootCompleter(t *testing.T) { + candidates, err := fileRemoteRootCompleter(completion.Context{ + Words: []string{"crater", "file", "upload", "local.bin", "a"}, + Current: 5, + }) + if err != nil { + t.Fatal(err) + } + got := make([]string, len(candidates)) + for index := range candidates { + got[index] = candidates[index].Value + } + if !reflect.DeepEqual(got, []string{"account"}) { + t.Fatalf("candidate values = %#v", got) + } +} + +func TestFileLocalPathCompleter(t *testing.T) { + directory := t.TempDir() + if err := os.WriteFile(filepath.Join(directory, "alpha.txt"), []byte("a"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(directory, ".hidden"), []byte("h"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(filepath.Join(directory, "artifacts"), 0o700); err != nil { + t.Fatal(err) + } + + prefix := filepath.Join(directory, "a") + candidates, err := fileLocalPathCompleter(completion.Context{ + Words: []string{"crater", "file", "upload", prefix}, + Current: 4, + }) + if err != nil { + t.Fatal(err) + } + got := make([]string, len(candidates)) + for index := range candidates { + got[index] = candidates[index].Value + } + want := []string{ + filepath.Join(directory, "alpha.txt"), + filepath.Join(directory, "artifacts") + string(filepath.Separator), + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("candidate values = %#v, want %#v", got, want) + } + + hiddenPrefix := filepath.Join(directory, ".h") + candidates, err = fileLocalPathCompleter(completion.Context{ + Words: []string{"crater", "file", "upload", hiddenPrefix}, + Current: 4, + }) + if err != nil { + t.Fatal(err) + } + if len(candidates) != 1 || candidates[0].Value != filepath.Join(directory, ".hidden") { + t.Fatalf("hidden candidates = %#v", candidates) + } +} + +func writeUploadFixture(t *testing.T, data []byte) string { + t.Helper() + path := filepath.Join(t.TempDir(), "fixture.bin") + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + return path +} diff --git a/cli/docs/COMMANDS.md b/cli/docs/COMMANDS.md index 4b75a41c3..49990136c 100644 --- a/cli/docs/COMMANDS.md +++ b/cli/docs/COMMANDS.md @@ -772,3 +772,35 @@ This section records the read-only API surface covered by the CLI after the broa - `crater admin cronjobs`: `/api/v1/admin/operations/cronjob`. - `crater admin whitelist`: `/api/v1/admin/operations/whitelist`. - These commands surface existing admin GET APIs only. They do not perform update/delete/reconcile actions. + +--- + +## 8. 远端文件模块 (file) + +本模块面向普通用户访问 storage service 暴露的逻辑文件空间。远端路径不是本机路径，只允许以 `user`、`public` 或 `account` 为首段；CLI 会规范化安全的 `.`、重复分隔符和首尾分隔符，逐段进行 URL 编码，保留合法的空格与非 ASCII 文件名，并在请求前拒绝任何 `..` 段、反斜杠和控制字符。 + +### `crater file upload ` + +- **描述**：把一个本地普通文件流式上传到远端逻辑路径。 +- **位置参数**： + - ``（必填）：本地普通文件。目录、管道、设备和 socket 会在请求前被拒绝。 + - ``（必填）：`user`、`public` 或 `account` 下的完整目标文件路径，不能只给逻辑根。 +- **选项**： + - `--overwrite`（bool）：允许替换已存在的远端普通文件；默认拒绝覆盖。 +- **处理逻辑**： + - 调用 `POST /api/ss/upload/*path?overwrite=`，请求体直接流式读取本地文件，不把完整内容载入内存。 + - storage service 在目标同目录写入临时文件，完成 `chmod`、`sync` 和 `close` 后才发布。新文件通过原子 no-clobber 链接发布；显式覆盖通过同目录原子重命名替换。 + - 服务端是覆盖策略的最终裁决者：即使预检后并发出现同名文件，未指定 `--overwrite` 也不会覆盖；上传失败不会暴露部分新文件或截断旧文件。 + - 父目录必须预先存在，本命令不会自动创建目录。 + - 不支持递归目录、glob、多文件、断点续传、分片或进度条。 +- **输出格式**： + - 默认模式：成功后展示本地路径、远端路径和已上传字节数。 + - `--json`：stdout 仅输出结果元数据，不包含文件内容。 +- **`--json` 的 `data`**： + - `local_path`（字符串）：本地输入路径。 + - `remote_path`（字符串）：规范化后的远端逻辑路径。 + - `bytes`（整数）：服务端完整接收并发布的字节数。 + - `overwrite`（布尔）：本次是否显式启用了覆盖选项。 + - `overwritten`（布尔）：本次是否实际替换了已有普通文件。 +- **兼容性**：安全上传端点随本功能新增。旧 storage service 会返回 404，CLI 不会回退到可能截断文件的旧 WebDAV PUT。 +- **状态**：[x] Completed diff --git a/cli/internal/api/file.go b/cli/internal/api/file.go new file mode 100644 index 000000000..555e12ce1 --- /dev/null +++ b/cli/internal/api/file.go @@ -0,0 +1,186 @@ +package api + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/url" + "strconv" + "strings" + + "github.com/imroc/req/v3" +) + +const maxFileErrorBody = 32 << 10 + +// FileUploadClient exposes the ordinary-user APIs needed to upload one file. +type FileUploadClient interface { + UploadFile(ctx context.Context, remotePath string, source io.Reader, overwrite bool) (FileUploadResult, error) +} + +// NewFileUploadClient creates a typed remote-file upload client. +func NewFileUploadClient(baseURL, token string) FileUploadClient { + return NewClient(baseURL).SetToken(token) +} + +// FileUploadResult is the server-confirmed metadata for an atomic upload. +type FileUploadResult struct { + RemotePath string `json:"remote_path"` + Bytes int64 `json:"bytes"` + Overwritten bool `json:"overwritten"` +} + +// SourceReadError identifies a failure reading the caller-owned upload source. +type SourceReadError struct { + Cause error +} + +func (e *SourceReadError) Error() string { + return "read upload source: " + e.Cause.Error() +} + +func (e *SourceReadError) Unwrap() error { + if e == nil { + return nil + } + return e.Cause +} + +// UploadFile streams one source to the storage service without buffering it in +// memory. The dedicated endpoint stages the body and atomically publishes it. +func (c *Client) UploadFile( + ctx context.Context, + remotePath string, + source io.Reader, + overwrite bool, +) (FileUploadResult, error) { + requestPath := FileUploadPath + "/" + escapeRemotePath(remotePath) + tracked := &uploadSourceReader{source: source} + request := c.httpClient.R(). + SetContext(ctx). + SetContentType("application/octet-stream"). + SetQueryParam("overwrite", strconv.FormatBool(overwrite)). + SetBody(tracked). + DisableAutoReadResponse() + + resp, err := request.Post(requestPath) + if err != nil { + if tracked.readErr != nil { + return FileUploadResult{}, &SourceReadError{Cause: tracked.readErr} + } + if resp != nil && resp.Response != nil { + return FileUploadResult{}, uploadProtocolError(resp, "failed to process upload response: "+err.Error()) + } + return FileUploadResult{}, &NetworkError{Cause: err} + } + if resp.Body != nil { + defer resp.Body.Close() + } + if tracked.readErr != nil { + return FileUploadResult{}, &SourceReadError{Cause: tracked.readErr} + } + if !resp.IsSuccessState() { + if resp.Body == nil { + return FileUploadResult{}, &RequestError{ + HTTPStatus: resp.GetStatusCode(), + Msg: http.StatusText(resp.GetStatusCode()), + } + } + return FileUploadResult{}, rawFileRequestError(resp) + } + if resp.Body == nil { + return FileUploadResult{}, uploadProtocolError(resp, "upload response body is empty") + } + body, readErr := io.ReadAll(io.LimitReader(resp.Body, maxFileErrorBody+1)) + if readErr != nil { + return FileUploadResult{}, uploadProtocolError(resp, "failed to read upload response: "+readErr.Error()) + } + if len(body) > maxFileErrorBody { + return FileUploadResult{}, uploadProtocolError(resp, "upload response exceeds size limit") + } + var result Response[FileUploadResult] + if err := json.Unmarshal(body, &result); err != nil { + return FileUploadResult{}, uploadProtocolError(resp, "invalid upload response: "+err.Error()) + } + if err := errorFromResponse(resp, result.Code, result.Message); err != nil { + return FileUploadResult{}, err + } + if result.Data.RemotePath != remotePath { + return FileUploadResult{}, uploadProtocolError(resp, "upload response remote_path does not match the request") + } + if result.Data.Bytes < 0 || result.Data.Bytes != tracked.read { + return FileUploadResult{}, uploadProtocolError(resp, "upload response byte count does not match the streamed source") + } + if result.Data.Overwritten && !overwrite { + return FileUploadResult{}, uploadProtocolError(resp, "server reported an overwrite without client authorization") + } + return result.Data, nil +} + +func uploadProtocolError(resp *req.Response, message string) *RequestError { + status := 0 + if resp != nil && resp.Response != nil { + status = resp.GetStatusCode() + } + return &RequestError{ + HTTPStatus: status, + Msg: message, + } +} + +type uploadSourceReader struct { + source io.Reader + read int64 + readErr error +} + +func (reader *uploadSourceReader) Read(data []byte) (int, error) { + read, err := reader.source.Read(data) + reader.read += int64(read) + if err != nil && !errors.Is(err, io.EOF) { + reader.readErr = err + } + return read, err +} + +func rawFileRequestError(resp *req.Response) error { + body, readErr := io.ReadAll(io.LimitReader(resp.Body, maxFileErrorBody+1)) + if readErr != nil { + message := http.StatusText(resp.GetStatusCode()) + if message == "" { + message = "failed to read error response" + } + return &RequestError{ + HTTPStatus: resp.GetStatusCode(), + Msg: message + ": " + readErr.Error(), + } + } + if len(body) > maxFileErrorBody { + body = body[:maxFileErrorBody] + } + + var envelope Response[json.RawMessage] + _ = json.Unmarshal(body, &envelope) + message := strings.TrimSpace(envelope.Message) + if message == "" { + message = strings.TrimSpace(strings.ToValidUTF8(string(body), "\uFFFD")) + } + if message == "" { + message = http.StatusText(resp.GetStatusCode()) + } + return &RequestError{ + HTTPStatus: resp.GetStatusCode(), + CraterCode: envelope.Code, + Msg: message, + } +} + +func escapeRemotePath(remotePath string) string { + segments := strings.Split(remotePath, "/") + for index := range segments { + segments[index] = url.PathEscape(segments[index]) + } + return strings.Join(segments, "/") +} diff --git a/cli/internal/api/file_test.go b/cli/internal/api/file_test.go new file mode 100644 index 000000000..f5aea02ac --- /dev/null +++ b/cli/internal/api/file_test.go @@ -0,0 +1,379 @@ +package api + +import ( + "bytes" + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/imroc/req/v3" +) + +func TestUploadFileStreamsAndEncodesPathSegments(t *testing.T) { + requestStarted := make(chan struct{}) + firstChunkRead := make(chan struct{}) + handlerDone := make(chan error, 1) + firstChunk := []byte{0x00, 0x01, 0xff, 'A'} + secondChunk := []byte("第二块") + + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + close(requestStarted) + if request.Method != http.MethodPost { + handlerDone <- errors.New("unexpected method: " + request.Method) + return + } + if request.URL.EscapedPath() != "/api/ss/upload/user/%E5%AE%9E%E9%AA%8C%20%231/100%25.bin" { + handlerDone <- errors.New("unexpected escaped path: " + request.URL.EscapedPath()) + return + } + if request.URL.Query().Get("overwrite") != "false" { + handlerDone <- errors.New("unexpected overwrite query") + return + } + if request.Header.Get("Content-Type") != "application/octet-stream" { + handlerDone <- errors.New("unexpected content type: " + request.Header.Get("Content-Type")) + return + } + if request.Header.Get("Authorization") != "Bearer secret" { + handlerDone <- errors.New("unexpected authorization header") + return + } + gotFirst := make([]byte, len(firstChunk)) + if _, err := io.ReadFull(request.Body, gotFirst); err != nil { + handlerDone <- err + return + } + if !bytes.Equal(gotFirst, firstChunk) { + handlerDone <- errors.New("unexpected first chunk") + return + } + close(firstChunkRead) + gotRest, err := io.ReadAll(request.Body) + if err != nil { + handlerDone <- err + return + } + if !bytes.Equal(gotRest, secondChunk) { + handlerDone <- errors.New("unexpected second chunk") + return + } + writer.WriteHeader(http.StatusCreated) + _, _ = io.WriteString(writer, `{"code":0,"data":{"remote_path":"user/实验 #1/100%.bin","bytes":13,"overwritten":false},"msg":""}`) + handlerDone <- nil + })) + defer server.Close() + + reader, writer := io.Pipe() + type uploadResult struct { + upload FileUploadResult + err error + } + result := make(chan uploadResult, 1) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + go func() { + upload, err := NewClient(server.URL).SetToken("secret").UploadFile( + ctx, + "user/实验 #1/100%.bin", + reader, + false, + ) + result <- uploadResult{upload: upload, err: err} + }() + + select { + case <-requestStarted: + case <-time.After(time.Second): + t.Fatal("request did not start before the complete source was available") + } + if _, err := writer.Write(firstChunk); err != nil { + t.Fatal(err) + } + select { + case <-firstChunkRead: + case <-time.After(time.Second): + t.Fatal("server did not receive the first chunk incrementally") + } + if _, err := writer.Write(secondChunk); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + + got := <-result + if got.err != nil { + t.Fatalf("UploadFile: %v", got.err) + } + if handlerErr := <-handlerDone; handlerErr != nil { + t.Fatal(handlerErr) + } + wantBytes := int64(len(firstChunk) + len(secondChunk)) + if got.upload.Bytes != wantBytes || got.upload.RemotePath != "user/实验 #1/100%.bin" || got.upload.Overwritten { + t.Fatalf("upload = %#v, want %d bytes", got.upload, wantBytes) + } +} + +func TestUploadFileOverwriteSetsExplicitQuery(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.URL.Query().Get("overwrite") != "true" { + t.Errorf("overwrite query = %q", request.URL.Query().Get("overwrite")) + } + _, _ = io.Copy(io.Discard, request.Body) + writer.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(writer, `{"code":0,"data":{"remote_path":"user/result.bin","bytes":11,"overwritten":true},"msg":""}`) + })) + defer server.Close() + + upload, err := NewClient(server.URL).UploadFile( + context.Background(), + "user/result.bin", + bytes.NewBufferString("replacement"), + true, + ) + if err != nil { + t.Fatalf("UploadFile: %v", err) + } + if upload.Bytes != int64(len("replacement")) || !upload.Overwritten { + t.Fatalf("upload = %#v", upload) + } +} + +func TestUploadFileAcceptsEmptySource(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + body, err := io.ReadAll(request.Body) + if err != nil { + t.Errorf("read body: %v", err) + } + if len(body) != 0 { + t.Errorf("body = %v, want empty", body) + } + writer.Header().Set("Content-Type", "application/json") + writer.WriteHeader(http.StatusCreated) + _, _ = io.WriteString(writer, `{"code":0,"data":{"remote_path":"user/empty.bin","bytes":0,"overwritten":false},"msg":""}`) + })) + defer server.Close() + + upload, err := NewClient(server.URL).UploadFile( + context.Background(), + "user/empty.bin", + bytes.NewReader(nil), + false, + ) + if err != nil { + t.Fatalf("UploadFile: %v", err) + } + if upload.Bytes != 0 || upload.Overwritten { + t.Fatalf("upload = %#v", upload) + } +} + +func TestUploadFileDecodesJSONAndPlainTextErrors(t *testing.T) { + tests := []struct { + name string + status int + body string + content string + wantCode int + wantMsg string + }{ + { + name: "Crater envelope", + status: http.StatusConflict, + body: `{"code":40901,"data":null,"msg":"target file already exists"}`, + content: "application/json", + wantCode: 40901, + wantMsg: "target file already exists", + }, + { + name: "plain text", + status: http.StatusConflict, + body: "parent directory does not exist", + content: "text/plain", + wantMsg: "parent directory does not exist", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + _, _ = io.Copy(io.Discard, request.Body) + writer.Header().Set("Content-Type", test.content) + writer.WriteHeader(test.status) + _, _ = io.WriteString(writer, test.body) + })) + defer server.Close() + + _, err := NewClient(server.URL).UploadFile( + context.Background(), + "user/result.bin", + bytes.NewBufferString("data"), + false, + ) + var requestErr *RequestError + if !errors.As(err, &requestErr) { + t.Fatalf("error = %T %v, want *RequestError", err, err) + } + if requestErr.HTTPStatus != test.status || + requestErr.CraterCode != test.wantCode || + requestErr.Msg != test.wantMsg { + t.Fatalf("request error = %#v", requestErr) + } + }) + } +} + +func TestUploadFileRejectsInvalidSuccessResponsesWithHTTPStatus(t *testing.T) { + tests := []struct { + name string + body string + overwrite bool + }{ + {name: "empty body"}, + {name: "malformed JSON", body: `{"code":`}, + {name: "null metadata", body: `{"code":0,"data":null,"msg":""}`}, + {name: "wrong path", body: `{"code":0,"data":{"remote_path":"user/other.bin","bytes":4,"overwritten":false},"msg":""}`}, + {name: "wrong byte count", body: `{"code":0,"data":{"remote_path":"user/result.bin","bytes":3,"overwritten":false},"msg":""}`}, + {name: "negative byte count", body: `{"code":0,"data":{"remote_path":"user/result.bin","bytes":-1,"overwritten":false},"msg":""}`}, + {name: "unauthorized overwrite", body: `{"code":0,"data":{"remote_path":"user/result.bin","bytes":4,"overwritten":true},"msg":""}`}, + {name: "oversized body", body: strings.Repeat("x", maxFileErrorBody+1)}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + _, _ = io.Copy(io.Discard, request.Body) + writer.Header().Set("Content-Type", "application/json") + writer.WriteHeader(http.StatusCreated) + _, _ = io.WriteString(writer, test.body) + })) + defer server.Close() + + _, err := NewClient(server.URL).UploadFile( + context.Background(), + "user/result.bin", + bytes.NewBufferString("data"), + test.overwrite, + ) + var requestErr *RequestError + if !errors.As(err, &requestErr) { + t.Fatalf("error = %T %v, want *RequestError", err, err) + } + if requestErr.HTTPStatus != http.StatusCreated { + t.Fatalf("HTTP status = %d, want %d", requestErr.HTTPStatus, http.StatusCreated) + } + }) + } +} + +type failingUploadReader struct { + err error +} + +func (reader failingUploadReader) Read([]byte) (int, error) { + return 0, reader.err +} + +func TestUploadFileIdentifiesSourceReadFailure(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + _, _ = io.Copy(io.Discard, request.Body) + writer.WriteHeader(http.StatusBadRequest) + })) + defer server.Close() + + sentinel := errors.New("local disk failed") + _, err := NewClient(server.URL).UploadFile( + context.Background(), + "user/result.bin", + failingUploadReader{err: sentinel}, + false, + ) + var sourceErr *SourceReadError + if !errors.As(err, &sourceErr) { + t.Fatalf("error = %T %v, want *SourceReadError", err, err) + } + if !errors.Is(sourceErr, sentinel) { + t.Fatalf("error = %v, want sentinel", sourceErr) + } +} + +type failingFileReadCloser struct { + err error +} + +func (reader failingFileReadCloser) Read([]byte) (int, error) { + return 0, reader.err +} + +func (failingFileReadCloser) Close() error { + return nil +} + +func TestUploadFileKeepsHTTPStatusWhenErrorBodyReadFails(t *testing.T) { + sentinel := errors.New("broken error body") + client := NewClient("https://example.invalid") + client.httpClient.GetTransport().WrapRoundTripFunc(func(_ http.RoundTripper) req.HttpRoundTripFunc { + return func(request *http.Request) (*http.Response, error) { + _, _ = io.Copy(io.Discard, request.Body) + return &http.Response{ + StatusCode: http.StatusConflict, + Status: "409 Conflict", + Header: make(http.Header), + Body: failingFileReadCloser{err: sentinel}, + Request: request, + }, nil + } + }) + + _, err := client.UploadFile( + context.Background(), + "user/result.bin", + bytes.NewBufferString("data"), + false, + ) + var requestErr *RequestError + if !errors.As(err, &requestErr) { + t.Fatalf("error = %T %v, want *RequestError", err, err) + } + if requestErr.HTTPStatus != http.StatusConflict || + requestErr.Msg != "Conflict: broken error body" { + t.Fatalf("request error = %#v", requestErr) + } +} + +func TestUploadFileKeepsHTTPStatusWhenSuccessBodyReadFails(t *testing.T) { + sentinel := errors.New("broken success body") + client := NewClient("https://example.invalid") + client.httpClient.GetTransport().WrapRoundTripFunc(func(_ http.RoundTripper) req.HttpRoundTripFunc { + return func(request *http.Request) (*http.Response, error) { + _, _ = io.Copy(io.Discard, request.Body) + return &http.Response{ + StatusCode: http.StatusCreated, + Status: "201 Created", + Header: make(http.Header), + Body: failingFileReadCloser{err: sentinel}, + Request: request, + }, nil + } + }) + + _, err := client.UploadFile( + context.Background(), + "user/result.bin", + bytes.NewBufferString("data"), + false, + ) + var requestErr *RequestError + if !errors.As(err, &requestErr) { + t.Fatalf("error = %T %v, want *RequestError", err, err) + } + if requestErr.HTTPStatus != http.StatusCreated || + requestErr.Msg != "failed to read upload response: broken success body" { + t.Fatalf("request error = %#v", requestErr) + } +} diff --git a/cli/internal/api/paths.go b/cli/internal/api/paths.go index 6ccc6fce4..8811cbe64 100644 --- a/cli/internal/api/paths.go +++ b/cli/internal/api/paths.go @@ -34,6 +34,7 @@ const ( SPJobsPrefix = "/api/v1/spjobs" VCJobsPrefix = "/api/v1/vcjobs" AdminVCJobsPrefix = "/api/v1/admin/vcjobs" + StoragePrefix = "/api/ss" ) const CompatibilityPath = CompatibilityPrefix + "/compatibility" @@ -54,4 +55,5 @@ const ( VCJobListPath = VCJobsPrefix VCJobBillingPath = VCJobsPrefix + "/billing" AdminVCJobBillingPath = AdminVCJobsPrefix + "/billing" + FileUploadPath = StoragePrefix + "/upload" ) diff --git a/cli/internal/i18n/catalog_file.go b/cli/internal/i18n/catalog_file.go new file mode 100644 index 000000000..760e7d5ac --- /dev/null +++ b/cli/internal/i18n/catalog_file.go @@ -0,0 +1,50 @@ +package i18n + +var catalogFile = map[Language]map[string]string{ + En: { + "file_short": "Upload remote files", + "file_long": "Upload files to user, public, and account storage spaces.", + "file_upload_short": "Upload one local file", + "file_upload_long": "Stream one local regular file to a path below user, public, or account storage.", + "file_upload_flag_overwrite": "Replace an existing remote file", + "file_label_local_file": "local file", + "file_label_remote_file": "remote path", + "err_file_path_invalid": "invalid remote path %q", + "err_file_path_root": "remote path must start with user, public, or account: %q", + "err_file_path_not_file": "remote file must name an entry below user, public, or account: %q", + "err_file_local_open": "failed to open local file %q: %s", + "err_file_local_stat": "failed to inspect local file %q: %s", + "err_file_local_not_regular": "local path is not a regular file: %q", + "err_file_local_read": "failed to read local file %q: %s", + "err_file_output": "failed to write command output: %s", + "file_upload_success": "Uploaded %s to %s (%d bytes)", + "file_root_user_desc": "Your private user storage.", + "file_root_public_desc": "Shared public storage.", + "file_root_account_desc": "Storage for the current account.", + "file_local_regular_desc": "Local regular file.", + "file_local_directory_desc": "Local directory.", + }, + ZhCN: { + "file_short": "上传远端文件", + "file_long": "向用户、公共及当前账户存储空间上传文件。", + "file_upload_short": "上传单个本地文件", + "file_upload_long": "将一个本地普通文件流式上传到 user、public 或 account 存储空间下的路径。", + "file_upload_flag_overwrite": "替换已存在的远端文件", + "file_label_local_file": "本地文件", + "file_label_remote_file": "远端路径", + "err_file_path_invalid": "无效的远端路径 %q", + "err_file_path_root": "远端路径必须以 user、public 或 account 开头：%q", + "err_file_path_not_file": "远端文件必须指向 user、public 或 account 下的具体条目：%q", + "err_file_local_open": "打开本地文件 %q 失败：%s", + "err_file_local_stat": "检查本地文件 %q 失败：%s", + "err_file_local_not_regular": "本地路径不是普通文件：%q", + "err_file_local_read": "读取本地文件 %q 失败：%s", + "err_file_output": "写入命令输出失败：%s", + "file_upload_success": "已将 %s 上传到 %s（%d 字节）", + "file_root_user_desc": "当前用户的私有存储空间", + "file_root_public_desc": "共享公共存储空间", + "file_root_account_desc": "当前账户的存储空间", + "file_local_regular_desc": "本地普通文件", + "file_local_directory_desc": "本地目录", + }, +} diff --git a/cli/internal/i18n/i18n.go b/cli/internal/i18n/i18n.go index c2a68e3a2..b2f2fa1d7 100644 --- a/cli/internal/i18n/i18n.go +++ b/cli/internal/i18n/i18n.go @@ -29,6 +29,7 @@ var translations = mergeCatalogs( catalogOrder, catalogErrors, catalogJob, + catalogFile, ) func mergeCatalogs(catalogs ...map[Language]map[string]string) map[Language]map[string]string { diff --git a/cli/skills/crater-cli-file/SKILL.md b/cli/skills/crater-cli-file/SKILL.md new file mode 100644 index 000000000..33f3c5d0a --- /dev/null +++ b/cli/skills/crater-cli-file/SKILL.md @@ -0,0 +1,60 @@ +--- +name: crater-cli-file +version: 0.1.0 +description: "Use Crater CLI to stream one local regular file into ordinary-user remote storage with explicit, atomic overwrite semantics." +metadata: + requires: + bins: ["crater"] + cliHelp: "crater file --help" +--- + +# Crater CLI File Upload + +**CRITICAL — Before doing anything else, MUST read `crater-cli-shared` (possible path: [`../crater-cli-shared/SKILL.md`](../crater-cli-shared/SKILL.md)) for global options, non-interactive use, errors, and sensitive information handling.** + +Use `crater file upload` when a user wants to copy one local regular file into Crater storage. + +## Supported workflow + +- Create a new remote file: + + ```bash + crater file upload ./train.py user/jobs/train.py + ``` + +- Upload a binary file to current-account storage: + + ```bash + crater file upload ./weights.bin "account/模型/weights.bin" + ``` + +- Replace an existing regular remote file only after the user explicitly asks for it: + + ```bash + crater file upload ./train.py user/jobs/train.py --overwrite + ``` + +- Return structured metadata: + + ```bash + crater file upload ./train.py user/jobs/train.py --json --no-interactive + ``` + +## Safety + +- The local path must resolve to one open regular file. Directories, devices, sockets, and pipes are rejected before any API request. +- Remote paths must start with `user`, `public`, or `account` and must name an entry below that root. +- Never add `--overwrite` unless replacing that exact remote target is part of the user's request. +- The server stages the complete stream in the target directory and atomically publishes it. A failed transfer never exposes a partial new file or truncates the previous file. +- Parent directories are never created automatically. +- This command uploads one file only. Do not pass a directory or shell glob. +- JSON stdout contains metadata only; it never includes file bytes. +- Do not ask the user to provide a token or Keyring content. + +## Troubleshooting + +1. Run `crater auth ls --json` and confirm an active context exists. +2. Use `crater file upload --help` to verify the local binary supports the command. +3. If the target exists, choose a new path or obtain explicit permission to add `--overwrite`. +4. A `404` from `/api/ss/upload` means the storage service is older than this CLI feature; upgrade the server rather than falling back to unsafe WebDAV PUT. +5. For API errors, inspect `category`, `code`, and `context.http_status` from JSON stderr without exposing credentials. diff --git a/cli/test/snapshots/file/file_test.go b/cli/test/snapshots/file/file_test.go new file mode 100644 index 000000000..66b799f6e --- /dev/null +++ b/cli/test/snapshots/file/file_test.go @@ -0,0 +1,59 @@ +package file_test + +import ( + "os" + "testing" + + "github.com/raids-lab/crater/cli/internal/snaptest" +) + +const goldenStemFileUpload = "file_upload" + +func TestFileUploadSnapshotsEN(t *testing.T) { + runFileUploadSnapshots(t, "en") +} + +func TestFileUploadSnapshotsZhCN(t *testing.T) { + runFileUploadSnapshots(t, "zh-CN") +} + +func runFileUploadSnapshots(t *testing.T, language string) { + t.Helper() + path := snaptest.GoldenFileT(t, "file", goldenStemFileUpload, language) + home := t.TempDir() + baseEnv := append(snaptest.EnvMinimal(home, language), "CRATER_TEST_SANDBOX_HTTP=error404") + binary := snaptest.CraterExecutable(t) + localFixture := ".snapshot-upload-" + language + ".bin" + if err := os.WriteFile(localFixture, []byte{0x00, 0xff, 'C', 'L', 'I'}, 0o600); err != nil { + t.Fatal(err) + } + defer os.Remove(localFixture) + + cases := []snaptest.Case{ + {ID: "01-file-typo-json", Args: []string{"file", "get", "--json", "--no-interactive"}}, + {ID: "01b-file-typo-text", Args: []string{"file", "get", "--no-interactive"}}, + {ID: "02-file-upload-missing-local-json", Args: []string{"file", "upload", "--json", "--no-interactive"}}, + {ID: "03-file-upload-missing-remote-json", Args: []string{"file", "upload", localFixture, "--json", "--no-interactive"}}, + {ID: "04-file-upload-extra-arg-json", Args: []string{"file", "upload", localFixture, "user/a.bin", "extra", "--json", "--no-interactive"}}, + {ID: "05-file-upload-traversal-json", Args: []string{"file", "upload", localFixture, "user/../public/a.bin", "--json", "--no-interactive"}}, + {ID: "06-file-upload-root-json", Args: []string{"file", "upload", localFixture, "user", "--json", "--no-interactive"}}, + {ID: "07-file-upload-directory-json", Args: []string{"file", "upload", ".", "user/a.bin", "--json", "--no-interactive"}}, + {ID: "08-file-upload-404-json", Args: []string{"file", "upload", localFixture, "user/实验 data/result.bin", "--json", "--no-interactive"}}, + {ID: "09-file-help", Args: []string{"file", "--help"}}, + {ID: "10-file-upload-help", Args: []string{"file", "upload", "--help"}}, + } + + results := make([]*snaptest.Result, len(cases)) + for index := range cases { + result, err := snaptest.Run(binary, baseEnv, cases[index].Args) + if err != nil { + t.Fatalf("case %s: %v", cases[index].ID, err) + } + results[index] = result + } + + update := os.Getenv("UPDATE_SNAPSHOTS") == "1" || os.Getenv("UPDATE_SNAPSHOTS") == "true" + if err := snaptest.MatchOrUpdateGolden(path, language, cases, results, update); err != nil { + t.Fatal(err) + } +} diff --git a/cli/testdata/snapshots/file/file_upload.en.txtar b/cli/testdata/snapshots/file/file_upload.en.txtar new file mode 100644 index 000000000..d93435bd9 --- /dev/null +++ b/cli/testdata/snapshots/file/file_upload.en.txtar @@ -0,0 +1,142 @@ +# Crater CLI snapshot bundle (txtar). Regenerate: make snapshot-update (or UPDATE_SNAPSHOTS=1 go test ./test/snapshots/...) +-- en/01-file-typo-json/argv -- +crater file get --json --no-interactive +-- en/01-file-typo-json/exit -- +2 +-- en/01-file-typo-json/stdout -- +-- en/01-file-typo-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_UNKNOWN_COMMAND", + "message": "unknown command \"get\" for \"crater file\"\nRun \"crater file --help\" for usage." +} +-- en/01b-file-typo-text/argv -- +crater file get --no-interactive +-- en/01b-file-typo-text/exit -- +2 +-- en/01b-file-typo-text/stdout -- +-- en/01b-file-typo-text/stderr -- +Error: + unknown command "get" for "crater file" + Run "crater file --help" for usage. +-- en/02-file-upload-missing-local-json/argv -- +crater file upload --json --no-interactive +-- en/02-file-upload-missing-local-json/exit -- +2 +-- en/02-file-upload-missing-local-json/stdout -- +-- en/02-file-upload-missing-local-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_MISSING_REQUIRED_FLAG", + "message": "local file is required (\u003clocal-file\u003e)" +} +-- en/03-file-upload-missing-remote-json/argv -- +crater file upload .snapshot-upload-en.bin --json --no-interactive +-- en/03-file-upload-missing-remote-json/exit -- +2 +-- en/03-file-upload-missing-remote-json/stdout -- +-- en/03-file-upload-missing-remote-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_MISSING_REQUIRED_FLAG", + "message": "remote path is required (\u003cremote-path\u003e)" +} +-- en/04-file-upload-extra-arg-json/argv -- +crater file upload .snapshot-upload-en.bin user/a.bin extra --json --no-interactive +-- en/04-file-upload-extra-arg-json/exit -- +2 +-- en/04-file-upload-extra-arg-json/stdout -- +-- en/04-file-upload-extra-arg-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_INVALID_FLAG_VALUE", + "message": "too many arguments for crater file upload: got 3, want at most 2" +} +-- en/05-file-upload-traversal-json/argv -- +crater file upload .snapshot-upload-en.bin user/../public/a.bin --json --no-interactive +-- en/05-file-upload-traversal-json/exit -- +2 +-- en/05-file-upload-traversal-json/stdout -- +-- en/05-file-upload-traversal-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_INVALID_FLAG_VALUE", + "message": "invalid remote path \"user/../public/a.bin\"" +} +-- en/06-file-upload-root-json/argv -- +crater file upload .snapshot-upload-en.bin user --json --no-interactive +-- en/06-file-upload-root-json/exit -- +2 +-- en/06-file-upload-root-json/stdout -- +-- en/06-file-upload-root-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_INVALID_FLAG_VALUE", + "message": "remote file must name an entry below user, public, or account: \"user\"" +} +-- en/07-file-upload-directory-json/argv -- +crater file upload . user/a.bin --json --no-interactive +-- en/07-file-upload-directory-json/exit -- +2 +-- en/07-file-upload-directory-json/stdout -- +-- en/07-file-upload-directory-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_INVALID_FLAG_VALUE", + "message": "local path is not a regular file: \".\"" +} +-- en/08-file-upload-404-json/argv -- +crater file upload .snapshot-upload-en.bin user/实验 data/result.bin --json --no-interactive +-- en/08-file-upload-404-json/exit -- +4 +-- en/08-file-upload-404-json/stdout -- +-- en/08-file-upload-404-json/stderr -- +{ + "category": "api_error", + "code": "ERR_NOT_FOUND_404", + "message": "HTTP 404: simulated", + "context": { + "crater_code": 404, + "http_status": 404, + "msg": "simulated" + } +} +-- en/09-file-help/argv -- +crater file --help +-- en/09-file-help/exit -- +0 +-- en/09-file-help/stdout -- +Upload files to user, public, and account storage spaces. + +Usage: + crater file [flags] + crater file [command] + +Available Commands: + upload Upload one local file + +Global Flags: + -h, --help Help for crater + --json Output in raw JSON format + --no-interactive Disable interactive prompts + +Use "crater file [command] --help" for more information about a command. +-- en/09-file-help/stderr -- +-- en/10-file-upload-help/argv -- +crater file upload --help +-- en/10-file-upload-help/exit -- +0 +-- en/10-file-upload-help/stdout -- +Stream one local regular file to a path below user, public, or account storage. + +Usage: + crater file upload [flags] + +Flags: + --overwrite Replace an existing remote file + +Global Flags: + -h, --help Help for crater + --json Output in raw JSON format + --no-interactive Disable interactive prompts +-- en/10-file-upload-help/stderr -- diff --git a/cli/testdata/snapshots/file/file_upload.zh-CN.txtar b/cli/testdata/snapshots/file/file_upload.zh-CN.txtar new file mode 100644 index 000000000..c03e10013 --- /dev/null +++ b/cli/testdata/snapshots/file/file_upload.zh-CN.txtar @@ -0,0 +1,142 @@ +# Crater CLI snapshot bundle (txtar). Regenerate: make snapshot-update (or UPDATE_SNAPSHOTS=1 go test ./test/snapshots/...) +-- zh-CN/01-file-typo-json/argv -- +crater file get --json --no-interactive +-- zh-CN/01-file-typo-json/exit -- +2 +-- zh-CN/01-file-typo-json/stdout -- +-- zh-CN/01-file-typo-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_UNKNOWN_COMMAND", + "message": "unknown command \"get\" for \"crater file\"\nRun \"crater file --help\" for usage." +} +-- zh-CN/01b-file-typo-text/argv -- +crater file get --no-interactive +-- zh-CN/01b-file-typo-text/exit -- +2 +-- zh-CN/01b-file-typo-text/stdout -- +-- zh-CN/01b-file-typo-text/stderr -- +Error: + unknown command "get" for "crater file" + Run "crater file --help" for usage. +-- zh-CN/02-file-upload-missing-local-json/argv -- +crater file upload --json --no-interactive +-- zh-CN/02-file-upload-missing-local-json/exit -- +2 +-- zh-CN/02-file-upload-missing-local-json/stdout -- +-- zh-CN/02-file-upload-missing-local-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_MISSING_REQUIRED_FLAG", + "message": "缺少必要参数：本地文件 (\u003clocal-file\u003e)" +} +-- zh-CN/03-file-upload-missing-remote-json/argv -- +crater file upload .snapshot-upload-zh-CN.bin --json --no-interactive +-- zh-CN/03-file-upload-missing-remote-json/exit -- +2 +-- zh-CN/03-file-upload-missing-remote-json/stdout -- +-- zh-CN/03-file-upload-missing-remote-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_MISSING_REQUIRED_FLAG", + "message": "缺少必要参数：远端路径 (\u003cremote-path\u003e)" +} +-- zh-CN/04-file-upload-extra-arg-json/argv -- +crater file upload .snapshot-upload-zh-CN.bin user/a.bin extra --json --no-interactive +-- zh-CN/04-file-upload-extra-arg-json/exit -- +2 +-- zh-CN/04-file-upload-extra-arg-json/stdout -- +-- zh-CN/04-file-upload-extra-arg-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_INVALID_FLAG_VALUE", + "message": "crater file upload 的参数过多：收到 3 个，最多允许 2 个" +} +-- zh-CN/05-file-upload-traversal-json/argv -- +crater file upload .snapshot-upload-zh-CN.bin user/../public/a.bin --json --no-interactive +-- zh-CN/05-file-upload-traversal-json/exit -- +2 +-- zh-CN/05-file-upload-traversal-json/stdout -- +-- zh-CN/05-file-upload-traversal-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_INVALID_FLAG_VALUE", + "message": "无效的远端路径 \"user/../public/a.bin\"" +} +-- zh-CN/06-file-upload-root-json/argv -- +crater file upload .snapshot-upload-zh-CN.bin user --json --no-interactive +-- zh-CN/06-file-upload-root-json/exit -- +2 +-- zh-CN/06-file-upload-root-json/stdout -- +-- zh-CN/06-file-upload-root-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_INVALID_FLAG_VALUE", + "message": "远端文件必须指向 user、public 或 account 下的具体条目：\"user\"" +} +-- zh-CN/07-file-upload-directory-json/argv -- +crater file upload . user/a.bin --json --no-interactive +-- zh-CN/07-file-upload-directory-json/exit -- +2 +-- zh-CN/07-file-upload-directory-json/stdout -- +-- zh-CN/07-file-upload-directory-json/stderr -- +{ + "category": "usage_error", + "code": "ERR_INVALID_FLAG_VALUE", + "message": "本地路径不是普通文件：\".\"" +} +-- zh-CN/08-file-upload-404-json/argv -- +crater file upload .snapshot-upload-zh-CN.bin user/实验 data/result.bin --json --no-interactive +-- zh-CN/08-file-upload-404-json/exit -- +4 +-- zh-CN/08-file-upload-404-json/stdout -- +-- zh-CN/08-file-upload-404-json/stderr -- +{ + "category": "api_error", + "code": "ERR_NOT_FOUND_404", + "message": "请求失败（HTTP 404）：simulated", + "context": { + "crater_code": 404, + "http_status": 404, + "msg": "simulated" + } +} +-- zh-CN/09-file-help/argv -- +crater file --help +-- zh-CN/09-file-help/exit -- +0 +-- zh-CN/09-file-help/stdout -- +向用户、公共及当前账户存储空间上传文件。 + +Usage: + crater file [flags] + crater file [command] + +Available Commands: + upload 上传单个本地文件 + +Global Flags: + -h, --help 显示帮助信息 + --json 以原始 JSON 格式输出 + --no-interactive 禁用交互式提示 + +Use "crater file [command] --help" for more information about a command. +-- zh-CN/09-file-help/stderr -- +-- zh-CN/10-file-upload-help/argv -- +crater file upload --help +-- zh-CN/10-file-upload-help/exit -- +0 +-- zh-CN/10-file-upload-help/stdout -- +将一个本地普通文件流式上传到 user、public 或 account 存储空间下的路径。 + +Usage: + crater file upload [flags] + +Flags: + --overwrite 替换已存在的远端文件 + +Global Flags: + -h, --help 显示帮助信息 + --json 以原始 JSON 格式输出 + --no-interactive 禁用交互式提示 +-- zh-CN/10-file-upload-help/stderr -- diff --git a/output/playwright/issue-480-file-upload.png b/output/playwright/issue-480-file-upload.png new file mode 100644 index 000000000..70936666e Binary files /dev/null and b/output/playwright/issue-480-file-upload.png differ