From 0b12b4d26b291a7b3d84c6bcf46cda7d300a9cb6 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 28 Jun 2026 15:33:16 -0700 Subject: [PATCH] feat(attachment): expose sha256 checksums for attachments --- models/migrations/migrations.go | 2 + models/migrations/v1_28/v343.go | 14 +++++ models/repo/attachment.go | 1 + modules/structs/attachment.go | 2 + routers/api/v1/repo/issue_attachment.go | 2 + .../api/v1/repo/issue_comment_attachment.go | 2 + routers/api/v1/repo/release.go | 4 ++ routers/api/v1/repo/release_attachment.go | 2 + routers/api/v1/repo/release_tags.go | 2 + routers/web/repo/issue.go | 2 + routers/web/repo/issue_comment.go | 2 + routers/web/repo/release.go | 4 ++ services/attachment/attachment.go | 6 ++- services/attachment/attachment_test.go | 7 +++ services/attachment/checksum.go | 51 +++++++++++++++++++ services/convert/attachment.go | 1 + templates/repo/release/list.tmpl | 12 +++-- templates/swagger/v1_json.tmpl | 5 ++ templates/swagger/v1_openapi3_json.tmpl | 5 ++ tests/e2e/release.test.ts | 41 ++++++++++++++- .../api_comment_attachment_test.go | 5 ++ .../integration/api_issue_attachment_test.go | 5 ++ tests/integration/api_releases_test.go | 8 +++ web_src/css/repo/release-tag.css | 3 +- 24 files changed, 181 insertions(+), 7 deletions(-) create mode 100644 models/migrations/v1_28/v343.go create mode 100644 services/attachment/checksum.go diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index f77ce032014f3..d4de9d3410c81 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -28,6 +28,7 @@ import ( "gitea.dev/models/migrations/v1_25" "gitea.dev/models/migrations/v1_26" "gitea.dev/models/migrations/v1_27" + "gitea.dev/models/migrations/v1_28" "gitea.dev/models/migrations/v1_6" "gitea.dev/models/migrations/v1_7" "gitea.dev/models/migrations/v1_8" @@ -420,6 +421,7 @@ func prepareMigrationTasks() []*migration { newMigration(340, "Add ContinueOnError column to ActionRunJob", v1_27.AddContinueOnErrorToActionRunJob), newMigration(341, "Convert legacy MSSQL DATETIME columns to DATETIME2", v1_27.FixLegacyMSSQLDateTimeColumns), newMigration(342, "Add scoped workflows schema", v1_27.AddScopedWorkflowsSchema), + newMigration(343, "Add sha256 checksum column to attachment", v1_28.AddChecksumsToAttachment), } return preparedMigrations } diff --git a/models/migrations/v1_28/v343.go b/models/migrations/v1_28/v343.go new file mode 100644 index 0000000000000..b0a9fa7141ddc --- /dev/null +++ b/models/migrations/v1_28/v343.go @@ -0,0 +1,14 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package v1_28 + +import "gitea.dev/models/db" + +func AddChecksumsToAttachment(x db.EngineMigration) error { + type Attachment struct { + ID int64 `xorm:"pk autoincr"` + HashSHA256 string `xorm:"hash_sha256 VARCHAR(64) NOT NULL DEFAULT ''"` + } + return x.Sync(new(Attachment)) +} diff --git a/models/repo/attachment.go b/models/repo/attachment.go index cb74f310d4770..7d426d259e583 100644 --- a/models/repo/attachment.go +++ b/models/repo/attachment.go @@ -31,6 +31,7 @@ type Attachment struct { UploaderID int64 `xorm:"INDEX DEFAULT 0"` // Notice: will be zero before this column added CommentID int64 `xorm:"INDEX"` Name string + HashSHA256 string `xorm:"hash_sha256 VARCHAR(64) NOT NULL DEFAULT ''"` DownloadCount int64 `xorm:"DEFAULT 0"` Size int64 `xorm:"DEFAULT 0"` CreatedUnix timeutil.TimeStamp `xorm:"created"` diff --git a/modules/structs/attachment.go b/modules/structs/attachment.go index 5d1788f71514f..cf1d7e30761f2 100644 --- a/modules/structs/attachment.go +++ b/modules/structs/attachment.go @@ -25,6 +25,8 @@ type Attachment struct { UUID string `json:"uuid"` // DownloadURL is the URL to download the attachment DownloadURL string `json:"browser_download_url"` + // HashSHA256 is the SHA256 hash of the attachment + HashSHA256 string `json:"sha256"` } // EditAttachmentOptions options for editing attachments diff --git a/routers/api/v1/repo/issue_attachment.go b/routers/api/v1/repo/issue_attachment.go index 123f66399d41d..18149a31dfe06 100644 --- a/routers/api/v1/repo/issue_attachment.go +++ b/routers/api/v1/repo/issue_attachment.go @@ -66,6 +66,7 @@ func GetIssueAttachment(ctx *context.APIContext) { if attach == nil { return } + attachment_service.EnsureSHA256(ctx, attach) ctx.JSON(http.StatusOK, convert.ToAPIAttachment(ctx.Repo.Repository, attach)) } @@ -109,6 +110,7 @@ func ListIssueAttachments(ctx *context.APIContext) { ctx.APIErrorInternal(err) return } + attachment_service.EnsureSHA256(ctx, issue.Attachments...) ctx.JSON(http.StatusOK, convert.ToAPIIssue(ctx, ctx.Doer, issue).Attachments) } diff --git a/routers/api/v1/repo/issue_comment_attachment.go b/routers/api/v1/repo/issue_comment_attachment.go index 56d494190e484..fa5095f678606 100644 --- a/routers/api/v1/repo/issue_comment_attachment.go +++ b/routers/api/v1/repo/issue_comment_attachment.go @@ -71,6 +71,7 @@ func GetIssueCommentAttachment(ctx *context.APIContext) { ctx.APIErrorNotFound("attachment not in comment") return } + attachment_service.EnsureSHA256(ctx, attachment) ctx.JSON(http.StatusOK, convert.ToAPIAttachment(ctx.Repo.Repository, attachment)) } @@ -113,6 +114,7 @@ func ListIssueCommentAttachments(ctx *context.APIContext) { ctx.APIErrorInternal(err) return } + attachment_service.EnsureSHA256(ctx, comment.Attachments...) ctx.JSON(http.StatusOK, convert.ToAPIAttachments(ctx.Repo.Repository, comment.Attachments)) } diff --git a/routers/api/v1/repo/release.go b/routers/api/v1/repo/release.go index a4fc03ae32100..a35b5a4fd5090 100644 --- a/routers/api/v1/repo/release.go +++ b/routers/api/v1/repo/release.go @@ -14,6 +14,7 @@ import ( api "gitea.dev/modules/structs" "gitea.dev/modules/web" "gitea.dev/routers/api/v1/utils" + attachment_service "gitea.dev/services/attachment" "gitea.dev/services/context" "gitea.dev/services/convert" release_service "gitea.dev/services/release" @@ -84,6 +85,7 @@ func GetRelease(ctx *context.APIContext) { ctx.APIErrorInternal(err) return } + attachment_service.EnsureSHA256(ctx, release.Attachments...) ctx.JSON(http.StatusOK, convert.ToAPIRelease(ctx, ctx.Repo.Repository, release)) } @@ -125,6 +127,7 @@ func GetLatestRelease(ctx *context.APIContext) { ctx.APIErrorInternal(err) return } + attachment_service.EnsureSHA256(ctx, release.Attachments...) ctx.JSON(http.StatusOK, convert.ToAPIRelease(ctx, ctx.Repo.Repository, release)) } @@ -191,6 +194,7 @@ func ListReleases(ctx *context.APIContext) { ctx.APIErrorInternal(err) return } + attachment_service.EnsureSHA256(ctx, release.Attachments...) rels[i] = convert.ToAPIRelease(ctx, ctx.Repo.Repository, release) } diff --git a/routers/api/v1/repo/release_attachment.go b/routers/api/v1/repo/release_attachment.go index 915ac725b06ad..68ec6b3bccdb1 100644 --- a/routers/api/v1/repo/release_attachment.go +++ b/routers/api/v1/repo/release_attachment.go @@ -97,6 +97,7 @@ func GetReleaseAttachment(ctx *context.APIContext) { ctx.APIErrorNotFound() return } + attachment_service.EnsureSHA256(ctx, attach) // FIXME Should prove the existence of the given repo, but results in unnecessary database requests ctx.JSON(http.StatusOK, convert.ToAPIAttachment(ctx.Repo.Repository, attach)) } @@ -153,6 +154,7 @@ func ListReleaseAttachments(ctx *context.APIContext) { ctx.APIErrorInternal(err) return } + attachment_service.EnsureSHA256(ctx, release.Attachments...) ctx.JSON(http.StatusOK, convert.ToAPIRelease(ctx, ctx.Repo.Repository, release).Attachments) } diff --git a/routers/api/v1/repo/release_tags.go b/routers/api/v1/repo/release_tags.go index 65338607c5677..9a65f788ba000 100644 --- a/routers/api/v1/repo/release_tags.go +++ b/routers/api/v1/repo/release_tags.go @@ -8,6 +8,7 @@ import ( repo_model "gitea.dev/models/repo" unit_model "gitea.dev/models/unit" + attachment_service "gitea.dev/services/attachment" "gitea.dev/services/context" "gitea.dev/services/convert" release_service "gitea.dev/services/release" @@ -70,6 +71,7 @@ func GetReleaseByTag(ctx *context.APIContext) { ctx.APIErrorInternal(err) return } + attachment_service.EnsureSHA256(ctx, release.Attachments...) ctx.JSON(http.StatusOK, convert.ToAPIRelease(ctx, ctx.Repo.Repository, release)) } diff --git a/routers/web/repo/issue.go b/routers/web/repo/issue.go index 10c9d7d726ba3..615cec7ffbb24 100644 --- a/routers/web/repo/issue.go +++ b/routers/web/repo/issue.go @@ -29,6 +29,7 @@ import ( "gitea.dev/modules/util" "gitea.dev/modules/web" "gitea.dev/routers/common" + attachment_service "gitea.dev/services/attachment" "gitea.dev/services/context" "gitea.dev/services/convert" "gitea.dev/services/forms" @@ -579,6 +580,7 @@ func GetIssueAttachments(ctx *context.Context) { if ctx.Written() { return } + attachment_service.EnsureSHA256(ctx, issue.Attachments...) attachments := make([]*api.Attachment, len(issue.Attachments)) for i := 0; i < len(issue.Attachments); i++ { attachments[i] = convert.ToAttachment(ctx.Repo.Repository, issue.Attachments[i]) diff --git a/routers/web/repo/issue_comment.go b/routers/web/repo/issue_comment.go index 4681c2fecd992..9fbf92f528244 100644 --- a/routers/web/repo/issue_comment.go +++ b/routers/web/repo/issue_comment.go @@ -23,6 +23,7 @@ import ( api "gitea.dev/modules/structs" "gitea.dev/modules/util" "gitea.dev/modules/web" + attachment_service "gitea.dev/services/attachment" "gitea.dev/services/context" "gitea.dev/services/convert" "gitea.dev/services/forms" @@ -444,6 +445,7 @@ func GetCommentAttachments(ctx *context.Context) { ctx.ServerError("LoadAttachments", err) return } + attachment_service.EnsureSHA256(ctx, comment.Attachments...) for i := 0; i < len(comment.Attachments); i++ { attachments = append(attachments, convert.ToAttachment(ctx.Repo.Repository, comment.Attachments[i])) } diff --git a/routers/web/repo/release.go b/routers/web/repo/release.go index 128fcd5369505..7f56f270a196d 100644 --- a/routers/web/repo/release.go +++ b/routers/web/repo/release.go @@ -28,6 +28,7 @@ import ( "gitea.dev/modules/web" "gitea.dev/routers/web/feed" shared_user "gitea.dev/routers/web/shared/user" + attachment_service "gitea.dev/services/attachment" "gitea.dev/services/context" "gitea.dev/services/context/upload" "gitea.dev/services/forms" @@ -90,6 +91,9 @@ func getReleaseInfos(ctx *context.Context, opts *repo_model.FindReleasesOptions) if err = repo_model.GetReleaseAttachments(ctx, releases...); err != nil { return nil, err } + for _, release := range releases { + attachment_service.EnsureSHA256(ctx, release.Attachments...) + } // Temporary cache commits count of used branches to speed up. countCache := make(map[string]int64) diff --git a/services/attachment/attachment.go b/services/attachment/attachment.go index 372cfb62f9fcc..a15cb054e3d56 100644 --- a/services/attachment/attachment.go +++ b/services/attachment/attachment.go @@ -6,6 +6,8 @@ package attachment import ( "bytes" "context" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "io" @@ -29,11 +31,13 @@ func NewAttachment(ctx context.Context, attach *repo_model.Attachment, file io.R err := db.WithTx(ctx, func(ctx context.Context) error { attach.UUID = uuid.New().String() - size, err := storage.Attachments.Save(attach.RelativePath(), file, size) + hasher := sha256.New() + size, err := storage.Attachments.Save(attach.RelativePath(), io.TeeReader(file, hasher), size) if err != nil { return fmt.Errorf("Attachments.Save: %w", err) } attach.Size = size + attach.HashSHA256 = hex.EncodeToString(hasher.Sum(nil)) return db.Insert(ctx, attach) }) diff --git a/services/attachment/attachment_test.go b/services/attachment/attachment_test.go index 7df200769b974..117974f3c6491 100644 --- a/services/attachment/attachment_test.go +++ b/services/attachment/attachment_test.go @@ -4,6 +4,8 @@ package attachment import ( + "crypto/sha256" + "encoding/hex" "os" "path/filepath" "testing" @@ -42,4 +44,9 @@ func TestUploadAttachment(t *testing.T) { assert.NoError(t, err) assert.Equal(t, user.ID, attachment.UploaderID) assert.Equal(t, int64(0), attachment.DownloadCount) + + content, err := os.ReadFile(fPath) + assert.NoError(t, err) + sha256Sum := sha256.Sum256(content) + assert.Equal(t, hex.EncodeToString(sha256Sum[:]), attachment.HashSHA256) } diff --git a/services/attachment/checksum.go b/services/attachment/checksum.go new file mode 100644 index 0000000000000..66e5f6a74f581 --- /dev/null +++ b/services/attachment/checksum.go @@ -0,0 +1,51 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package attachment + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "io" + + "gitea.dev/models/db" + repo_model "gitea.dev/models/repo" + "gitea.dev/modules/log" + "gitea.dev/modules/storage" +) + +// EnsureSHA256 backfills missing attachment sha256 values on demand. +func EnsureSHA256(ctx context.Context, attachments ...*repo_model.Attachment) { + for _, attachment := range attachments { + if attachment == nil || attachment.ID == 0 || attachment.HashSHA256 != "" { + continue + } + if err := backfillSHA256(ctx, attachment); err != nil { + log.Warn("Unable to backfill attachment sha256 for %s: %v", attachment.UUID, err) + } + } +} + +func backfillSHA256(ctx context.Context, attachment *repo_model.Attachment) error { + fr, err := storage.Attachments.Open(attachment.RelativePath()) + if err != nil { + return err + } + defer fr.Close() + + hasher := sha256.New() + if _, err := io.Copy(hasher, fr); err != nil { + return err + } + + attach := &repo_model.Attachment{ + ID: attachment.ID, + HashSHA256: hex.EncodeToString(hasher.Sum(nil)), + } + if _, err := db.GetEngine(ctx).ID(attachment.ID).Cols("hash_sha256").Update(attach); err != nil { + return err + } + attachment.HashSHA256 = attach.HashSHA256 + return nil +} diff --git a/services/convert/attachment.go b/services/convert/attachment.go index 5ab1fadd06326..539d96b800a12 100644 --- a/services/convert/attachment.go +++ b/services/convert/attachment.go @@ -33,6 +33,7 @@ func toAttachment(repo *repo_model.Repository, a *repo_model.Attachment, getDown Name: a.Name, Created: a.CreatedUnix.AsTime(), DownloadCount: a.DownloadCount, + HashSHA256: a.HashSHA256, Size: a.Size, UUID: a.UUID, DownloadURL: getDownloadURL(repo, a), // for web request json and api request json, return different download urls diff --git a/templates/repo/release/list.tmpl b/templates/repo/release/list.tmpl index 1009d224372f8..48f5df321b71b 100644 --- a/templates/repo/release/list.tmpl +++ b/templates/repo/release/list.tmpl @@ -80,15 +80,19 @@