From 320c22545bd1ab49441b4083e9c24571fc9885b5 Mon Sep 17 00:00:00 2001 From: Stano Bocinec <4834459+sbocinec@users.noreply.github.com> Date: Mon, 22 Jun 2026 18:18:27 +0200 Subject: [PATCH] fix(git): bound protobuf git-sync memory (shallow clone) + allow disabling refresh (#2521) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(git): shallow-clone protobuf git-sync to bound in-memory usage Console clones the configured Git repository into go-git's in-memory object store to read protobuf schema files. The clone had no depth limit, so the entire repository history was materialized (decompressed and un-deltified) in RAM, and the periodic refresh used an incremental tree.Pull() that kept appending every newly fetched object to the same never-pruned store. For a large, active monorepo this is catastrophic: heap profiling of a production Console pod showed ~6.2 GB (92.75% of the heap) held by go-git's MemoryObject.Write under CloneRepository. The target repo packs to ~600 MB on disk but inflates to ~8.5 GB across full history, growing ~0.5 GB/month, eventually OOM-killing the node. Console only ever reads the current revision of the tracked files, so: - Clone shallowly: SingleBranch + Depth 1 + NoTags. Fetches just the tip snapshot instead of the full history. - Refresh by re-cloning instead of pulling. A fresh shallow clone keeps the in-memory store bounded to a single revision (the previous store is released for GC) and avoids go-git's unreliable shallow-pull behavior. CloneRepository already refreshes the file cache and fires OnFilesUpdatedHook; on failure the last good cache is retained. This cuts the footprint from multiple GB to roughly one checkout and makes it insensitive to repository history growth. * feat(git): allow refreshInterval 0 to disable periodic refresh Console proto schemas change rarely, so a frequent git refresh is often unnecessary and — now that each refresh is a shallow re-clone — wasteful on bandwidth. Operators may legitimately want "clone once at startup, refresh only on restart", but Validate() rejected refreshInterval: 0 even though SyncRepo already treated 0 as disabled (and the sample config documented 0 as the way to disable it). Allow it: - config: drop the Validate() rejection of RefreshInterval == 0; document on the field that 0 disables periodic refresh (clone once at startup). - service: guard SyncRepo with <= 0 instead of == 0 so a negative value also disables sync and can't panic time.NewTicker; make the log generic. Default remains 1m, so behavior is unchanged unless 0 is set explicitly. --- backend/pkg/config/git.go | 9 +++--- backend/pkg/git/service.go | 58 ++++++++++++++------------------------ 2 files changed, 26 insertions(+), 41 deletions(-) diff --git a/backend/pkg/config/git.go b/backend/pkg/config/git.go index 667a327728..27f497c772 100644 --- a/backend/pkg/config/git.go +++ b/backend/pkg/config/git.go @@ -29,7 +29,9 @@ type Git struct { // Whether or not to use the filename or the full filepath as key in the map IndexByFullFilepath bool `yaml:"-"` - // RefreshInterval specifies how often the repository shall be pulled to check for new changes. + // RefreshInterval specifies how often the repository shall be refreshed to check for new changes. + // A value of 0 disables periodic refresh: the repository is cloned once at startup and only + // refreshed again on restart. RefreshInterval time.Duration `yaml:"refreshInterval"` // Repository that contains markdown files that document a Kafka topic. @@ -54,9 +56,8 @@ func (c *Git) Validate() error { if !c.Enabled { return nil } - if c.RefreshInterval == 0 { - return fmt.Errorf("git config is enabled but refresh interval is set to 0 (disabled)") - } + // A refresh interval of 0 (or less) is allowed and disables periodic refresh; the repository + // is cloned once at startup. See SetDefaults for the default interval when none is configured. if c.MaxFileSize <= 0 { return fmt.Errorf("git config is enabled but file max size is <= 0") } diff --git a/backend/pkg/git/service.go b/backend/pkg/git/service.go index 5d3e7f69bb..6095ed2a73 100644 --- a/backend/pkg/git/service.go +++ b/backend/pkg/git/service.go @@ -11,7 +11,6 @@ package git import ( "context" - "errors" "fmt" "os" "os/signal" @@ -113,6 +112,13 @@ func (c *Service) CloneRepository(ctx context.Context) error { URL: c.Cfg.Repository.URL, Auth: c.auth, ReferenceName: referenceName, + // Console only ever reads the current revision of the tracked files, so + // we fetch a shallow, single-branch snapshot without tags. This avoids + // materializing the repository's full history in the in-memory object + // store, which for large/active repositories can consume many GB of RAM. + SingleBranch: true, + Depth: 1, + Tags: git.NoTags, } if c.Cfg.CloneSubmodules { @@ -144,18 +150,16 @@ func (c *Service) CloneRepository(ctx context.Context) error { return nil } -// SyncRepo periodically pulls the repository contents to ensure it's always up to date. When changes appear -// the file cache will be updated. This function will periodically pull the repository until the passed context -// is done. +// SyncRepo periodically refreshes the repository contents to ensure they are always up to date. When changes +// appear the file cache will be updated. This function periodically refreshes the repository until a termination +// signal is received. +// +// Because the repository is cloned shallowly into an in-memory store (see CloneRepository), refreshing is done +// by re-cloning rather than pulling: a re-clone keeps the in-memory object store bounded to a single revision +// (the previous store is released and garbage collected) and avoids go-git's unreliable shallow-pull behavior. func (c *Service) SyncRepo() { - if c.Cfg.RefreshInterval == 0 { - c.logger.Info("refresh interval for sync is set to 0 (disabled)") - return - } - - tree, err := c.repo.Worktree() - if err != nil { - c.logger.Error("failed to get work tree from repository. stopping git sync", zap.Error(err)) + if c.Cfg.RefreshInterval <= 0 { + c.logger.Info("periodic git refresh disabled (refresh interval <= 0); repository was cloned once at startup") return } @@ -170,33 +174,13 @@ func (c *Service) SyncRepo() { c.logger.Info("stopped sync", zap.String("reason", "received signal")) return case <-ticker.C: - var referenceName plumbing.ReferenceName - if c.Cfg.Repository.Branch != "" { - referenceName = plumbing.NewBranchReferenceName(c.Cfg.Repository.Branch) - } - err := tree.Pull(&git.PullOptions{Auth: c.auth, ReferenceName: referenceName}) - if err != nil { - if errors.Is(err, git.NoErrAlreadyUpToDate) { - continue - } - c.logger.Error("pulling the repo has failed", zap.Error(err)) + // Re-clone the latest shallow snapshot. CloneRepository rebuilds the in-memory filesystem and + // object store, updates the file cache, and fires OnFilesUpdatedHook on success. On failure it + // leaves the previously cached file contents intact, so we keep serving the last good revision. + if err := c.CloneRepository(context.Background()); err != nil { + c.logger.Error("refreshing the repo has failed", zap.Error(err)) continue } - - // Update cache with new markdowns - empty := make(map[string]filesystem.File) - files, err := c.readFiles(c.memFs, empty, c.Cfg.Repository.BaseDirectory, c.Cfg.Repository.MaxDepth) - if err != nil { - c.logger.Error("failed to read files after pulling", zap.Error(err)) - continue - } - c.setFileContents(files) - c.logger.Info("successfully pulled git repository", - zap.Int("read_files", len(files))) - - if c.OnFilesUpdatedHook != nil { - c.OnFilesUpdatedHook() - } } } }