fix(git): bound protobuf git-sync memory (shallow clone) + allow disabling refresh (#2521) [backport release-2.8]

backend/pkg/config/git.go

@@ -29,7 +29,9 @@ type Git struct {
 	// Whether or not to use the filename or the full filepath as key in the map
 	IndexByFullFilepath bool `yaml:"-"`
 
-	// RefreshInterval specifies how often the repository shall be pulled to check for new changes.
+	// RefreshInterval specifies how often the repository shall be refreshed to check for new changes.
+	// A value of 0 disables periodic refresh: the repository is cloned once at startup and only
+	// refreshed again on restart.
 	RefreshInterval time.Duration `yaml:"refreshInterval"`
 
 	// Repository that contains markdown files that document a Kafka topic.
@@ -54,9 +56,8 @@ func (c *Git) Validate() error {
 	if !c.Enabled {
 		return nil
 	}
-	if c.RefreshInterval == 0 {
-		return fmt.Errorf("git config is enabled but refresh interval is set to 0 (disabled)")
-	}
+	// A refresh interval of 0 (or less) is allowed and disables periodic refresh; the repository
+	// is cloned once at startup. See SetDefaults for the default interval when none is configured.
 	if c.MaxFileSize <= 0 {
 		return fmt.Errorf("git config is enabled but file max size is <= 0")
 	}

backend/pkg/git/service.go

@@ -11,7 +11,6 @@ package git
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"os"
 	"os/signal"
@@ -113,6 +112,13 @@ func (c *Service) CloneRepository(ctx context.Context) error {
 		URL:           c.Cfg.Repository.URL,
 		Auth:          c.auth,
 		ReferenceName: referenceName,
+		// Console only ever reads the current revision of the tracked files, so
+		// we fetch a shallow, single-branch snapshot without tags. This avoids
+		// materializing the repository's full history in the in-memory object
+		// store, which for large/active repositories can consume many GB of RAM.
+		SingleBranch: true,
+		Depth:        1,
+		Tags:         git.NoTags,
 	}
 
 	if c.Cfg.CloneSubmodules {
@@ -144,18 +150,16 @@ func (c *Service) CloneRepository(ctx context.Context) error {
 	return nil
 }
 
-// SyncRepo periodically pulls the repository contents to ensure it's always up to date. When changes appear
-// the file cache will be updated. This function will periodically pull the repository until the passed context
-// is done.
+// SyncRepo periodically refreshes the repository contents to ensure they are always up to date. When changes
+// appear the file cache will be updated. This function periodically refreshes the repository until a termination
+// signal is received.
+//
+// Because the repository is cloned shallowly into an in-memory store (see CloneRepository), refreshing is done
+// by re-cloning rather than pulling: a re-clone keeps the in-memory object store bounded to a single revision
+// (the previous store is released and garbage collected) and avoids go-git's unreliable shallow-pull behavior.
 func (c *Service) SyncRepo() {
-	if c.Cfg.RefreshInterval == 0 {
-		c.logger.Info("refresh interval for sync is set to 0 (disabled)")
-		return
-	}
-
-	tree, err := c.repo.Worktree()
-	if err != nil {
-		c.logger.Error("failed to get work tree from repository. stopping git sync", zap.Error(err))
+	if c.Cfg.RefreshInterval <= 0 {
+		c.logger.Info("periodic git refresh disabled (refresh interval <= 0); repository was cloned once at startup")
 		return
 	}
 
@@ -170,33 +174,13 @@ func (c *Service) SyncRepo() {
 			c.logger.Info("stopped sync", zap.String("reason", "received signal"))
 			return
 		case <-ticker.C:
-			var referenceName plumbing.ReferenceName
-			if c.Cfg.Repository.Branch != "" {
-				referenceName = plumbing.NewBranchReferenceName(c.Cfg.Repository.Branch)
-			}
-			err := tree.Pull(&git.PullOptions{Auth: c.auth, ReferenceName: referenceName})
-			if err != nil {
-				if errors.Is(err, git.NoErrAlreadyUpToDate) {
-					continue
-				}
-				c.logger.Error("pulling the repo has failed", zap.Error(err))
+			// Re-clone the latest shallow snapshot. CloneRepository rebuilds the in-memory filesystem and
+			// object store, updates the file cache, and fires OnFilesUpdatedHook on success. On failure it
+			// leaves the previously cached file contents intact, so we keep serving the last good revision.
+			if err := c.CloneRepository(context.Background()); err != nil {
+				c.logger.Error("refreshing the repo has failed", zap.Error(err))
 				continue
 			}
-
-			// Update cache with new markdowns
-			empty := make(map[string]filesystem.File)
-			files, err := c.readFiles(c.memFs, empty, c.Cfg.Repository.BaseDirectory, c.Cfg.Repository.MaxDepth)
-			if err != nil {
-				c.logger.Error("failed to read files after pulling", zap.Error(err))
-				continue
-			}
-			c.setFileContents(files)
-			c.logger.Info("successfully pulled git repository",
-				zap.Int("read_files", len(files)))
-
-			if c.OnFilesUpdatedHook != nil {
-				c.OnFilesUpdatedHook()
-			}
 		}
 	}
 }