TRT-2741: Add partitioned prefix sum tables for matview refresh optimization

cmd/sippy/backfill.go

@@ -0,0 +1,102 @@
+package main
+
+import (
+	"fmt"
+	"time"
+
+	"cloud.google.com/go/civil"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	"k8s.io/apimachinery/pkg/util/sets"
+
+	"github.com/openshift/sippy/pkg/flags"
+	"github.com/openshift/sippy/pkg/sippyserver"
+)
+
+type BackfillFlags struct {
+	DBFlags   *flags.PostgresFlags
+	Table     string
+	StartDate string
+	EndDate   string
+}
+
+func NewBackfillFlags() *BackfillFlags {
+	return &BackfillFlags{
+		DBFlags: flags.NewPostgresDatabaseFlags(),
+	}
+}
+
+func (f *BackfillFlags) BindFlags(fs *pflag.FlagSet) {
+	f.DBFlags.BindFlags(fs)
+	fs.StringVar(&f.Table, "table", "", "Table to backfill (daily-summaries, daily-totals, cumulative-summaries)")
+	fs.StringVar(&f.StartDate, "start-date", "", "Start date (YYYY-MM-DD)")
+	fs.StringVar(&f.EndDate, "end-date", "", "End date (YYYY-MM-DD)")
+}
+
+func NewBackfillCommand() *cobra.Command {
+	f := NewBackfillFlags()
+
+	cmd := &cobra.Command{
+		Use:   "backfill",
+		Short: "Backfill a specific summary table for a date range",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			validTables := sets.New[string]("daily-summaries", "daily-totals", "cumulative-summaries")
+			if f.Table == "" {
+				return fmt.Errorf("--table is required")
+			}
+			if !validTables.Has(f.Table) {
+				return fmt.Errorf("invalid --table %q: must be one of %v", f.Table, sets.List(validTables))
+			}
+			if f.StartDate == "" {
+				return fmt.Errorf("--start-date is required")
+			}
+
+			startDate, err := civil.ParseDate(f.StartDate)
+			if err != nil {
+				return fmt.Errorf("invalid --start-date %q: %w", f.StartDate, err)
+			}
+
+			endDate := civil.DateOf(time.Now().UTC())
+			if f.EndDate != "" {
+				endDate, err = civil.ParseDate(f.EndDate)
+				if err != nil {
+					return fmt.Errorf("invalid --end-date %q: %w", f.EndDate, err)
+				}
+			}
+
+			if startDate.After(endDate) {
+				return fmt.Errorf("--start-date (%s) is after --end-date (%s)", startDate, endDate)
+			}
+
+			dbc, err := f.DBFlags.GetDBClient()
+			if err != nil {
+				return fmt.Errorf("getting db client: %w", err)
+			}
+
+			log.WithFields(log.Fields{
+				"table": f.Table,
+				"start": startDate,
+				"end":   endDate,
+			}).Info("starting backfill")
+
+			var releases []string
+			if err := dbc.DB.Table("release_definitions").
+				Order("major DESC, minor DESC").
+				Pluck("release", &releases).Error; err != nil {
+				return fmt.Errorf("querying releases: %w", err)
+			}
+			startTime := time.Date(startDate.Year, startDate.Month, startDate.Day, 0, 0, 0, 0, time.UTC)
+			endTime := time.Date(endDate.Year, endDate.Month, endDate.Day, 0, 0, 0, 0, time.UTC).AddDate(0, 0, 1)
+			if _, err := dbc.EnsurePartitions(releases, startTime, endTime, false); err != nil {
+				return fmt.Errorf("ensuring partitions: %w", err)
+			}
+
+			return sippyserver.BackfillData(dbc, f.Table, startDate, endDate)
+		},
+	}
+
+	f.BindFlags(cmd.Flags())
+
+	return cmd
+}

cmd/sippy/load.go

@@ -41,7 +41,6 @@ import (
 	"github.com/openshift/sippy/pkg/dataloader/releaseloader"
 	"github.com/openshift/sippy/pkg/dataloader/testownershiploader"
 	"github.com/openshift/sippy/pkg/db"
-	"github.com/openshift/sippy/pkg/db/dailysummary"
 	"github.com/openshift/sippy/pkg/flags"
 	"github.com/openshift/sippy/pkg/github/commenter"
 )
@@ -371,7 +370,10 @@ func NewLoadCommand() *cobra.Command {
 			log.WithField("elapsed", elapsed).Info("database load complete")
 
 			if refreshMatviews && !f.SkipMatviewRefresh {
-				sippyserver.RefreshData(dbc, cacheClient, false, dailysummary.Options{})
+				if err := sippyserver.RefreshData(dbc, cacheClient, sippyserver.RefreshOptions{}); err != nil {
+					log.WithError(err).Error("refresh failed")
+					allErrs = append(allErrs, err)
+				}
 			}
 
 			elapsed = time.Since(start)

cmd/sippy/main.go

@@ -39,6 +39,7 @@ func main() {
 		NewLoadCommand(),
 		NewSnapshotCommand(),
 		NewRefreshCommand(),
+		NewBackfillCommand(),
 		NewComponentReadinessCommand(),
 		NewAutomateJiraCommand(),
 		NewVariantsCommand(),

cmd/sippy/refresh.go

@@ -1,25 +1,18 @@
 package main
 
 import (
-	"fmt"
-	"time"
-
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 
-	"github.com/openshift/sippy/pkg/db/dailysummary"
 	"github.com/openshift/sippy/pkg/flags"
 	"github.com/openshift/sippy/pkg/sippyserver"
 )
 
 type RefreshFlags struct {
-	DBFlags                 *flags.PostgresFlags
-	CacheFlags              *flags.CacheFlags
-	RefreshOnlyIfEmpty      bool
-	RebuildDailySummaries   bool
-	DailySummariesStartDate string
-	DailySummariesEndDate   string
+	DBFlags            *flags.PostgresFlags
+	CacheFlags         *flags.CacheFlags
+	RefreshOnlyIfEmpty bool
 }
 
 func NewRefreshFlags() *RefreshFlags {
@@ -33,32 +26,6 @@ func (f *RefreshFlags) BindFlags(fs *pflag.FlagSet) {
 	f.DBFlags.BindFlags(fs)
 	f.CacheFlags.BindFlags(fs)
 	fs.BoolVar(&f.RefreshOnlyIfEmpty, "refresh-only-if-empty", f.RefreshOnlyIfEmpty, "only refresh matviews if they're empty")
-	fs.BoolVar(&f.RebuildDailySummaries, "rebuild-daily-summaries", false, "Truncate and rebuild the daily summaries table for the default lookback period (use --daily-summaries-start for older data)")
-	fs.StringVar(&f.DailySummariesStartDate, "daily-summaries-start", "", "Override start date for daily summaries (YYYY-MM-DD)")
-	fs.StringVar(&f.DailySummariesEndDate, "daily-summaries-end", "", "Override end date for daily summaries (YYYY-MM-DD)")
-}
-
-func (f *RefreshFlags) dailySummaryOptions() (dailysummary.Options, error) {
-	opts := dailysummary.Options{Rebuild: f.RebuildDailySummaries}
-	if f.DailySummariesStartDate != "" {
-		t, err := time.Parse("2006-01-02", f.DailySummariesStartDate)
-		if err != nil {
-			return opts, fmt.Errorf("invalid --daily-summaries-start %q: %w", f.DailySummariesStartDate, err)
-		}
-		opts.StartOverride = &t
-	}
-	if f.DailySummariesEndDate != "" {
-		t, err := time.Parse("2006-01-02", f.DailySummariesEndDate)
-		if err != nil {
-			return opts, fmt.Errorf("invalid --daily-summaries-end %q: %w", f.DailySummariesEndDate, err)
-		}
-		opts.EndOverride = &t
-	}
-	if opts.StartOverride != nil && opts.EndOverride != nil && opts.StartOverride.After(*opts.EndOverride) {
-		return opts, fmt.Errorf("--daily-summaries-start (%s) is after --daily-summaries-end (%s)",
-			f.DailySummariesStartDate, f.DailySummariesEndDate)
-	}
-	return opts, nil
 }
 
 func NewRefreshCommand() *cobra.Command {
@@ -79,12 +46,9 @@ func NewRefreshCommand() *cobra.Command {
 			} else if cacheClient == nil {
 				logrus.Warn("no cache provided; refresh will not update cached timestamps, so cached data may not be properly invalidated")
 			}
-			dailySummaryOpts, err := f.dailySummaryOptions()
-			if err != nil {
-				return err
-			}
-			sippyserver.RefreshData(dbc, cacheClient, f.RefreshOnlyIfEmpty, dailySummaryOpts)
-			return nil
+			return sippyserver.RefreshData(dbc, cacheClient, sippyserver.RefreshOptions{
+				RefreshOnlyIfEmpty: f.RefreshOnlyIfEmpty,
+			})
 		},
 	}
 

cmd/sippy/seed_data.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 	"time"
 
+	"cloud.google.com/go/civil"
 	"github.com/lib/pq"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
@@ -23,7 +24,6 @@ import (
 	"github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts"
 	v1 "github.com/openshift/sippy/pkg/apis/sippyprocessing/v1"
 	"github.com/openshift/sippy/pkg/db"
-	"github.com/openshift/sippy/pkg/db/dailysummary"
 	"github.com/openshift/sippy/pkg/db/models"
 	"github.com/openshift/sippy/pkg/db/models/jobrunscan"
 	"github.com/openshift/sippy/pkg/flags"
@@ -439,8 +439,17 @@ func seedSyntheticData(dbc *db.DB) error {
 	}
 
 	log.Info("Refreshing materialized views...")
-	seedStart := time.Now().Add(-190 * 24 * time.Hour)
-	sippyserver.RefreshData(dbc, nil, false, dailysummary.Options{StartOverride: &seedStart})
+	seedToday := civil.DateOf(time.Now().UTC())
+	seedStart := seedToday.AddDays(-190)
+	seedEnd := seedToday
+	for _, table := range []string{"daily-summaries", "daily-totals", "cumulative-summaries"} {
+		if err := sippyserver.BackfillData(dbc, table, seedStart, seedEnd); err != nil {
+			return fmt.Errorf("failed to backfill %s: %w", table, err)
+		}
+	}
+	if err := sippyserver.RefreshData(dbc, nil, sippyserver.RefreshOptions{}); err != nil {
+		return fmt.Errorf("failed to refresh data: %w", err)
+	}
 
 	log.Info("Syncing regressions...")
 	if err := syncRegressions(dbc); err != nil {